/ Check-in [00f08fc0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:In the pager, cache a pointer to the first page on the freelist that does not need to be synced. This makes a fetch of a page that is not in cache go a lot faster when the cache is full. This check-in also adds some performance instrumentation to the OS layer. (CVS 842)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 00f08fc0b5b6b9c5efbf15a62f9a1cc1cfa71283
User & Date: drh 2003-01-21 02:39:37
Context
2003-01-21
23:06
fix a typo on the quickstart.html page. (CVS 843) check-in: 61869bb5 user: drh tags: trunk
02:39
In the pager, cache a pointer to the first page on the freelist that does not need to be synced. This makes a fetch of a page that is not in cache go a lot faster when the cache is full. This check-in also adds some performance instrumentation to the OS layer. (CVS 842) check-in: 00f08fc0 user: drh tags: trunk
2003-01-19
03:59
Update comments. No changes to code. (CVS 841) check-in: f6a87068 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/os.c.

    48     48   #endif
    49     49   
    50     50   /*
    51     51   ** Macros for performance tracing.  Normally turned off
    52     52   */
    53     53   #if 0
    54     54   static int last_page = 0;
    55         -#define SEEK(X)       last_page=(X)
    56         -#define TRACE1(X)     fprintf(stderr,X)
    57         -#define TRACE2(X,Y)   fprintf(stderr,X,Y)
    58         -#define TRACE3(X,Y,Z) fprintf(stderr,X,Y,Z)
           55  +__inline__ unsigned long long int hwtime(void){
           56  +  unsigned long long int x;
           57  +  __asm__("rdtsc\n\t"
           58  +          "mov %%edx, %%ecx\n\t"
           59  +          :"=A" (x));
           60  +  return x;
           61  +}
           62  +static unsigned long long int g_start;
           63  +static unsigned int elapse;
           64  +#define TIMER_START       g_start=hwtime()
           65  +#define TIMER_END         elapse=hwtime()-g_start
           66  +#define SEEK(X)           last_page=(X)
           67  +#define TRACE1(X)         fprintf(stderr,X)
           68  +#define TRACE2(X,Y)       fprintf(stderr,X,Y)
           69  +#define TRACE3(X,Y,Z)     fprintf(stderr,X,Y,Z)
           70  +#define TRACE4(X,Y,Z,A)   fprintf(stderr,X,Y,Z,A)
           71  +#define TRACE5(X,Y,Z,A,B) fprintf(stderr,X,Y,Z,A,B)
    59     72   #else
           73  +#define TIMER_START
           74  +#define TIMER_END
    60     75   #define SEEK(X)
    61     76   #define TRACE1(X)
    62     77   #define TRACE2(X,Y)
    63     78   #define TRACE3(X,Y,Z)
           79  +#define TRACE4(X,Y,Z,A)
           80  +#define TRACE5(X,Y,Z,A,B)
    64     81   #endif
    65     82   
    66     83   
    67     84   #if OS_UNIX
    68     85   /*
    69     86   ** Here is the dirt on POSIX advisory locks:  ANSI STD 1003.1 (1996)
    70     87   ** section 6.5.2.2 lines 483 through 490 specify that when a process
................................................................................
   660    677   ** bytes were read successfully and SQLITE_IOERR if anything goes
   661    678   ** wrong.
   662    679   */
   663    680   int sqliteOsRead(OsFile *id, void *pBuf, int amt){
   664    681   #if OS_UNIX
   665    682     int got;
   666    683     SimulateIOError(SQLITE_IOERR);
   667         -  TRACE3("READ    %-3d %d\n", id->fd, last_page);
          684  +  TIMER_START;
   668    685     got = read(id->fd, pBuf, amt);
          686  +  TIMER_END;
          687  +  TRACE4("READ    %-3d %7d %d\n", id->fd, last_page, elapse);
          688  +  SEEK(0);
   669    689     /* if( got<0 ) got = 0; */
   670    690     if( got==amt ){
   671    691       return SQLITE_OK;
   672    692     }else{
   673    693       return SQLITE_IOERR;
   674    694     }
   675    695   #endif
................................................................................
   708    728   ** Write data from a buffer into a file.  Return SQLITE_OK on success
   709    729   ** or some other error code on failure.
   710    730   */
   711    731   int sqliteOsWrite(OsFile *id, const void *pBuf, int amt){
   712    732   #if OS_UNIX
   713    733     int wrote = 0;
   714    734     SimulateIOError(SQLITE_IOERR);
   715         -  TRACE3("WRITE   %-3d %d\n", id->fd, last_page);
          735  +  TIMER_START;
   716    736     while( amt>0 && (wrote = write(id->fd, pBuf, amt))>0 ){
   717    737       amt -= wrote;
   718    738       pBuf = &((char*)pBuf)[wrote];
   719    739     }
          740  +  TIMER_END;
          741  +  TRACE4("WRITE   %-3d %7d %d\n", id->fd, last_page, elapse);
          742  +  SEEK(0);
   720    743     if( amt>0 ){
   721    744       return SQLITE_FULL;
   722    745     }
   723    746     return SQLITE_OK;
   724    747   #endif
   725    748   #if OS_WIN
   726    749     int rc;

Changes to src/pager.c.

    14     14   ** The pager is used to access a database disk file.  It implements
    15     15   ** atomic commit and rollback through the use of a journal file that
    16     16   ** is separate from the database file.  The pager also implements file
    17     17   ** locking to prevent two processes from writing the same database
    18     18   ** file simultaneously, or one process from reading the database while
    19     19   ** another is writing.
    20     20   **
    21         -** @(#) $Id: pager.c,v 1.68 2003/01/16 13:42:43 drh Exp $
           21  +** @(#) $Id: pager.c,v 1.69 2003/01/21 02:39:37 drh Exp $
    22     22   */
    23     23   #include "os.h"         /* Must be first to enable large file support */
    24     24   #include "sqliteInt.h"
    25     25   #include "pager.h"
    26     26   #include <assert.h>
    27     27   #include <string.h>
    28     28   
................................................................................
   158    158     u8 needSync;                /* True if an fsync() is needed on the journal */
   159    159     u8 dirtyFile;               /* True if database file has changed in any way */
   160    160     u8 alwaysRollback;          /* Disable dont_rollback() for all pages */
   161    161     u8 journalFormat;           /* Version number of the journal file */
   162    162     u8 *aInJournal;             /* One bit for each page in the database file */
   163    163     u8 *aInCkpt;                /* One bit for each page in the database */
   164    164     PgHdr *pFirst, *pLast;      /* List of free pages */
          165  +  PgHdr *pFirstSynced;        /* First free page with PgHdr.needSync==0 */
   165    166     PgHdr *pAll;                /* List of all pages */
   166    167     PgHdr *pCkpt;               /* List of pages in the checkpoint journal */
   167    168     PgHdr *aHash[N_PG_HASH];    /* Hash table to map page number of PgHdr */
   168    169   };
   169    170   
   170    171   /*
   171    172   ** These are bits that can be set in Pager.errMask.
................................................................................
   342    343   static void pager_reset(Pager *pPager){
   343    344     PgHdr *pPg, *pNext;
   344    345     for(pPg=pPager->pAll; pPg; pPg=pNext){
   345    346       pNext = pPg->pNextAll;
   346    347       sqliteFree(pPg);
   347    348     }
   348    349     pPager->pFirst = 0;
          350  +  pPager->pFirstSynced = 0;
   349    351     pPager->pLast = 0;
   350    352     pPager->pAll = 0;
   351    353     memset(pPager->aHash, 0, sizeof(pPager->aHash));
   352    354     pPager->nPage = 0;
   353    355     if( pPager->state>=SQLITE_WRITELOCK ){
   354    356       sqlitepager_rollback(pPager);
   355    357     }
................................................................................
   733    735     pPager->state = SQLITE_UNLOCK;
   734    736     pPager->errMask = 0;
   735    737     pPager->tempFile = tempFile;
   736    738     pPager->readOnly = readOnly;
   737    739     pPager->needSync = 0;
   738    740     pPager->noSync = pPager->tempFile || !useJournal;
   739    741     pPager->pFirst = 0;
          742  +  pPager->pFirstSynced = 0;
   740    743     pPager->pLast = 0;
   741    744     pPager->nExtra = nExtra;
   742    745     memset(pPager->aHash, 0, sizeof(pPager->aHash));
   743    746     *ppPager = pPager;
   744    747     return SQLITE_OK;
   745    748   }
   746    749   
................................................................................
   833    836   ** currently on the freelist (the reference count is zero) then
   834    837   ** remove it from the freelist.
   835    838   */
   836    839   #define page_ref(P)   ((P)->nRef==0?_page_ref(P):(void)(P)->nRef++)
   837    840   static void _page_ref(PgHdr *pPg){
   838    841     if( pPg->nRef==0 ){
   839    842       /* The page is currently on the freelist.  Remove it. */
          843  +    if( pPg==pPg->pPager->pFirstSynced ){
          844  +      PgHdr *p = pPg->pNextFree;
          845  +      while( p && p->needSync ){ p = p->pNextFree; }
          846  +      pPg->pPager->pFirstSynced = p;
          847  +    }
   840    848       if( pPg->pPrevFree ){
   841    849         pPg->pPrevFree->pNextFree = pPg->pNextFree;
   842    850       }else{
   843    851         pPg->pPager->pFirst = pPg->pNextFree;
   844    852       }
   845    853       if( pPg->pNextFree ){
   846    854         pPg->pNextFree->pPrevFree = pPg->pPrevFree;
................................................................................
   897    905   #ifndef NDEBUG
   898    906         rc = sqliteOsFileSize(&pPager->jfd, &pPager->syncJSize);
   899    907         if( rc!=0 ) return rc;
   900    908   #endif
   901    909         pPager->journalStarted = 1;
   902    910       }
   903    911       pPager->needSync = 0;
          912  +
          913  +    /* Erase the needSync flag from every page.
          914  +    */
          915  +    for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
          916  +      pPg->needSync = 0;
          917  +    }
          918  +    pPager->pFirstSynced = pPager->pFirst;
   904    919     }
   905    920   
   906         -  /* Erase the needSync flag from every page.
          921  +#ifndef NDEBUG
          922  +  /* If the Pager.needSync flag is clear then the PgHdr.needSync
          923  +  ** flag must also be clear for all pages.  Verify that this
          924  +  ** invariant is true.
   907    925     */
   908         -  for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
   909         -    pPg->needSync = 0;
          926  +  else{
          927  +    for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
          928  +      assert( pPg->needSync==0 );
          929  +    }
          930  +    assert( pPager->pFirstSynced==pPager->pFirst );
   910    931     }
          932  +#endif
          933  +  
          934  +
   911    935   
   912    936     return rc;
   913    937   }
   914    938   
   915    939   /*
   916    940   ** Acquire a page.
   917    941   **
................................................................................
  1028   1052         pPg->pPrevAll = 0;
  1029   1053         pPager->pAll = pPg;
  1030   1054         pPager->nPage++;
  1031   1055       }else{
  1032   1056         /* Find a page to recycle.  Try to locate a page that does not
  1033   1057         ** require us to do an fsync() on the journal.
  1034   1058         */
  1035         -      pPg = pPager->pFirst;
  1036         -      while( pPg && pPg->needSync ){
  1037         -        pPg = pPg->pNextFree;
  1038         -      }
         1059  +      pPg = pPager->pFirstSynced;
  1039   1060   
  1040   1061         /* If we could not find a page that does not require an fsync()
  1041   1062         ** on the journal file then fsync the journal file.  This is a
  1042   1063         ** very slow operation, so we work hard to avoid it.  But sometimes
  1043   1064         ** it can't be helped.
  1044   1065         */
  1045   1066         if( pPg==0 ){
................................................................................
  1079   1100         */
  1080   1101         if( pPg->alwaysRollback ){
  1081   1102           pPager->alwaysRollback = 1;
  1082   1103         }
  1083   1104   
  1084   1105         /* Unlink the old page from the free list and the hash table
  1085   1106         */
         1107  +      if( pPg==pPager->pFirstSynced ){
         1108  +        PgHdr *p = pPg->pNextFree;
         1109  +        while( p && p->needSync ){ p = p->pNextFree; }
         1110  +        pPager->pFirstSynced = p;
         1111  +      }
  1086   1112         if( pPg->pPrevFree ){
  1087   1113           pPg->pPrevFree->pNextFree = pPg->pNextFree;
  1088   1114         }else{
  1089   1115           assert( pPager->pFirst==pPg );
  1090   1116           pPager->pFirst = pPg->pNextFree;
  1091   1117         }
  1092   1118         if( pPg->pNextFree ){
................................................................................
  1221   1247       pPg->pNextFree = 0;
  1222   1248       pPg->pPrevFree = pPager->pLast;
  1223   1249       pPager->pLast = pPg;
  1224   1250       if( pPg->pPrevFree ){
  1225   1251         pPg->pPrevFree->pNextFree = pPg;
  1226   1252       }else{
  1227   1253         pPager->pFirst = pPg;
         1254  +    }
         1255  +    if( pPg->needSync==0 && pPager->pFirstSynced==0 ){
         1256  +      pPager->pFirstSynced = pPg;
  1228   1257       }
  1229   1258       if( pPager->xDestructor ){
  1230   1259         pPager->xDestructor(pData);
  1231   1260       }
  1232   1261     
  1233   1262       /* When all pages reach the freelist, drop the read lock from
  1234   1263       ** the database file.
................................................................................
  1573   1602     if( pPager->state!=SQLITE_WRITELOCK ){
  1574   1603       return SQLITE_ERROR;
  1575   1604     }
  1576   1605     TRACE1("COMMIT\n");
  1577   1606     if( pPager->dirtyFile==0 ){
  1578   1607       /* Exit early (without doing the time-consuming sqliteOsSync() calls)
  1579   1608       ** if there have been no changes to the database file. */
         1609  +    assert( pPager->needSync==0 );
  1580   1610       rc = pager_unwritelock(pPager);
  1581   1611       pPager->dbSize = -1;
  1582   1612       return rc;
  1583   1613     }
  1584   1614     assert( pPager->journalOpen );
  1585         -  if( !pPager->journalStarted && !pPager->noSync ) pPager->needSync = 1;
  1586         -  assert( pPager->dirtyFile || !pPager->needSync );
  1587   1615     if( pPager->needSync && sqliteOsSync(&pPager->jfd)!=SQLITE_OK ){
  1588   1616       goto commit_abort;
  1589   1617     }
  1590   1618     dbChanged = 0;
  1591   1619     for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
  1592   1620       if( pPg->dirty==0 ) continue;
  1593   1621       TRACE2("COMMIT-PAGE %d\n", pPg->pgno);