/ Check-in [67d2a89e]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix bug in log recovery (last frame in log was being ignored). Also remove an incorrect assert statement.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal
Files: files | file ages | folders
SHA1: 67d2a89ec2d593a077eb19a6ea2b06cb1c2e9ba8
User & Date: dan 2010-04-16 11:30:18
Context
2010-04-16
13:59
Change the log file format to include a small (12 byte) header at the start of the file. check-in: 9865d14d user: dan tags: wal
11:30
Fix bug in log recovery (last frame in log was being ignored). Also remove an incorrect assert statement. check-in: 67d2a89e user: dan tags: wal
2010-04-15
16:45
Allow writers to write dirty pages to the log mid-transaction in order to free memory. check-in: ecd828f9 user: dan tags: wal
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/log.c.

     8      8   
     9      9   #include <unistd.h>
    10     10   #include <fcntl.h>
    11     11   #include <sys/mman.h>
    12     12   
    13     13   typedef struct LogSummaryHdr LogSummaryHdr;
    14     14   typedef struct LogSummary LogSummary;
    15         -typedef struct LogCheckpoint LogCheckpoint;
           15  +typedef struct LogIterator LogIterator;
    16     16   typedef struct LogLock LogLock;
    17     17   
    18     18   
    19     19   /*
    20     20   ** The following structure may be used to store the same data that
    21     21   ** is stored in the log-summary header.
    22     22   **
................................................................................
   105    105   ** This structure is used to implement an iterator that iterates through
   106    106   ** all frames in the log in database page order. Where two or more frames
   107    107   ** correspond to the same database page, the iterator visits only the 
   108    108   ** frame most recently written to the log.
   109    109   **
   110    110   ** The internals of this structure are only accessed by:
   111    111   **
   112         -**   logCheckpointInit() - Create a new iterator,
   113         -**   logCheckpointNext() - Step an iterator,
   114         -**   logCheckpointFree() - Free an iterator.
          112  +**   logIteratorInit() - Create a new iterator,
          113  +**   logIteratorNext() - Step an iterator,
          114  +**   logIteratorFree() - Free an iterator.
   115    115   **
   116    116   ** This functionality is used by the checkpoint code (see logCheckpoint()).
   117    117   */
   118         -struct LogCheckpoint {
   119         -  int nSegment;                   /* Size of LogCheckpoint.aSummary[] array */
          118  +struct LogIterator {
          119  +  int nSegment;                   /* Size of LogIterator.aSegment[] array */
   120    120     int nFinal;                     /* Elements in segment nSegment-1 */
   121    121     struct LogSegment {
   122    122       int iNext;                    /* Next aIndex index */
   123    123       u8 *aIndex;                   /* Pointer to index array */
   124    124       u32 *aDbPage;                 /* Pointer to db page array */
   125    125     } aSegment[1];
   126    126   };
................................................................................
   342    342     u32 *aCksum,                    /* IN/OUT: Checksum values */
   343    343     u32 *piPage,                    /* OUT: Database page number for frame */
   344    344     u32 *pnTruncate,                /* OUT: New db size (or 0 if not commit) */
   345    345     int nData,                      /* Database page size (size of aData[]) */
   346    346     u8 *aData,                      /* Pointer to page data (for checksum) */
   347    347     u8 *aFrame                      /* Frame data */
   348    348   ){
          349  +  assert( LOG_FRAME_HDRSIZE==20 );
          350  +
   349    351     logChecksumBytes(aFrame, 12, aCksum);
   350    352     logChecksumBytes(aData, nData, aCksum);
   351    353   
   352    354     if( aCksum[0]!=sqlite3Get4byte(&aFrame[12]) 
   353    355      || aCksum[1]!=sqlite3Get4byte(&aFrame[16]) 
   354    356     ){
   355    357       /* Checksum failed. */
................................................................................
   509    511         return SQLITE_NOMEM;
   510    512       }
   511    513       aData = &aFrame[LOG_FRAME_HDRSIZE];
   512    514   
   513    515       /* Read all frames from the log file. */
   514    516       iFrame = 0;
   515    517       iOffset = 0;
   516         -    for(iOffset=0; (iOffset+nFrame)<nSize; iOffset+=nFrame){
          518  +    for(iOffset=0; (iOffset+nFrame)<=nSize; iOffset+=nFrame){
   517    519         u32 pgno;                   /* Database page number for frame */
   518    520         u32 nTruncate;              /* dbsize field from frame header */
   519    521         int isValid;                /* True if this frame is valid */
   520    522   
   521    523         /* Read and decode the next log frame. */
   522    524         rc = sqlite3OsRead(pFd, aFrame, nFrame, iOffset);
   523    525         if( rc!=SQLITE_OK ) break;
................................................................................
   709    711       assert( !pSummary || pSummary->nRef==0 );
   710    712       sqlite3_free(pSummary);
   711    713     }
   712    714     *ppLog = pRet;
   713    715     return rc;
   714    716   }
   715    717   
   716         -static int logCheckpointNext(
   717         -  LogCheckpoint *p,               /* Iterator */
          718  +static int logIteratorNext(
          719  +  LogIterator *p,               /* Iterator */
   718    720     u32 *piPage,                    /* OUT: Next db page to write */
   719    721     u32 *piFrame                    /* OUT: Log frame to read from */
   720    722   ){
   721    723     u32 iMin = *piPage;
   722    724     u32 iRet = 0xFFFFFFFF;
   723    725     int i;
   724    726     int nBlock = p->nFinal;
................................................................................
   740    742       nBlock = 256;
   741    743     }
   742    744   
   743    745     *piPage = iRet;
   744    746     return (iRet==0xFFFFFFFF);
   745    747   }
   746    748   
   747         -static LogCheckpoint *logCheckpointInit(Log *pLog){
          749  +static LogIterator *logIteratorInit(Log *pLog){
   748    750     u32 *aData = pLog->pSummary->aData;
   749         -  LogCheckpoint *p;               /* Return value */
          751  +  LogIterator *p;                 /* Return value */
   750    752     int nSegment;                   /* Number of segments to merge */
   751    753     u32 iLast;                      /* Last frame in log */
   752    754     int nByte;                      /* Number of bytes to allocate */
   753    755     int i;                          /* Iterator variable */
   754    756     int nFinal;                     /* Number of unindexed entries */
   755    757     struct LogSegment *pFinal;      /* Final (unindexed) segment */
   756    758     u8 *aTmp;                       /* Temp space used by merge-sort */
   757    759   
   758    760     iLast = pLog->hdr.iLastPg;
   759    761     nSegment = (iLast >> 8) + 1;
   760    762     nFinal = (iLast & 0x000000FF);
   761    763   
   762         -  nByte = sizeof(LogCheckpoint) + (nSegment-1)*sizeof(struct LogSegment) + 512;
   763         -  p = (LogCheckpoint *)sqlite3_malloc(nByte);
          764  +  nByte = sizeof(LogIterator) + (nSegment-1)*sizeof(struct LogSegment) + 512;
          765  +  p = (LogIterator *)sqlite3_malloc(nByte);
   764    766     if( p ){
   765    767       memset(p, 0, nByte);
   766    768       p->nSegment = nSegment;
   767    769       p->nFinal = nFinal;
   768    770     }
   769    771   
   770    772     for(i=0; i<nSegment-1; i++){
................................................................................
   782    784     logMergesort8(pFinal->aDbPage, aTmp, pFinal->aIndex, &nFinal);
   783    785     p->nFinal = nFinal;
   784    786   
   785    787     return p;
   786    788   }
   787    789   
   788    790   /* 
   789         -** Free a log iterator allocated by logCheckpointInit().
          791  +** Free a log iterator allocated by logIteratorInit().
   790    792   */
   791         -static void logCheckpointFree(LogCheckpoint *p){
          793  +static void logIteratorFree(LogIterator *p){
   792    794     sqlite3_free(p);
   793    795   }
   794    796   
   795    797   /*
   796    798   ** Checkpoint the contents of the log file.
   797    799   */
   798    800   static int logCheckpoint(
   799    801     Log *pLog,                      /* Log connection */
   800    802     sqlite3_file *pFd,              /* File descriptor open on db file */
   801    803     u8 *zBuf                        /* Temporary buffer to use */
   802    804   ){
   803    805     int rc;                         /* Return code */
   804    806     int pgsz = pLog->hdr.pgsz;      /* Database page-size */
   805         -  LogCheckpoint *pIter = 0;       /* Log iterator context */
          807  +  LogIterator *pIter = 0;         /* Log iterator context */
   806    808     u32 iDbpage = 0;                /* Next database page to write */
   807    809     u32 iFrame = 0;                 /* Log frame containing data for iDbpage */
   808    810   
   809    811     if( pLog->hdr.iLastPg==0 ){
   810    812       return SQLITE_OK;
   811    813     }
   812    814   
   813    815     /* Allocate the iterator */
   814         -  pIter = logCheckpointInit(pLog);
          816  +  pIter = logIteratorInit(pLog);
   815    817     if( !pIter ) return SQLITE_NOMEM;
   816    818   
   817    819     /* Sync the log file to disk */
   818    820     rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags);
   819    821     if( rc!=SQLITE_OK ) goto out;
   820    822   
   821    823     /* Iterate through the contents of the log, copying data to the db file. */
   822         -  while( 0==logCheckpointNext(pIter, &iDbpage, &iFrame) ){
          824  +  while( 0==logIteratorNext(pIter, &iDbpage, &iFrame) ){
   823    825       rc = sqlite3OsRead(pLog->pFd, zBuf, pgsz, 
   824    826           (iFrame-1) * (pgsz+LOG_FRAME_HDRSIZE) + LOG_FRAME_HDRSIZE
   825    827       );
   826    828       if( rc!=SQLITE_OK ) goto out;
   827    829       rc = sqlite3OsWrite(pFd, zBuf, pgsz, (iDbpage-1)*pgsz);
   828    830       if( rc!=SQLITE_OK ) goto out;
   829    831     }
................................................................................
   857    859     memset(zBuf, 0, LOG_FRAME_HDRSIZE);
   858    860     rc = sqlite3OsWrite(pLog->pFd, zBuf, LOG_FRAME_HDRSIZE, 0);
   859    861     if( rc!=SQLITE_OK ) goto out;
   860    862     rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags);
   861    863   #endif
   862    864   
   863    865    out:
   864         -  logCheckpointFree(pIter);
          866  +  logIteratorFree(pIter);
   865    867     return rc;
   866    868   }
   867    869   
   868    870   /*
   869    871   ** Close a connection to a log file.
   870    872   */
   871    873   int sqlite3LogClose(

Changes to src/pager.c.

  3249   3249   static int pagerStress(void *p, PgHdr *pPg){
  3250   3250     Pager *pPager = (Pager *)p;
  3251   3251     int rc = SQLITE_OK;
  3252   3252   
  3253   3253     assert( pPg->pPager==pPager );
  3254   3254     assert( pPg->flags&PGHDR_DIRTY );
  3255   3255   
         3256  +  pPg->pDirty = 0;
  3256   3257     if( pagerUseLog(pPager) ){
  3257   3258       /* Write a single frame for this page to the log. */
  3258         -    assert( pPg->pDirty==0 );
  3259   3259       rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pPg, 0, 0, 0);
  3260   3260     }else{
  3261   3261       /* The doNotSync flag is set by the sqlite3PagerWrite() function while it
  3262   3262       ** is journalling a set of two or more database pages that are stored
  3263   3263       ** on the same disk sector. Syncing the journal is not allowed while
  3264   3264       ** this is happening as it is important that all members of such a
  3265   3265       ** set of pages are synced to disk together. So, if the page this function
................................................................................
  3320   3320           rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg)
  3321   3321       ) ){
  3322   3322         rc = subjournalPage(pPg);
  3323   3323       }
  3324   3324     
  3325   3325       /* Write the contents of the page out to the database file. */
  3326   3326       if( rc==SQLITE_OK ){
  3327         -      pPg->pDirty = 0;
  3328   3327         rc = pager_write_pagelist(pPg);
  3329   3328       }
  3330   3329     }
  3331   3330   
  3332   3331     /* Mark the page as clean. */
  3333   3332     if( rc==SQLITE_OK ){
  3334   3333       PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno));

Changes to test/wal.test.

   616    616       PRAGMA integrity_check;
   617    617     }
   618    618   } {17 ok}
   619    619   do_test wal-11.14 {
   620    620     list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044]
   621    621   } {37 38}
   622    622   
          623  +
          624  +#-------------------------------------------------------------------------
          625  +# This block of tests, wal-12.*, tests a problem...
          626  +#
          627  +reopen_db
          628  +do_test wal-12.1 {
          629  +  execsql {
          630  +    PRAGMA page_size = 1024;
          631  +    CREATE TABLE t1(x, y);
          632  +    CREATE TABLE t2(x, y);
          633  +    INSERT INTO t1 VALUES('A', 1);
          634  +  }
          635  +  list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044]
          636  +} {0 5}
          637  +do_test wal-12.2 {
          638  +  db close
          639  +  sqlite3_wal db test.db
          640  +  execsql {
          641  +    UPDATE t1 SET y = 0 WHERE x = 'A';
          642  +  }
          643  +  list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044]
          644  +} {3 1}
          645  +do_test wal-12.3 {
          646  +  execsql { INSERT INTO t2 VALUES('B', 1) }
          647  +  list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044]
          648  +} {3 2}
          649  +
          650  +do_test wal-12.4 {
          651  +  file copy -force test.db test2.db
          652  +  file copy -force test.db-wal test2.db-wal
          653  +  sqlite3_wal db2 test2.db
          654  +breakpoint
          655  +  execsql { SELECT * FROM t2 } db2
          656  +} {B 1}
          657  +db2 close
          658  +
          659  +file copy -force test.db-wal A
          660  +do_test wal-12.5 {
          661  +  execsql {
          662  +    PRAGMA checkpoint;
          663  +    UPDATE t2 SET y = 2 WHERE x = 'B'; 
          664  +    PRAGMA checkpoint;
          665  +    UPDATE t1 SET y = 1 WHERE x = 'A';
          666  +    PRAGMA checkpoint;
          667  +    UPDATE t1 SET y = 0 WHERE x = 'A';
          668  +    SELECT * FROM t2;
          669  +  }
          670  +} {B 2}
          671  +file copy -force test.db-wal B
          672  +
          673  +do_test wal-12.4 {
          674  +  file copy -force test.db test2.db
          675  +  file copy -force test.db-wal test2.db-wal
          676  +  sqlite3_wal db2 test2.db
          677  +  execsql { SELECT * FROM t2 } db2
          678  +} {B 2}
          679  +db2 close
          680  +
   623    681   
   624    682   finish_test
   625    683