/ Check-in [6a944f02]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes for problems with small caches and SAVEPOINT rollback in WAL mode.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal
Files: files | file ages | folders
SHA1: 6a944f028d4a070bef29e1fbc6fbef481ebcd34c
User & Date: dan 2010-04-26 16:57:10
Context
2010-04-26
17:42
Do not attempt to set journal_mode=wal on :memory: or temp file databases. check-in: 30d01344 user: dan tags: wal
16:57
Fixes for problems with small caches and SAVEPOINT rollback in WAL mode. check-in: 6a944f02 user: dan tags: wal
12:39
Add the "wal" permutation to run existing test files savepoint.test and savepoint2.test in WAL mode. check-in: 205e5d8a user: dan tags: wal
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/pager.c.

   217    217   typedef struct PagerSavepoint PagerSavepoint;
   218    218   struct PagerSavepoint {
   219    219     i64 iOffset;                 /* Starting offset in main journal */
   220    220     i64 iHdrOffset;              /* See above */
   221    221     Bitvec *pInSavepoint;        /* Set of pages in this savepoint */
   222    222     Pgno nOrig;                  /* Original number of pages in file */
   223    223     Pgno iSubRec;                /* Index of first record in sub-journal */
          224  +  u32 iFrame;                  /* Last frame in WAL when savepoint opened */
   224    225   };
   225    226   
   226    227   /*
   227    228   ** A open page cache is an instance of the following structure.
   228    229   **
   229    230   ** errCode
   230    231   **
................................................................................
  1535   1536     assert( (isMainJrnl&~1)==0 );      /* isMainJrnl is 0 or 1 */
  1536   1537     assert( (isSavepnt&~1)==0 );       /* isSavepnt is 0 or 1 */
  1537   1538     assert( isMainJrnl || pDone );     /* pDone always used on sub-journals */
  1538   1539     assert( isSavepnt || pDone==0 );   /* pDone never used on non-savepoint */
  1539   1540   
  1540   1541     aData = pPager->pTmpSpace;
  1541   1542     assert( aData );         /* Temp storage must have already been allocated */
         1543  +  assert( pagerUseLog(pPager)==0 || (!isMainJrnl && isSavepnt) );
  1542   1544   
  1543   1545     /* Read the page number and page data from the journal or sub-journal
  1544   1546     ** file. Return an error code to the caller if an IO error occurs.
  1545   1547     */
  1546   1548     jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
  1547   1549     rc = read32bits(jfd, *pOffset, &pgno);
  1548   1550     if( rc!=SQLITE_OK ) return rc;
................................................................................
  1604   1606     ** in the main journal either because the page is not in cache or else
  1605   1607     ** the page is marked as needSync==0.
  1606   1608     **
  1607   1609     ** 2008-04-14:  When attempting to vacuum a corrupt database file, it
  1608   1610     ** is possible to fail a statement on a database that does not yet exist.
  1609   1611     ** Do not attempt to write if database file has never been opened.
  1610   1612     */
  1611         -  pPg = pager_lookup(pPager, pgno);
         1613  +  if( pagerUseLog(pPager) ){
         1614  +    pPg = 0;
         1615  +  }else{
         1616  +    pPg = pager_lookup(pPager, pgno);
         1617  +  }
  1612   1618     assert( pPg || !MEMDB );
  1613   1619     PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
  1614   1620              PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData),
  1615   1621              (isMainJrnl?"main-journal":"sub-journal")
  1616   1622     ));
  1617   1623     if( isMainJrnl ){
  1618   1624       isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr);
................................................................................
  1621   1627     }
  1622   1628     if( (pPager->state>=PAGER_EXCLUSIVE)
  1623   1629      && isOpen(pPager->fd)
  1624   1630      && isSynced
  1625   1631     ){
  1626   1632       i64 ofst = (pgno-1)*(i64)pPager->pageSize;
  1627   1633       testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 );
         1634  +    assert( !pagerUseLog(pPager) );
  1628   1635       rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst);
  1629   1636       if( pgno>pPager->dbFileSize ){
  1630   1637         pPager->dbFileSize = pgno;
  1631   1638       }
  1632   1639       if( pPager->pBackup ){
  1633   1640         CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM);
  1634   1641         sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData);
................................................................................
  1685   1692         ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to
  1686   1693         ** again within this transaction, it will be marked as dirty but
  1687   1694         ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially
  1688   1695         ** be written out into the database file before its journal file
  1689   1696         ** segment is synced. If a crash occurs during or following this,
  1690   1697         ** database corruption may ensue.
  1691   1698         */
         1699  +      assert( !pagerUseLog(pPager) );
  1692   1700         sqlite3PcacheMakeClean(pPg);
  1693   1701       }
  1694   1702   #ifdef SQLITE_CHECK_PAGES
  1695   1703       pPg->pageHash = pager_pagehash(pPg);
  1696   1704   #endif
  1697   1705       /* If this was page 1, then restore the value of Pager.dbFileVers.
  1698   1706       ** Do this before any decoding. */
................................................................................
  2424   2432     /* Finally,  rollback pages from the sub-journal.  Page that were
  2425   2433     ** previously rolled back out of the main journal (and are hence in pDone)
  2426   2434     ** will be skipped.  Out-of-range pages are also skipped.
  2427   2435     */
  2428   2436     if( pSavepoint ){
  2429   2437       u32 ii;            /* Loop counter */
  2430   2438       i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize);
         2439  +
         2440  +    if( pagerUseLog(pPager) ){
         2441  +      rc = sqlite3WalSavepointUndo(pPager->pLog, pSavepoint->iFrame);
         2442  +    }
  2431   2443       for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && ii<pPager->nSubRec; ii++){
  2432   2444         assert( offset==ii*(4+pPager->pageSize) );
  2433   2445         rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1);
  2434   2446       }
  2435   2447       assert( rc!=SQLITE_DONE );
  2436   2448     }
  2437   2449   
  2438   2450     sqlite3BitvecDestroy(pDone);
  2439   2451     if( rc==SQLITE_OK ){
  2440   2452       pPager->journalOff = szJ;
  2441   2453     }
         2454  +
  2442   2455     return rc;
  2443   2456   }
  2444   2457   
  2445   2458   /*
  2446   2459   ** Change the maximum number of in-memory pages that are allowed.
  2447   2460   */
  2448   2461   void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){
................................................................................
  3322   3335   
  3323   3336     assert( pPg->pPager==pPager );
  3324   3337     assert( pPg->flags&PGHDR_DIRTY );
  3325   3338   
  3326   3339     pPg->pDirty = 0;
  3327   3340     if( pagerUseLog(pPager) ){
  3328   3341       /* Write a single frame for this page to the log. */
  3329         -    rc = pagerLogFrames(pPager, pPg, 0, 0, 0);
         3342  +    if( subjRequiresPage(pPg) ){ 
         3343  +      rc = subjournalPage(pPg); 
         3344  +    }
         3345  +    if( rc==SQLITE_OK ){
         3346  +      rc = pagerLogFrames(pPager, pPg, 0, 0, 0);
         3347  +    }
  3330   3348     }else{
  3331   3349       /* The doNotSync flag is set by the sqlite3PagerWrite() function while it
  3332   3350       ** is journalling a set of two or more database pages that are stored
  3333   3351       ** on the same disk sector. Syncing the journal is not allowed while
  3334   3352       ** this is happening as it is important that all members of such a
  3335   3353       ** set of pages are synced to disk together. So, if the page this function
  3336   3354       ** is trying to make clean will require a journal sync and the doNotSync
................................................................................
  5338   5356           aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager);
  5339   5357         }
  5340   5358         aNew[ii].iSubRec = pPager->nSubRec;
  5341   5359         aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage);
  5342   5360         if( !aNew[ii].pInSavepoint ){
  5343   5361           return SQLITE_NOMEM;
  5344   5362         }
         5363  +      if( pagerUseLog(pPager) ){
         5364  +        aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pLog);
         5365  +      }
  5345   5366       }
  5346   5367   
  5347   5368       /* Open the sub-journal, if it is not already opened. */
  5348   5369       rc = openSubJournal(pPager);
  5349   5370       assertTruncateConstraint(pPager);
  5350   5371     }
  5351   5372   

Changes to src/wal.c.

  1664   1664     assert( pLog->isWriteLocked );
  1665   1665     logSummaryReadHdr(pLog, 0);
  1666   1666     for(iFrame=pLog->hdr.iLastPg+1; iFrame<=iMax && rc==SQLITE_OK; iFrame++){
  1667   1667       rc = xUndo(pUndoCtx, pLog->pSummary->aData[logSummaryEntry(iFrame)]);
  1668   1668     }
  1669   1669     return rc;
  1670   1670   }
         1671  +
         1672  +u32 sqlite3WalSavepoint(Log *pLog){
         1673  +  assert( pLog->isWriteLocked );
         1674  +  return pLog->hdr.iLastPg;
         1675  +}
         1676  +
         1677  +int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame){
         1678  +  int rc = SQLITE_OK;
         1679  +  u8 aCksum[8];
         1680  +  assert( pLog->isWriteLocked );
         1681  +
         1682  +  pLog->hdr.iLastPg = iFrame;
         1683  +  if( iFrame>0 ){
         1684  +    i64 iOffset = logFrameOffset(iFrame, pLog->hdr.pgsz) + sizeof(u32)*2;
         1685  +    rc = sqlite3OsRead(pLog->pFd, aCksum, sizeof(aCksum), iOffset);
         1686  +    pLog->hdr.iCheck1 = sqlite3Get4byte(&aCksum[0]);
         1687  +    pLog->hdr.iCheck2 = sqlite3Get4byte(&aCksum[4]);
         1688  +  }
         1689  +
         1690  +  return rc;
         1691  +}
  1671   1692   
  1672   1693   /* 
  1673   1694   ** Return true if data has been written but not committed to the log file. 
  1674   1695   */
  1675   1696   int sqlite3WalDirty(Log *pLog){
  1676   1697     assert( pLog->isWriteLocked );
  1677   1698     return( pLog->hdr.iLastPg!=((LogSummaryHdr*)pLog->pSummary->aData)->iLastPg );
  1678   1699   }
  1679   1700   
  1680   1701   /* 
  1681         -** Write a set of frames to the log. The caller must hold at least a
  1682         -** RESERVED lock on the database file.
         1702  +** Write a set of frames to the log. The caller must hold the write-lock
         1703  +** on the log file (obtained using sqlite3WalWriteLock()).
  1683   1704   */
  1684   1705   int sqlite3WalFrames(
  1685   1706     Log *pLog,                      /* Log handle to write to */
  1686   1707     int nPgsz,                      /* Database page-size in bytes */
  1687   1708     PgHdr *pList,                   /* List of dirty pages to write */
  1688   1709     Pgno nTruncate,                 /* Database size after this commit */
  1689   1710     int isCommit,                   /* True if this is a commit */

Changes to src/wal.h.

    36     36   
    37     37   /* Obtain or release the WRITER lock. */
    38     38   int sqlite3WalWriteLock(Log *pLog, int op);
    39     39   
    40     40   /* Undo any frames written (but not committed) to the log */
    41     41   int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx);
    42     42   
           43  +u32 sqlite3WalSavepoint(Log *pLog);
           44  +int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame);
           45  +
    43     46   /* Return true if data has been written but not committed to the log file. */
    44     47   int sqlite3WalDirty(Log *pLog);
    45     48   
    46     49   /* Write a frame or frames to the log. */
    47     50   int sqlite3WalFrames(Log *pLog, int, PgHdr *, Pgno, int, int);
    48     51   
    49     52   /* Copy pages from the log to the database file */ 

Changes to test/permutations.test.

   757    757   }
   758    758   
   759    759   run_tests "wal" -description {
   760    760     Run tests with journal_mode=WAL
   761    761   } -include {
   762    762     savepoint.test
   763    763     savepoint2.test
          764  +  savepoint6.test
   764    765   }
   765    766   
   766    767   # End of tests
   767    768   #############################################################################
   768    769   
   769    770   if {$::perm::testmode eq "targets"} { puts "" ; exit }
   770    771   

Changes to test/savepoint6.test.

   218    218     execsql {SELECT count(*) FROM t1}
   219    219   } {44}
   220    220   
   221    221   foreach zSetup [list {
   222    222     set testname normal
   223    223     sqlite3 db test.db
   224    224   } {
          225  +  if {[wal_is_wal_mode]} continue
   225    226     set testname tempdb
   226    227     sqlite3 db ""
   227    228   } {
   228    229     if {[catch {set ::permutations_test_prefix} z] == 0 && $z eq "journaltest"} {
   229    230       continue
   230    231     }
   231    232     set testname nosync
................................................................................
   237    238     sql { PRAGMA cache_size = 10 }
   238    239   }] {
   239    240   
   240    241     unset -nocomplain ::lSavepoint
   241    242     unset -nocomplain ::aEntry
   242    243   
   243    244     catch { db close }
   244         -  file delete -force test.db
          245  +  file delete -force test.db test.db-wal test.db-journal
   245    246     eval $zSetup
   246    247     sql $DATABASE_SCHEMA
          248  +
          249  +  wal_set_journal_mode
   247    250   
   248    251     do_test savepoint6-$testname.setup {
   249    252       savepoint one
   250    253       insert_rows [random_integers 100 1000]
   251    254       release one
   252    255       checkdb
   253    256     } {ok}
   254    257     
   255         -  for {set i 0} {$i < 1000} {incr i} {
          258  +  for {set i 0} {$i < 50} {incr i} {
   256    259       do_test savepoint6-$testname.$i.1 {
   257    260         savepoint_op
   258    261         checkdb
   259    262       } {ok}
   260    263     
   261    264       do_test savepoint6-$testname.$i.2 {
   262    265         database_op
   263    266         database_op
   264    267         checkdb
   265    268       } {ok}
   266    269     }
          270  +
          271  +  wal_check_journal_mode savepoint6-$testname.walok
   267    272   }
   268    273   
   269    274   unset -nocomplain ::lSavepoint
   270    275   unset -nocomplain ::aEntry
   271    276   
   272    277   finish_test

Changes to test/wal.test.

   162    162   do_test wal-4.3 {
   163    163     execsql {
   164    164       COMMIT;
   165    165       SELECT * FROM t1;
   166    166     }
   167    167   } {a b}
   168    168   
   169         -do_test wal-4.4 {
          169  +do_test wal-4.4.1 {
   170    170     db close
   171    171     sqlite3 db test.db
   172    172     db func blob blob
   173    173     list [execsql { SELECT * FROM t1 }] [file size test.db-wal]
   174    174   } {{a b} 0}
   175         -do_test wal-4.5 {
          175  +do_test wal-4.4.2 {
          176  +  execsql { PRAGMA cache_size = 10 }
          177  +  execsql {
          178  +    CREATE TABLE t2(a, b);
          179  +    INSERT INTO t2 VALUES(blob(400), blob(400));
          180  +    SAVEPOINT tr;
          181  +      INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /*  2 */
          182  +      INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /*  4 */
          183  +      INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /*  8 */
          184  +      INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 16 */
          185  +      INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 32 */
          186  +      INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /*  2 */
          187  +      INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /*  4 */
          188  +      INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /*  8 */
          189  +      INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 16 */
          190  +      INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 32 */
          191  +      SELECT count(*) FROM t2;
          192  +  }
          193  +} {32}
          194  +do_test wal-4.4.3 {
          195  +  execsql { ROLLBACK TO tr }
          196  +} {}
          197  +do_test wal-4.4.4 {
          198  +  set logsize [file size test.db-wal]
          199  +  execsql {
          200  +      INSERT INTO t1 VALUES('x', 'y');
          201  +    RELEASE tr;
          202  +  }
          203  +  expr { $logsize == [file size test.db-wal] }
          204  +} {1}
          205  +do_test wal-4.4.5 {
          206  +  execsql { SELECT count(*) FROM t2 }
          207  +} {1}
          208  +do_test wal-4.4.6 {
          209  +  file copy -force test.db test2.db
          210  +  file copy -force test.db-wal test2.db-wal
          211  +  sqlite3 db2 test2.db
          212  +  execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } db2
          213  +} {1 2}
          214  +do_test wal-4.4.7 {
          215  +  execsql { PRAGMA integrity_check } db2
          216  +} {ok}
          217  +db2 close
          218  +
          219  +do_test wal-4.5.1 {
          220  +  reopen_db
          221  +  db func blob blob
          222  +  execsql {
          223  +    PRAGMA journal_mode = WAL;
          224  +    CREATE TABLE t1(a, b);
          225  +    INSERT INTO t1 VALUES('a', 'b');
          226  +  }
          227  +  sqlite3 db test.db
          228  +  db func blob blob
          229  +  list [execsql { SELECT * FROM t1 }] [file size test.db-wal]
          230  +} {{a b} 0}
          231  +do_test wal-4.5.2 {
   176    232     execsql { PRAGMA cache_size = 10 }
   177    233     execsql {
   178    234       CREATE TABLE t2(a, b);
          235  +    BEGIN;
   179    236       INSERT INTO t2 VALUES(blob(400), blob(400));
   180    237       SAVEPOINT tr;
   181    238         INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /*  2 */
   182    239         INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /*  4 */
   183    240         INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /*  8 */
   184    241         INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 16 */
   185    242         INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 32 */
................................................................................
   187    244         INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /*  4 */
   188    245         INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /*  8 */
   189    246         INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 16 */
   190    247         INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 32 */
   191    248         SELECT count(*) FROM t2;
   192    249     }
   193    250   } {32}
   194         -do_test wal-4.6 {
          251  +do_test wal-4.5.3 {
          252  +breakpoint
   195    253     execsql { ROLLBACK TO tr }
   196    254   } {}
   197         -do_test wal-4.7 {
          255  +do_test wal-4.5.4 {
   198    256     set logsize [file size test.db-wal]
          257  +breakpoint
   199    258     execsql {
   200    259         INSERT INTO t1 VALUES('x', 'y');
   201    260       RELEASE tr;
          261  +    COMMIT;
   202    262     }
   203    263     expr { $logsize == [file size test.db-wal] }
   204    264   } {1}
   205         -do_test wal-4.8 {
   206         -  execsql { SELECT count(*) FROM t2 }
   207         -} {1}
   208         -do_test wal-4.9 {
          265  +do_test wal-4.5.5 {
          266  +  execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 }
          267  +} {1 2}
          268  +do_test wal-4.5.6 {
   209    269     file copy -force test.db test2.db
   210    270     file copy -force test.db-wal test2.db-wal
   211    271     sqlite3 db2 test2.db
          272  +breakpoint
   212    273     execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } db2
   213    274   } {1 2}
   214         -do_test wal-4.10 {
          275  +do_test wal-4.5.7 {
   215    276     execsql { PRAGMA integrity_check } db2
   216    277   } {ok}
   217    278   db2 close
          279  +
   218    280   
   219    281   reopen_db
   220    282   do_test wal-5.1 {
   221    283     execsql {
   222    284       CREATE TEMP TABLE t2(a, b);
   223    285       INSERT INTO t2 VALUES(1, 2);
   224    286     }
................................................................................
   676    738     execsql {
   677    739       PRAGMA cache_size = 10;
   678    740       BEGIN;
   679    741         INSERT INTO t1 SELECT blob(900) FROM t1;   -- 32
   680    742         SELECT count(*) FROM t1;
   681    743     }
   682    744     list [expr [file size test.db]/1024] [file size test.db-wal]
   683         -} [list 37 [log_file_size 35 1024]]
          745  +} [list 37 [log_file_size 37 1024]]
   684    746   do_test wal-11.11 {
   685    747     execsql {
   686    748         SELECT count(*) FROM t1;
   687    749       ROLLBACK;
   688    750       SELECT count(*) FROM t1;
   689    751     }
   690    752   } {32 16}
   691    753   do_test wal-11.12 {
   692    754     list [expr [file size test.db]/1024] [file size test.db-wal]
   693         -} [list 37 [log_file_size 35 1024]]
          755  +} [list 37 [log_file_size 37 1024]]
   694    756   do_test wal-11.13 {
   695    757     execsql {
   696    758       INSERT INTO t1 VALUES( blob(900) );
   697    759       SELECT count(*) FROM t1;
   698    760       PRAGMA integrity_check;
   699    761     }
   700    762   } {17 ok}
   701    763   do_test wal-11.14 {
   702    764     list [expr [file size test.db]/1024] [file size test.db-wal]
   703         -} [list 37 [log_file_size 35 1024]]
          765  +} [list 37 [log_file_size 37 1024]]
   704    766   
   705    767   
   706    768   #-------------------------------------------------------------------------
   707    769   # This block of tests, wal-12.*, tests the fix for a problem that 
   708    770   # could occur if a log that is a prefix of an older log is written 
   709    771   # into a reused log file.
   710    772   #