/ Check-in [c43deb33]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Delay opening the sub-journal until SQLite actually needs to write data to it.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: c43deb33ae5f191ea2e054181759beeeb9ea71bf
User & Date: dan 2010-06-03 12:35:28
Context
2010-06-03
16:58
Add extra tests for removing elements from wal-index hash tables as part of a rollback. check-in: af3e598a user: dan tags: trunk
12:35
Delay opening the sub-journal until SQLite actually needs to write data to it. check-in: c43deb33 user: dan tags: trunk
12:09
Remove global variables when compiled with SQLITE_OMIT_WSD check-in: dd10a547 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/pager.c.

  3402   3402       pList->pageHash = pager_pagehash(pList);
  3403   3403   #endif
  3404   3404       pList = pList->pDirty;
  3405   3405     }
  3406   3406   
  3407   3407     return rc;
  3408   3408   }
         3409  +
         3410  +/*
         3411  +** Ensure that the sub-journal file is open. If it is already open, this 
         3412  +** function is a no-op.
         3413  +**
         3414  +** SQLITE_OK is returned if everything goes according to plan. An 
         3415  +** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() 
         3416  +** fails.
         3417  +*/
         3418  +static int openSubJournal(Pager *pPager){
         3419  +  int rc = SQLITE_OK;
         3420  +  if( !isOpen(pPager->sjfd) ){
         3421  +    if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){
         3422  +      sqlite3MemJournalOpen(pPager->sjfd);
         3423  +    }else{
         3424  +      rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL);
         3425  +    }
         3426  +  }
         3427  +  return rc;
         3428  +}
  3409   3429   
  3410   3430   /*
  3411   3431   ** Append a record of the current state of page pPg to the sub-journal. 
  3412   3432   ** It is the callers responsibility to use subjRequiresPage() to check 
  3413   3433   ** that it is really required before calling this function.
  3414   3434   **
  3415   3435   ** If successful, set the bit corresponding to pPg->pgno in the bitvecs
................................................................................
  3419   3439   ** error code if the attempt to write to the sub-journal fails, or 
  3420   3440   ** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint
  3421   3441   ** bitvec.
  3422   3442   */
  3423   3443   static int subjournalPage(PgHdr *pPg){
  3424   3444     int rc = SQLITE_OK;
  3425   3445     Pager *pPager = pPg->pPager;
  3426         -  if( isOpen(pPager->sjfd) ){
  3427         -    void *pData = pPg->pData;
  3428         -    i64 offset = pPager->nSubRec*(4+pPager->pageSize);
  3429         -    char *pData2;
         3446  +  if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){
  3430   3447   
  3431         -    CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2);
  3432         -    PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno));
  3433         -  
         3448  +    /* Open the sub-journal, if it has not already been opened */
         3449  +    assert( pPager->useJournal );
         3450  +    assert( isOpen(pPager->jfd) || pagerUseWal(pPager) );
         3451  +    assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 );
  3434   3452       assert( pagerUseWal(pPager) 
  3435   3453            || pageInJournal(pPg) 
  3436   3454            || pPg->pgno>pPager->dbOrigSize 
  3437   3455       );
  3438         -    rc = write32bits(pPager->sjfd, offset, pPg->pgno);
         3456  +    rc = openSubJournal(pPager);
         3457  +
         3458  +    /* If the sub-journal was opened successfully (or was already open),
         3459  +    ** write the journal record into the file.  */
  3439   3460       if( rc==SQLITE_OK ){
  3440         -      rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4);
         3461  +      void *pData = pPg->pData;
         3462  +      i64 offset = pPager->nSubRec*(4+pPager->pageSize);
         3463  +      char *pData2;
         3464  +  
         3465  +      CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2);
         3466  +      PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno));
         3467  +      rc = write32bits(pPager->sjfd, offset, pPg->pgno);
         3468  +      if( rc==SQLITE_OK ){
         3469  +        rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4);
         3470  +      }
  3441   3471       }
  3442   3472     }
  3443   3473     if( rc==SQLITE_OK ){
  3444   3474       pPager->nSubRec++;
  3445   3475       assert( pPager->nSavepoint>0 );
  3446   3476       rc = addToSavepointBitvecs(pPager, pPg->pgno);
  3447   3477     }
................................................................................
  4396   4426     if( pPg ){
  4397   4427       Pager *pPager = pPg->pPager;
  4398   4428       sqlite3PcacheRelease(pPg);
  4399   4429       pagerUnlockIfUnused(pPager);
  4400   4430     }
  4401   4431   }
  4402   4432   
  4403         -/*
  4404         -** If the main journal file has already been opened, ensure that the
  4405         -** sub-journal file is open too. If the main journal is not open,
  4406         -** this function is a no-op.
  4407         -**
  4408         -** SQLITE_OK is returned if everything goes according to plan. 
  4409         -** An SQLITE_IOERR_XXX error code is returned if a call to 
  4410         -** sqlite3OsOpen() fails.
  4411         -*/
  4412         -static int openSubJournal(Pager *pPager){
  4413         -  int rc = SQLITE_OK;
  4414         -  if( (pagerUseWal(pPager) || isOpen(pPager->jfd)) && !isOpen(pPager->sjfd) ){
  4415         -    if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){
  4416         -      sqlite3MemJournalOpen(pPager->sjfd);
  4417         -    }else{
  4418         -      rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL);
  4419         -    }
  4420         -  }
  4421         -  return rc;
  4422         -}
  4423         -
  4424   4433   /*
  4425   4434   ** This function is called at the start of every write transaction.
  4426   4435   ** There must already be a RESERVED or EXCLUSIVE lock on the database 
  4427   4436   ** file when this routine is called.
  4428   4437   **
  4429   4438   ** Open the journal file for pager pPager and write a journal header
  4430   4439   ** to the start of it. If there are active savepoints, open the sub-journal
................................................................................
  4499   4508       pPager->needSync = 0;
  4500   4509       pPager->nRec = 0;
  4501   4510       pPager->journalOff = 0;
  4502   4511       pPager->setMaster = 0;
  4503   4512       pPager->journalHdr = 0;
  4504   4513       rc = writeJournalHdr(pPager);
  4505   4514     }
  4506         -  if( rc==SQLITE_OK && pPager->nSavepoint ){
  4507         -    rc = openSubJournal(pPager);
  4508         -  }
  4509   4515   
  4510   4516     if( rc!=SQLITE_OK ){
  4511   4517       sqlite3BitvecDestroy(pPager->pInJournal);
  4512   4518       pPager->pInJournal = 0;
  4513   4519     }
  4514   4520     return rc;
  4515   4521   }
................................................................................
  5465   5471         }
  5466   5472         if( pagerUseWal(pPager) ){
  5467   5473           sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData);
  5468   5474         }
  5469   5475         pPager->nSavepoint = ii+1;
  5470   5476       }
  5471   5477       assert( pPager->nSavepoint==nSavepoint );
  5472         -
  5473         -    /* Open the sub-journal, if it is not already opened. */
  5474         -    rc = openSubJournal(pPager);
  5475   5478       assertTruncateConstraint(pPager);
  5476   5479     }
  5477   5480   
  5478   5481     return rc;
  5479   5482   }
  5480   5483   
  5481   5484   /*

Changes to test/stmt.test.

    46     46     }
    47     47     set sqlite_open_file_count
    48     48   } {3}
    49     49   do_test stmt-1.5 {
    50     50     execsql COMMIT
    51     51     set sqlite_open_file_count
    52     52   } {1}
    53         -do_test stmt-1.6 {
           53  +do_test stmt-1.6.1 {
    54     54     execsql {
    55     55       BEGIN;
    56     56         INSERT INTO t1 SELECT a+2, b+2 FROM t1;
    57     57     }
    58     58     set sqlite_open_file_count
           59  +} {2}
           60  +do_test stmt-1.6.2 {
           61  +  execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 }
           62  +  set sqlite_open_file_count
    59     63   } {3}
    60     64   do_test stmt-1.7 {
    61     65     execsql COMMIT
    62     66     set sqlite_open_file_count
    63     67   } {1}
    64     68   
    65     69   
................................................................................
    69     73       execsql { $sql }
    70     74       set ret [set sqlite_open_file_count]
    71     75       execsql ROLLBACK
    72     76       set ret
    73     77     }] $expected]
    74     78   }
    75     79   
    76         -filecount stmt-2.1 { INSERT INTO t1 VALUES(5, 5)  } 2
    77         -filecount stmt-2.2 { REPLACE INTO t1 VALUES(5, 5) } 2
    78         -filecount stmt-2.3 { INSERT INTO t1 SELECT 5, 5   } 3
           80  +filecount stmt-2.1 { INSERT INTO t1 VALUES(9, 9)  } 2
           81  +filecount stmt-2.2 { REPLACE INTO t1 VALUES(9, 9) } 2
           82  +filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9   } 2
           83  +filecount stmt-2.4 { 
           84  +    INSERT INTO t1 SELECT 9, 9;
           85  +    INSERT INTO t1 SELECT 10, 10;
           86  +} 3
    79     87   
    80         -do_test stmt-2.4 {
           88  +do_test stmt-2.5 {
    81     89     execsql { CREATE INDEX i1 ON t1(b) }
    82     90   } {}
    83         -filecount stmt-2.5 { REPLACE INTO t1 VALUES(5, 5) } 3
           91  +filecount stmt-2.6 { 
           92  +  REPLACE INTO t1 VALUES(5, 5);
           93  +  REPLACE INTO t1 VALUES(5, 5); 
           94  +} 3
    84     95   
    85     96   finish_test

Changes to test/tempdb.test.

    70     70   do_test tempdb-2.2 {
    71     71     execsql {
    72     72       CREATE TABLE t1 (a PRIMARY KEY, b, c);
    73     73       CREATE TABLE t2 (a, b, c);
    74     74       BEGIN;
    75     75         INSERT INTO t1 VALUES(1, 2, 3);
    76     76         INSERT INTO t1 VALUES(4, 5, 6);
           77  +      INSERT INTO t2 VALUES(7, 8, 9);
    77     78         INSERT INTO t2 SELECT * FROM t1;
    78     79     }
    79     80     catchsql { INSERT INTO t1 SELECT * FROM t2 }
    80     81     set sqlite_open_file_count
    81     82   } [expr 1 + (0==$jrnl_in_memory) + (0==$subj_in_memory)]
    82     83   do_test tempdb-2.3 {
    83     84     execsql {

Changes to test/walfault.test.

    23     23   # This test case, walfault-1-*, simulates faults while executing a
    24     24   #
    25     25   #   PRAGMA journal_mode = WAL;
    26     26   #
    27     27   # statement immediately after creating a new database.
    28     28   #
    29     29   do_test walfault-1-pre-1 {
    30         -  db close
    31         -  file delete -force test.db test.db-wal test.db-journal
           30  +  faultsim_delete_and_reopen
    32     31     faultsim_save_and_close
    33     32   } {}
    34     33   do_faultsim_test walfault-1 -prep {
    35     34     faultsim_restore_and_reopen
    36     35   } -body {
    37     36     db eval { PRAGMA main.journal_mode = WAL }
    38     37   } -test {
................................................................................
   105    104       PRAGMA journal_mode = WAL;
   106    105       CREATE TABLE abc(a PRIMARY KEY);
   107    106       INSERT INTO abc VALUES(randomblob(1500));
   108    107     }
   109    108     db close
   110    109     faultsim_save_and_close
   111    110   } {}
   112         -
   113    111   do_faultsim_test walfault-3 -prep {
   114    112     faultsim_restore_and_reopen
   115    113   } -body {
   116    114     db eval {
   117    115       DELETE FROM abc;
   118    116       PRAGMA wal_checkpoint;
   119    117     }
   120    118   } -test {
   121    119     faultsim_test_result {0 {}}
   122    120   }
   123    121   
   124         -file delete -force test.db test.db-wal test.db-journal
          122  +#--------------------------------------------------------------------------
          123  +#
          124  +faultsim_delete_and_reopen
   125    125   faultsim_save_and_close
   126    126   do_faultsim_test walfault-4 -prep {
   127    127     faultsim_restore_and_reopen
   128    128   } -body {
   129    129     execsql {
   130    130       PRAGMA journal_mode = WAL;
   131    131       CREATE TABLE t1(a PRIMARY KEY, b);
................................................................................
   134    134       SELECT * FROM t1;
   135    135     }
   136    136   } -test {
   137    137     faultsim_test_result {0 {wal a b}}
   138    138     faultsim_integrity_check
   139    139   } 
   140    140   
          141  +#--------------------------------------------------------------------------
          142  +#
   141    143   do_test walfault-5-pre-1 {
   142         -  catch { db close }
   143         -  file delete -force test.db test.db-wal test.db-journal
   144         -  sqlite3 db test.db
          144  +  faultsim_delete_and_reopen
   145    145     execsql {
   146    146       PRAGMA page_size = 512;
   147    147       PRAGMA journal_mode = WAL;
   148    148     }
   149    149     faultsim_save_and_close
   150    150   } {}
   151    151   do_faultsim_test walfault-5 -faults shmerr* -prep {
................................................................................
   175    175       SELECT count(*) FROM t1;
   176    176     }
   177    177   } -test {
   178    178     faultsim_test_result {0 16384}
   179    179     faultsim_integrity_check
   180    180   }
   181    181   
   182         -
          182  +#--------------------------------------------------------------------------
          183  +#
   183    184   do_test walfault-6-pre-1 {
   184         -  catch { db close }
   185         -  file delete -force test.db test.db-wal test.db-journal
   186         -  sqlite3 db test.db
          185  +  faultsim_delete_and_reopen
   187    186     execsql {
   188    187       PRAGMA page_size = 512;
   189    188       PRAGMA journal_mode = WAL;
   190    189       PRAGMA wal_autocheckpoint = 0;
   191    190       CREATE TABLE t1(x);
   192    191       BEGIN;
   193    192         INSERT INTO t1 VALUES(randomblob(400));           /* 1 */
................................................................................
   217    216   } -test {
   218    217     faultsim_test_result {0 16384}
   219    218     faultsim_integrity_check
   220    219     set n [db one {SELECT count(*) FROM t1}]
   221    220     if {$n != 16384 && $n != 0} { error "Incorrect number of rows: $n" }
   222    221   }
   223    222   
          223  +#--------------------------------------------------------------------------
          224  +#
   224    225   do_test walfault-7-pre-1 {
   225    226     faultsim_delete_and_reopen
   226    227     execsql {
   227    228       PRAGMA page_size = 512;
   228    229       PRAGMA journal_mode = WAL;
   229    230       PRAGMA wal_autocheckpoint = 0;
   230    231       CREATE TABLE t1(x);
................................................................................
   242    243     execsql { SELECT count(*) FROM t1 }
   243    244   } -test {
   244    245     faultsim_test_result {0 4}
   245    246     set n [db one {SELECT count(*) FROM t1}]
   246    247     if {$n != 4 && $n != 0} { error "Incorrect number of rows: $n" }
   247    248   }
   248    249   
          250  +#--------------------------------------------------------------------------
          251  +#
   249    252   do_test walfault-8-pre-1 {
   250    253     faultsim_delete_and_reopen
   251    254     execsql {
   252    255       PRAGMA journal_mode = WAL;
   253    256       CREATE TABLE abc(a PRIMARY KEY);
   254    257       INSERT INTO abc VALUES(randomblob(900));
   255    258     }
................................................................................
   275    278     catch { db eval ROLLBACK }
   276    279     faultsim_integrity_check
   277    280   
   278    281     set n [db one {SELECT count(*) FROM abc}]
   279    282     if {$n != 1} { error "Incorrect number of rows: $n" }
   280    283   }
   281    284   
          285  +#--------------------------------------------------------------------------
          286  +#
   282    287   do_test walfault-9-pre-1 {
   283    288     faultsim_delete_and_reopen
   284    289     execsql {
   285    290       PRAGMA journal_mode = WAL;
   286    291       CREATE TABLE abc(a PRIMARY KEY);
   287    292       INSERT INTO abc VALUES(randomblob(900));
   288    293     }
................................................................................
   324    329   #   3. If the mapping obtained in (2) is not large enough to cover the
   325    330   #      entire wal-index, call xShmGet(nReq) to get a larger mapping.
   326    331   #   4. Do the checkpoint.
   327    332   #   5. Release the lock and mapping.
   328    333   #
   329    334   # This test case tests the outcome of an IO error in step 2.
   330    335   #
   331         -proc shmfault_vfs_cb_6 {method args} {
          336  +proc walfault_10_vfs_cb {method args} {
   332    337     switch -- $::shm_state {
   333    338       0 { return SQLITE_OK }
   334    339       1 {
   335    340         if {$method == "xShmGet"} {
   336    341           set ::wal_index [tvfs shm [lindex $args 0]]
   337    342           tvfs shm [lindex $args 0] [string range $::wal_index 0 65535]
   338    343           set ::shm_state 2
................................................................................
   343    348           tvfs shm [lindex $args 0] $::wal_index
   344    349           return SQLITE_IOERR
   345    350         }
   346    351       }
   347    352     }
   348    353     return SQLITE_OK
   349    354   }
   350         -do_test walfault-shm-6.1 {
          355  +do_test walfault-10.1 {
   351    356     set ::shm_state 0
   352    357     testvfs tvfs 
   353         -  tvfs script shmfault_vfs_cb_6
          358  +  tvfs script walfault_10_vfs_cb
   354    359   
   355    360     sqlite3 db  test.db -vfs tvfs
   356    361     sqlite3 db2 test.db -vfs tvfs
   357    362   
   358    363     execsql {
   359    364       PRAGMA journal_mode = WAL;
   360    365       PRAGMA wal_autocheckpoint = 0;
   361    366       CREATE TABLE t1(x);
   362    367       INSERT INTO t1 VALUES(randomblob(900));
   363    368     }
   364    369   } {wal 0}
   365         -do_test walfault-shm-6.2 {
          370  +do_test walfault-10.2 {
   366    371     execsql {
   367    372       PRAGMA wal_autocheckpoint = 0;
   368    373       BEGIN;
   369    374         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 2 */
   370    375         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 4 */
   371    376         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 8 */
   372    377         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 16 */
................................................................................
   379    384         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 2048 */
   380    385         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 4096 */
   381    386         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 8192 */
   382    387         INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 16384 */
   383    388       COMMIT;
   384    389     } db2
   385    390   } {0}
   386         -do_test walfault-shm-6.3 {
          391  +do_test walfault-10.3 {
   387    392     set ::shm_state 1
   388    393     catchsql { PRAGMA wal_checkpoint } db2
   389    394   } {1 {disk I/O error}}
   390    395   set ::shm_state 0
   391    396   db close
   392    397   db2 close
   393    398   tvfs delete
   394    399   unset -nocomplain ::wal_index ::shm_state
   395    400   
   396    401   finish_test
   397    402