/ Check-in [1c41250f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:If an SQLITE_IOERR error is encountered as part of an atomic commit on an F2FS file-system, retry the commit in legacy journal mode.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | exp-retry-atomic-commit
Files: files | file ages | folders
SHA3-256: 1c41250f67ac5de423b0426ef2ab8fe3303278a270225920033933ca9609592a
User & Date: dan 2018-07-14 20:25:13
Context
2018-07-16
20:44
Add new file doc/F2FS.txt, containing notes on the way SQLite uses the F2FS atomic commit feature. check-in: 59efb1bf user: dan tags: exp-retry-atomic-commit
2018-07-14
20:25
If an SQLITE_IOERR error is encountered as part of an atomic commit on an F2FS file-system, retry the commit in legacy journal mode. check-in: 1c41250f user: dan tags: exp-retry-atomic-commit
2018-07-13
20:28
Remove an unused function declaration from fts5. check-in: 148d9b61 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/pager.c.

  6378   6378     assert( isOpen(pPager->fd) || pPager->tempFile );
  6379   6379     if( 0==pagerFlushOnCommit(pPager, 1) ){
  6380   6380       /* If this is an in-memory db, or no pages have been written to, or this
  6381   6381       ** function has already been called, it is mostly a no-op.  However, any
  6382   6382       ** backup in progress needs to be restarted.  */
  6383   6383       sqlite3BackupRestart(pPager->pBackup);
  6384   6384     }else{
         6385  +    PgHdr *pList;
  6385   6386       if( pagerUseWal(pPager) ){
  6386         -      PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
  6387   6387         PgHdr *pPageOne = 0;
         6388  +      pList = sqlite3PcacheDirtyList(pPager->pPCache);
  6388   6389         if( pList==0 ){
  6389   6390           /* Must have at least one page for the WAL commit flag.
  6390   6391           ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */
  6391   6392           rc = sqlite3PagerGet(pPager, 1, &pPageOne, 0);
  6392   6393           pList = pPageOne;
  6393   6394           pList->pDirty = 0;
  6394   6395         }
................................................................................
  6403   6404       }else{
  6404   6405         /* The bBatch boolean is true if the batch-atomic-write commit method
  6405   6406         ** should be used.  No rollback journal is created if batch-atomic-write
  6406   6407         ** is enabled.
  6407   6408         */
  6408   6409         sqlite3_file *fd = pPager->fd;
  6409   6410   #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE
  6410         -      const int bBatch = zMaster==0    /* An SQLITE_IOCAP_BATCH_ATOMIC commit */
         6411  +      int bBatch = zMaster==0    /* An SQLITE_IOCAP_BATCH_ATOMIC commit */
  6411   6412           && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC)
  6412   6413           && !pPager->noSync
  6413   6414           && sqlite3JournalIsInMemory(pPager->jfd);
  6414   6415   #else
  6415   6416   # define bBatch 0
  6416   6417   #endif
  6417   6418   
................................................................................
  6460   6461           }else{
  6461   6462             rc = sqlite3JournalCreate(pPager->jfd);
  6462   6463             if( rc==SQLITE_OK ){
  6463   6464               rc = pager_incr_changecounter(pPager, 0);
  6464   6465             }
  6465   6466           }
  6466   6467         }
  6467         -#else 
         6468  +#else  /* SQLITE_ENABLE_ATOMIC_WRITE */
  6468   6469   #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE
  6469   6470         if( zMaster ){
  6470   6471           rc = sqlite3JournalCreate(pPager->jfd);
  6471   6472           if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
         6473  +        assert( bBatch==0 );
  6472   6474         }
  6473   6475   #endif
  6474   6476         rc = pager_incr_changecounter(pPager, 0);
  6475         -#endif
         6477  +#endif /* !SQLITE_ENABLE_ATOMIC_WRITE */
  6476   6478         if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
  6477   6479     
  6478   6480         /* Write the master journal name into the journal file. If a master 
  6479   6481         ** journal file name has already been written to the journal file, 
  6480   6482         ** or if zMaster is NULL (no master journal), then this call is a no-op.
  6481   6483         */
  6482   6484         rc = writeMasterJournal(pPager, zMaster);
................................................................................
  6492   6494         ** on a system under memory pressure it is just possible that this is 
  6493   6495         ** not the case. In this case it is likely enough that the redundant
  6494   6496         ** xSync() call will be changed to a no-op by the OS anyhow. 
  6495   6497         */
  6496   6498         rc = syncJournal(pPager, 0);
  6497   6499         if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
  6498   6500   
         6501  +      pList = sqlite3PcacheDirtyList(pPager->pPCache);
  6499   6502         if( bBatch ){
  6500         -        /* The pager is now in DBMOD state. But regardless of what happens
  6501         -        ** next, attempting to play the journal back into the database would
  6502         -        ** be unsafe. Close it now to make sure that does not happen.  */
  6503         -        sqlite3OsClose(pPager->jfd);
  6504   6503           rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0);
  6505         -        if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
  6506         -      }
  6507         -      rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache));
  6508         -      if( bBatch ){
  6509   6504           if( rc==SQLITE_OK ){
  6510         -          rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0);
         6505  +          rc = pager_write_pagelist(pPager, pList);
         6506  +          if( rc==SQLITE_OK ){
         6507  +            rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0);
         6508  +          }
         6509  +          if( rc!=SQLITE_OK ){
         6510  +            sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0);
         6511  +          }
  6511   6512           }
  6512         -        if( rc!=SQLITE_OK ){
  6513         -          sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0);
         6513  +
         6514  +        if( (rc&0xFF)==SQLITE_IOERR && rc!=SQLITE_IOERR_NOMEM ){
         6515  +          rc = sqlite3JournalCreate(pPager->jfd);
         6516  +          if( rc!=SQLITE_OK ){
         6517  +            sqlite3OsClose(pPager->jfd);
         6518  +          }
         6519  +          bBatch = 0;
         6520  +        }else{
         6521  +          sqlite3OsClose(pPager->jfd);
  6514   6522           }
  6515   6523         }
         6524  +
         6525  +      if( bBatch==0 && rc==SQLITE_OK ){
         6526  +        rc = pager_write_pagelist(pPager, pList);
         6527  +      }
  6516   6528   
  6517   6529         if( rc!=SQLITE_OK ){
  6518   6530           assert( rc!=SQLITE_IOERR_BLOCKED );
  6519   6531           goto commit_phase_one_exit;
  6520   6532         }
  6521   6533         sqlite3PcacheCleanAll(pPager->pPCache);
  6522   6534   

Changes to src/test_vfs.c.

   129    129   #define TESTVFS_TRUNCATE_MASK     0x00002000
   130    130   #define TESTVFS_ACCESS_MASK       0x00004000
   131    131   #define TESTVFS_FULLPATHNAME_MASK 0x00008000
   132    132   #define TESTVFS_READ_MASK         0x00010000
   133    133   #define TESTVFS_UNLOCK_MASK       0x00020000
   134    134   #define TESTVFS_LOCK_MASK         0x00040000
   135    135   #define TESTVFS_CKLOCK_MASK       0x00080000
          136  +#define TESTVFS_FCNTL_MASK        0x00100000
   136    137   
   137         -#define TESTVFS_ALL_MASK          0x000FFFFF
          138  +#define TESTVFS_ALL_MASK          0x001FFFFF
   138    139   
   139    140   
   140    141   #define TESTVFS_MAX_PAGES 1024
   141    142   
   142    143   /*
   143    144   ** A shared-memory buffer. There is one of these objects for each shared
   144    145   ** memory region opened by clients. If two clients open the same file,
................................................................................
   513    514     return sqlite3OsCheckReservedLock(pFd->pReal, pResOut);
   514    515   }
   515    516   
   516    517   /*
   517    518   ** File control method. For custom operations on an tvfs-file.
   518    519   */
   519    520   static int tvfsFileControl(sqlite3_file *pFile, int op, void *pArg){
   520         -  TestvfsFd *p = tvfsGetFd(pFile);
          521  +  TestvfsFd *pFd = tvfsGetFd(pFile);
          522  +  Testvfs *p = (Testvfs *)pFd->pVfs->pAppData;
   521    523     if( op==SQLITE_FCNTL_PRAGMA ){
   522    524       char **argv = (char**)pArg;
   523    525       if( sqlite3_stricmp(argv[1],"error")==0 ){
   524    526         int rc = SQLITE_ERROR;
   525    527         if( argv[2] ){
   526    528           const char *z = argv[2];
   527    529           int x = atoi(z);
................................................................................
   531    533             while( sqlite3Isspace(z[0]) ){ z++; }
   532    534           }
   533    535           if( z[0] ) argv[0] = sqlite3_mprintf("%s", z);
   534    536         }
   535    537         return rc;
   536    538       }
   537    539       if( sqlite3_stricmp(argv[1], "filename")==0 ){
   538         -      argv[0] = sqlite3_mprintf("%s", p->zFilename);
          540  +      argv[0] = sqlite3_mprintf("%s", pFd->zFilename);
   539    541         return SQLITE_OK;
   540    542       }
   541    543     }
   542         -  return sqlite3OsFileControl(p->pReal, op, pArg);
          544  +  if( p->pScript && (p->mask&TESTVFS_FCNTL_MASK) ){
          545  +    struct Fcntl {
          546  +      int iFnctl;
          547  +      const char *zFnctl;
          548  +    } aF[] = {
          549  +      { SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, "BEGIN_ATOMIC_WRITE" },
          550  +      { SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, "COMMIT_ATOMIC_WRITE" },
          551  +    };
          552  +    int i;
          553  +    for(i=0; i<sizeof(aF)/sizeof(aF[0]); i++){
          554  +      if( op==aF[i].iFnctl ) break;
          555  +    }
          556  +    if( i<sizeof(aF)/sizeof(aF[0]) ){
          557  +      int rc = 0;
          558  +      tvfsExecTcl(p, "xFileControl", 
          559  +          Tcl_NewStringObj(pFd->zFilename, -1), 
          560  +          Tcl_NewStringObj(aF[i].zFnctl, -1),
          561  +          0, 0
          562  +      );
          563  +      tvfsResultCode(p, &rc);
          564  +      if( rc ) return rc;
          565  +    }
          566  +  }
          567  +  return sqlite3OsFileControl(pFd->pReal, op, pArg);
   543    568   }
   544    569   
   545    570   /*
   546    571   ** Return the sector-size in bytes for an tvfs-file.
   547    572   */
   548    573   static int tvfsSectorSize(sqlite3_file *pFile){
   549    574     TestvfsFd *pFd = tvfsGetFd(pFile);
................................................................................
  1156   1181           { "xOpen",              TESTVFS_OPEN_MASK },
  1157   1182           { "xClose",             TESTVFS_CLOSE_MASK },
  1158   1183           { "xAccess",            TESTVFS_ACCESS_MASK },
  1159   1184           { "xFullPathname",      TESTVFS_FULLPATHNAME_MASK },
  1160   1185           { "xUnlock",            TESTVFS_UNLOCK_MASK },
  1161   1186           { "xLock",              TESTVFS_LOCK_MASK },
  1162   1187           { "xCheckReservedLock", TESTVFS_CKLOCK_MASK },
         1188  +        { "xFileControl",       TESTVFS_FCNTL_MASK },
  1163   1189         };
  1164   1190         Tcl_Obj **apElem = 0;
  1165   1191         int nElem = 0;
  1166   1192         int mask = 0;
  1167   1193         if( objc!=3 ){
  1168   1194           Tcl_WrongNumArgs(interp, 2, objv, "LIST");
  1169   1195           return TCL_ERROR;

Added test/atomic2.test.

            1  +# 2018-07-15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing that if an IO error is encountered
           13  +# as part of an atomic F2FS commit, an attempt is made to commit the
           14  +# transaction using a legacy journal commit.
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +source $testdir/malloc_common.tcl
           20  +set ::testprefix atomic2
           21  +
           22  +db close
           23  +if {[atomic_batch_write test.db]==0} {
           24  +  puts "No f2fs atomic-batch-write support. Skipping tests..."
           25  +  finish_test
           26  +  return
           27  +}
           28  +
           29  +reset_db
           30  +
           31  +do_execsql_test 1.0 {
           32  +  CREATE TABLE t1(x, y);
           33  +  CREATE INDEX i1x ON t1(x);
           34  +  CREATE INDEX i2x ON t1(y);
           35  +
           36  +  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 )
           37  +  INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
           38  +}
           39  +
           40  +set setup [list \
           41  +  -injectstart at_injectstart \
           42  +  -injectstop  at_injectstop  \
           43  +]
           44  +
           45  +set ::at_fail  0
           46  +set ::at_nfail 0
           47  +
           48  +proc at_injectstart {iFail} {
           49  +  set ::at_fail $iFail
           50  +  set ::at_nfail 0
           51  +}
           52  +proc at_injectstop {} {
           53  +  set ::at_fail 0
           54  +  return $::at_nfail
           55  +}
           56  +
           57  +proc at_vfs_callback {method file z args} {
           58  +  if {$::at_fail>0} {
           59  +    incr ::at_fail -1
           60  +    if {$::at_fail==0} {
           61  +      incr ::at_nfail
           62  +      return SQLITE_IOERR
           63  +    } elseif {$method=="xFileControl" && $z=="COMMIT_ATOMIC_WRITE"} {
           64  +      set ::at_fail 0
           65  +    }
           66  +  }
           67  +  return SQLITE_OK
           68  +}
           69  +
           70  +testvfs tvfs -default 1
           71  +tvfs script at_vfs_callback
           72  +tvfs filter {xFileControl xWrite}
           73  +
           74  +faultsim_save_and_close
           75  +
           76  +do_one_faultsim_test 2.0 {*}$setup -prep {
           77  +  faultsim_restore_and_reopen
           78  +} -body {
           79  +  execsql {
           80  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 )
           81  +    INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
           82  +  }
           83  +} -test {
           84  +  faultsim_test_result {0 {}}
           85  +
           86  +  set res [execsql {SELECT count(*) FROM t1; PRAGMA integrity_check}]
           87  +  if {$res!="200 ok"} {
           88  +    error "expected {200 ok}, got $res"
           89  +  }
           90  +}
           91  +
           92  +db close
           93  +tvfs delete
           94  +
           95  +finish_test