/ Check-in [2b10c5d2]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add ota tests to increase code coverage. Fix some minor issues in error handling within the ota code.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | ota-update
Files: files | file ages | folders
SHA1: 2b10c5d2b8b8b535d3dec0c68a777db16268e1e5
User & Date: dan 2015-02-18 20:16:15
Context
2015-02-18
20:17
Add new file ota12.test, containing tests for applying ota updates to live databases with other active reader/writer clients. check-in: 0864d127 user: dan tags: ota-update
20:16
Add ota tests to increase code coverage. Fix some minor issues in error handling within the ota code. check-in: 2b10c5d2 user: dan tags: ota-update
17:40
Fix a problem with OTA updates in the presence of database readers. check-in: 144bb29f user: dan tags: ota-update
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/ota/ota1.test.

   523    523          } {SQLITE_ERROR - table data_t1 may not have ota_rowid column}
   524    524   
   525    525          6 {
   526    526            CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID;
   527    527            CREATE TABLE ota.data_t1(a, b, ota_control);
   528    528            INSERT INTO ota.data_t1 VALUES(1, 2, 'x.x');
   529    529          } {SQLITE_ERROR - invalid ota_control value}
          530  +
          531  +       7 {
          532  +         CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID;
          533  +         CREATE TABLE ota.data_t1(a, b, ota_control);
          534  +         INSERT INTO ota.data_t1 VALUES(1, 2, NULL);
          535  +       } {SQLITE_ERROR - invalid ota_control value}
          536  +
          537  +       8 {
          538  +         CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID;
          539  +         CREATE TABLE ota.data_t1(a, b, ota_control);
          540  +         INSERT INTO ota.data_t1 VALUES(1, 2, 4);
          541  +       } {SQLITE_ERROR - invalid ota_control value}
          542  +
          543  +       9 {
          544  +         CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID;
          545  +         CREATE TABLE ota.data_t1(a, b, ota_control);
          546  +         INSERT INTO ota.data_t1 VALUES(1, 2, 2);
          547  +       } {SQLITE_ERROR - invalid ota_control value}
   530    548       
   531    549       } {
   532    550         reset_db
   533    551         forcedelete ota.db
   534    552         execsql { ATTACH 'ota.db' AS ota }
   535    553         execsql $schema
   536    554   

Changes to ext/ota/ota11.test.

   143    143     sqlite3ota ota test.db ota.db
   144    144     ota step
   145    145   } {SQLITE_DONE}
   146    146   
   147    147   do_test 4.4 {
   148    148     ota close
   149    149   } {SQLITE_DONE}
          150  +
          151  +do_test 4.5.1 {
          152  +  sqlite3 dbo ota.db
          153  +  dbo eval { INSERT INTO ota_state VALUES(100, 100) }
          154  +  dbo close
          155  +  sqlite3ota ota test.db ota.db
          156  +  ota step
          157  +} {SQLITE_CORRUPT}
          158  +do_test 4.5.2 {
          159  +  list [catch {ota close} msg] $msg
          160  +} {1 SQLITE_CORRUPT}
          161  +do_test 4.5.3 {
          162  +  sqlite3 dbo ota.db
          163  +  dbo eval { DELETE FROM ota_state WHERE k = 100 }
          164  +  dbo close 
          165  +} {}
   150    166   
   151    167   # Also, check that an invalid state value in the ota_state table is
   152    168   # detected and reported as corruption.
   153         -do_test 4.5 {
          169  +do_test 4.6.1 {
   154    170     sqlite3 dbo ota.db
   155         -  dbo eval { UPDATE ota_state SET v = -1 WHERE k = 1 }
          171  +  dbo eval { UPDATE ota_state SET v = v*-1 WHERE k = 1 }
   156    172     dbo close
   157    173     sqlite3ota ota test.db ota.db
   158    174     ota step
   159    175   } {SQLITE_CORRUPT}
   160         -do_test 4.6 {
          176  +do_test 4.6.2 {
   161    177     list [catch {ota close} msg] $msg
   162    178   } {1 SQLITE_CORRUPT}
          179  +do_test 4.6.3 {
          180  +  sqlite3 dbo ota.db
          181  +  dbo eval { UPDATE ota_state SET v = v*-1 WHERE k = 1 }
          182  +  dbo close 
          183  +} {}
          184  +
          185  +do_test 4.7.1 {
          186  +  sqlite3 dbo ota.db
          187  +  dbo eval { UPDATE ota_state SET v = 1 WHERE k = 1 }
          188  +  dbo eval { UPDATE ota_state SET v = 'nosuchtable' WHERE k = 2 }
          189  +  dbo close
          190  +  sqlite3ota ota test.db ota.db
          191  +  ota step
          192  +} {SQLITE_ERROR}
          193  +do_test 4.7.2 {
          194  +  list [catch {ota close} msg] $msg
          195  +} {1 {SQLITE_ERROR - ota_state mismatch error}}
   163    196   
   164    197   finish_test
   165    198   

Changes to ext/ota/ota3.test.

   165    165     sqlite3ota ota file:test.db?vfs=tvfs ota.db 
   166    166     list [catch { ota step } msg] $msg
   167    167   } {0 SQLITE_ERROR}
   168    168   do_test 4.2 {
   169    169     list [catch { ota close } msg] $msg
   170    170   } {1 {SQLITE_ERROR - ota vfs not found}}
   171    171   tvfs delete
          172  +
          173  +#-------------------------------------------------------------------------
          174  +# Test a large ota update to ensure that wal_autocheckpoint does not get
          175  +# in the way.
          176  +#
          177  +forcedelete ota.db
          178  +reset_db
          179  +do_execsql_test 5.1 {
          180  +  CREATE TABLE x1(a, b, c, PRIMARY KEY(a)) WITHOUT ROWID;
          181  +  CREATE INDEX i1 ON x1(a);
          182  +
          183  +  ATTACH 'ota.db' AS ota;
          184  +  CREATE TABLE ota.data_x1(a, b, c, ota_control);
          185  +  WITH s(a, b, c) AS (
          186  +    SELECT randomblob(300), randomblob(300), 1
          187  +    UNION ALL
          188  +    SELECT randomblob(300), randomblob(300), c+1 FROM s WHERE c<2000
          189  +  )
          190  +  INSERT INTO data_x1 SELECT a, b, c, 0 FROM s;
          191  +}
          192  +
          193  +do_test 5.2 {
          194  +  sqlite3ota ota test.db ota.db
          195  +  while {[ota step]=="SQLITE_OK" && [file exists test.db-wal]==0} {}
          196  +  ota close
          197  +} {SQLITE_OK}
          198  +
          199  +do_test 5.3 {
          200  +  expr {[file size test.db-wal] > (1024 * 1200)}
          201  +} 1
   172    202   
   173    203   finish_test
   174    204   
   175    205   

Changes to ext/ota/otafault.test.

    80     80       INSERT INTO data_t1 VALUES('D', NULL, NULL, 1);
    81     81       INSERT INTO data_t1 VALUES('A', 'Z', NULL, '.x.');
    82     82       INSERT INTO data_t1 VALUES('G', 'H', 'I', 0);
    83     83     } {
    84     84       SELECT * FROM t1 ORDER BY a;
    85     85     } {A Z C G H I}
    86     86   
           87  +  5 {
           88  +    CREATE TABLE t1(a, b, c);
           89  +    CREATE INDEX t1c ON t1(c, b);
           90  +
           91  +    CREATE TABLE ota.data_t1(a, b, c, ota_rowid, ota_control);
           92  +    INSERT INTO data_t1 VALUES('a', 'b', 'c', 1, 0);
           93  +    INSERT INTO data_t1 VALUES('d', 'e', 'f', '2', 0);
           94  +  } {
           95  +    SELECT * FROM t1 ORDER BY a;
           96  +  } {a b c d e f}
           97  +
    87     98   } {
    88     99     catch {db close}
    89    100     forcedelete ota.db test.db
    90    101     sqlite3 db test.db
    91    102     execsql {
    92    103       PRAGMA encoding = utf16;
    93    104       ATTACH 'ota.db' AS ota;
................................................................................
   114    125         {1 SQLITE_IOERR_WRITE}
   115    126         {1 SQLITE_IOERR_READ}
   116    127         {1 SQLITE_IOERR_FSYNC}
   117    128         {1 {SQLITE_ERROR - SQL logic error or missing database}}
   118    129         {1 {SQLITE_ERROR - unable to open database: ota.db}}
   119    130         {1 {SQLITE_IOERR - unable to open database: ota.db}}
   120    131       }
          132  +
          133  +    3 shmerr-*  {
          134  +      {0 SQLITE_DONE} 
          135  +      {1 {SQLITE_IOERR - disk I/O error}}
          136  +      {1 SQLITE_IOERR}
          137  +    }
   121    138     } {
   122    139   
   123    140       catch {db close}
   124    141       sqlite3_shutdown
   125    142       set lookaside_config [sqlite3_config_lookaside 0 0]
   126    143       sqlite3_initialize
   127    144       autoinstall_test_functions

Changes to ext/ota/sqlite3ota.c.

  1636   1636   }
  1637   1637   
  1638   1638   static int otaCaptureWalRead(sqlite3ota *pOta, i64 iOff, int iAmt){
  1639   1639     const u32 mReq = (1<<WAL_LOCK_WRITE)|(1<<WAL_LOCK_CKPT)|(1<<WAL_LOCK_READ0);
  1640   1640     u32 iFrame;
  1641   1641   
  1642   1642     if( pOta->mLock!=mReq ){
  1643         -    return SQLITE_BUSY;
         1643  +    pOta->rc = SQLITE_BUSY;
         1644  +    return SQLITE_INTERNAL;
  1644   1645     }
  1645   1646   
  1646   1647     pOta->pgsz = iAmt;
  1647   1648     if( pOta->nFrame==pOta->nFrameAlloc ){
  1648   1649       int nNew = (pOta->nFrameAlloc ? pOta->nFrameAlloc : 64) * 2;
  1649   1650       OtaFrame *aNew;
  1650   1651       aNew = (OtaFrame*)sqlite3_realloc(pOta->aFrame, nNew * sizeof(OtaFrame));
................................................................................
  1663   1664   
  1664   1665   static int otaCaptureDbWrite(sqlite3ota *pOta, i64 iOff){
  1665   1666     pOta->aFrame[pOta->nFrame-1].iDbPage = (u32)(iOff / pOta->pgsz) + 1;
  1666   1667     return SQLITE_OK;
  1667   1668   }
  1668   1669   
  1669   1670   static void otaCheckpointFrame(sqlite3ota *p, OtaFrame *pFrame){
  1670         -  if( p->rc==SQLITE_OK ){
  1671         -    sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
  1672         -    sqlite3_file *pDb = p->pTargetFd->pReal;
  1673         -    i64 iOff;
         1671  +  sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
         1672  +  sqlite3_file *pDb = p->pTargetFd->pReal;
         1673  +  i64 iOff;
  1674   1674   
  1675         -    iOff = (i64)(pFrame->iWalFrame-1) * (p->pgsz + 24) + 32 + 24;
  1676         -    p->rc = pWal->pMethods->xRead(pWal, p->aBuf, p->pgsz, iOff);
  1677         -    if( p->rc ) return;
         1675  +  assert( p->rc==SQLITE_OK );
         1676  +  iOff = (i64)(pFrame->iWalFrame-1) * (p->pgsz + 24) + 32 + 24;
         1677  +  p->rc = pWal->pMethods->xRead(pWal, p->aBuf, p->pgsz, iOff);
         1678  +  if( p->rc ) return;
  1678   1679   
  1679         -    iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
  1680         -    p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
  1681         -  }
         1680  +  iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
         1681  +  p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
  1682   1682   }
  1683   1683   
  1684   1684   
  1685   1685   /*
  1686   1686   ** Take an EXCLUSIVE lock on the database file.
  1687   1687   */
  1688   1688   static void otaLockDatabase(sqlite3ota *p){
         1689  +  sqlite3_file *pReal = p->pTargetFd->pReal;
         1690  +  assert( p->rc==SQLITE_OK );
         1691  +  p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED);
  1689   1692     if( p->rc==SQLITE_OK ){
  1690         -    sqlite3_file *pReal = p->pTargetFd->pReal;
  1691         -    p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED);
  1692         -    if( p->rc==SQLITE_OK ){
  1693         -      p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE);
  1694         -    }
         1693  +    p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE);
  1695   1694     }
  1696   1695   }
  1697   1696   
  1698   1697   /*
  1699   1698   ** The OTA handle is currently in OTA_STAGE_OAL state, with a SHARED lock
  1700   1699   ** on the database file. This proc moves the *-oal file to the *-wal path,
  1701   1700   ** then reopens the database file (this time in vanilla, non-oal, WAL mode).
................................................................................
  1856   1855            && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL
  1857   1856           ){
  1858   1857             p->rc = SQLITE_MISMATCH;
  1859   1858             p->zErrmsg = sqlite3_mprintf("datatype mismatch");
  1860   1859             goto step_out;
  1861   1860           }
  1862   1861   
  1863         -        if( eType==SQLITE_DELETE && pIter->zIdx==0 && pIter->abTblPk[i]==0 ){
         1862  +        if( eType==OTA_DELETE && pIter->abTblPk[i]==0 ){
  1864   1863             continue;
  1865   1864           }
  1866   1865   
  1867   1866           pVal = sqlite3_column_value(pIter->pSelect, i);
  1868   1867           p->rc = sqlite3_bind_value(pWriter, i+1, pVal);
  1869         -        if( p->rc==SQLITE_RANGE ) p->rc = SQLITE_OK;
  1870   1868           if( p->rc ) goto step_out;
  1871   1869         }
  1872   1870         if( pIter->zIdx==0
  1873   1871          && (pIter->eType==OTA_PK_VTAB || pIter->eType==OTA_PK_NONE) 
  1874   1872         ){
  1875   1873           /* For a virtual table, or a table with no primary key, the 
  1876   1874           ** SELECT statement is:
................................................................................
  1883   1881           pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
  1884   1882           p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal);
  1885   1883         }
  1886   1884         if( p->rc==SQLITE_OK ){
  1887   1885           sqlite3_step(pWriter);
  1888   1886           p->rc = resetAndCollectError(pWriter, &p->zErrmsg);
  1889   1887         }
  1890         -    }else if( eType==OTA_UPDATE ){
         1888  +    }else{
  1891   1889         sqlite3_value *pVal;
  1892   1890         sqlite3_stmt *pUpdate = 0;
         1891  +      assert( eType==OTA_UPDATE );
  1893   1892         otaGetUpdateStmt(p, pIter, zMask, &pUpdate);
  1894   1893         if( pUpdate ){
  1895   1894           for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){
  1896   1895             char c = zMask[pIter->aiSrcOrder[i]];
  1897   1896             pVal = sqlite3_column_value(pIter->pSelect, i);
  1898   1897             if( pIter->abTblPk[i] || c=='x' || c=='d' ){
  1899   1898               p->rc = sqlite3_bind_value(pUpdate, i+1, pVal);
................................................................................
  1908   1907             p->rc = sqlite3_bind_value(pUpdate, pIter->nCol+1, pVal);
  1909   1908           }
  1910   1909           if( p->rc==SQLITE_OK ){
  1911   1910             sqlite3_step(pUpdate);
  1912   1911             p->rc = resetAndCollectError(pUpdate, &p->zErrmsg);
  1913   1912           }
  1914   1913         }
  1915         -    }else{
  1916         -      /* no-op */
  1917         -      assert( eType==OTA_DELETE && pIter->zIdx );
  1918   1914       }
  1919   1915     }
  1920   1916   
  1921   1917    step_out:
  1922   1918     return p->rc;
  1923   1919   }
  1924   1920   
................................................................................
  1990   1986   ** Step the OTA object.
  1991   1987   */
  1992   1988   int sqlite3ota_step(sqlite3ota *p){
  1993   1989     if( p ){
  1994   1990       switch( p->eStage ){
  1995   1991         case OTA_STAGE_OAL: {
  1996   1992           OtaObjIter *pIter = &p->objiter;
  1997         -        while( p && p->rc==SQLITE_OK && pIter->zTbl ){
         1993  +        while( p->rc==SQLITE_OK && pIter->zTbl ){
  1998   1994   
  1999   1995             if( pIter->bCleanup ){
  2000   1996               /* Clean up the ota_tmp_xxx table for the previous table. It 
  2001   1997               ** cannot be dropped as there are currently active SQL statements.
  2002   1998               ** But the contents can be deleted.  */
  2003   1999               if( pIter->eType!=OTA_PK_VTAB ){
  2004   2000                 otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl);
................................................................................
  2018   2014                 p->nStep = 0;
  2019   2015               }
  2020   2016             }
  2021   2017   
  2022   2018             otaObjIterNext(p, pIter);
  2023   2019           }
  2024   2020   
  2025         -        if( p->rc==SQLITE_OK && pIter->zTbl==0 ){
         2021  +        if( p->rc==SQLITE_OK ){
         2022  +          assert( pIter->zTbl==0 );
  2026   2023             otaSaveState(p, OTA_STAGE_MOVE);
  2027   2024             otaIncrSchemaCookie(p);
  2028   2025             if( p->rc==SQLITE_OK ){
  2029   2026               p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg);
  2030   2027             }
  2031   2028             p->eStage = OTA_STAGE_MOVE;
  2032   2029           }
................................................................................
  2038   2035             otaMoveOalFile(p);
  2039   2036             p->nProgress++;
  2040   2037           }
  2041   2038           break;
  2042   2039         }
  2043   2040   
  2044   2041         case OTA_STAGE_CKPT: {
  2045         -        if( p->nStep>=p->nFrame ){
  2046         -          sqlite3_file *pDb = p->pTargetFd->pReal;
  2047         -
  2048         -          /* Sync the db file */
  2049         -          p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
  2050         -
  2051         -          /* Update nBackfill */
  2052         -          if( p->rc==SQLITE_OK ){
  2053         -            void volatile *ptr;
  2054         -            p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, &ptr);
         2042  +        if( p->rc==SQLITE_OK ){
         2043  +          if( p->nStep>=p->nFrame ){
         2044  +            sqlite3_file *pDb = p->pTargetFd->pReal;
         2045  +  
         2046  +            /* Sync the db file */
         2047  +            p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
         2048  +  
         2049  +            /* Update nBackfill */
         2050  +            if( p->rc==SQLITE_OK ){
         2051  +              void volatile *ptr;
         2052  +              p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, &ptr);
         2053  +              if( p->rc==SQLITE_OK ){
         2054  +                ((u32*)ptr)[12] = p->iMaxFrame;
         2055  +              }
         2056  +            }
         2057  +  
  2055   2058               if( p->rc==SQLITE_OK ){
  2056         -              ((u32*)ptr)[12] = p->iMaxFrame;
         2059  +              p->eStage = OTA_STAGE_DONE;
         2060  +              p->rc = SQLITE_DONE;
  2057   2061               }
  2058         -          }
  2059         -
  2060         -          if( p->rc==SQLITE_OK ){
  2061         -            p->eStage = OTA_STAGE_DONE;
  2062         -            p->rc = SQLITE_DONE;
         2062  +          }else{
         2063  +            OtaFrame *pFrame = &p->aFrame[p->nStep];
         2064  +            otaCheckpointFrame(p, pFrame);
         2065  +            p->nStep++;
  2063   2066             }
  2064         -        }else{
  2065         -          OtaFrame *pFrame = &p->aFrame[p->nStep];
  2066         -          otaCheckpointFrame(p, pFrame);
  2067         -          p->nStep++;
         2067  +          p->nProgress++;
  2068   2068           }
  2069         -
  2070         -        p->nProgress++;
  2071   2069           break;
  2072   2070         }
  2073   2071   
  2074   2072         default:
  2075   2073           break;
  2076   2074       }
  2077   2075       return p->rc;
................................................................................
  2165   2163   static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){
  2166   2164     assert( p->rc==SQLITE_OK );
  2167   2165     if( pState->zTbl ){
  2168   2166       OtaObjIter *pIter = &p->objiter;
  2169   2167       int rc = SQLITE_OK;
  2170   2168   
  2171   2169       while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup 
  2172         -       || otaStrCompare(pIter->zTbl, pState->zTbl) 
  2173   2170          || otaStrCompare(pIter->zIdx, pState->zIdx)
         2171  +       || otaStrCompare(pIter->zTbl, pState->zTbl) 
  2174   2172       )){
  2175         -      rc = otaObjIterNext(p, &p->objiter);
         2173  +      rc = otaObjIterNext(p, pIter);
  2176   2174       }
  2177   2175   
  2178         -    if( rc==SQLITE_OK && !p->objiter.zTbl ){
         2176  +    if( rc==SQLITE_OK && !pIter->zTbl ){
  2179   2177         rc = SQLITE_ERROR;
  2180   2178         p->zErrmsg = sqlite3_mprintf("ota_state mismatch error");
  2181   2179       }
  2182   2180   
  2183   2181       if( rc==SQLITE_OK ){
  2184   2182         p->nStep = pState->nRow;
  2185   2183         rc = otaObjIterPrepareAll(p, &p->objiter, p->nStep);
................................................................................
  2289   2287         }
  2290   2288       }
  2291   2289   
  2292   2290       if( p->rc==SQLITE_OK ){
  2293   2291         if( p->eStage==OTA_STAGE_OAL ){
  2294   2292   
  2295   2293           /* Open the transaction */
  2296         -        if( p->rc==SQLITE_OK ){
  2297         -          p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
  2298         -        }
         2294  +        p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
  2299   2295     
  2300   2296           /* Point the object iterator at the first object */
  2301   2297           if( p->rc==SQLITE_OK ){
  2302   2298             p->rc = otaObjIterFirst(p, &p->objiter);
  2303   2299           }
  2304   2300     
  2305   2301           if( p->rc==SQLITE_OK ){