/ Check-in [2c1d8dda]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest fixes and enhancements from trunk into apple-osx.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1:2c1d8ddab23b36dbdd30ff82401925f97ffc5d76
User & Date: drh 2014-12-16 00:29:06
Context
2015-01-02
19:17
Merge the latest changes from trunk into the apple-osx branch. check-in: df3cdf9f user: drh tags: apple-osx
2014-12-16
00:29
Merge latest fixes and enhancements from trunk into apple-osx. check-in: 2c1d8dda user: drh tags: apple-osx
00:20
Enhanced "stress2" testing in the threadtest3.c test program. check-in: ae43539e user: drh tags: trunk
2014-12-09
15:01
Increase the default PMA size from 10 to 250 pages and provide the SQLITE_SORTER_PMASZ compile-time option to change this default. Add needed mutex call when clearing the KeyInfo cache in shared-cache mode. check-in: 6e2da589 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to main.mk.

   608    608   test:	testfixture$(EXE) sqlite3$(EXE)
   609    609   	./testfixture$(EXE) $(TOP)/test/veryquick.test
   610    610   
   611    611   # The next two rules are used to support the "threadtest" target. Building
   612    612   # threadtest runs a few thread-safety tests that are implemented in C. This
   613    613   # target is invoked by the releasetest.tcl script.
   614    614   # 
   615         -threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c
   616         -	$(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \
   617         -		-o threadtest3$(EXE) $(THREADLIB)
          615  +THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
          616  +                  $(TOP)/test/tt3_checkpoint.c \
          617  +                  $(TOP)/test/tt3_index.c      \
          618  +                  $(TOP)/test/tt3_vacuum.c      \
          619  +                  $(TOP)/test/tt3_stress.c      \
          620  +                  $(TOP)/test/tt3_lookaside1.c
          621  +
          622  +threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC)
          623  +	$(TCCX) $(TOP)/test/threadtest3.c sqlite3.o -o $@ $(THREADLIB)
   618    624   
   619    625   threadtest: threadtest3$(EXE)
   620    626   	./threadtest3$(EXE)
   621    627   
   622    628   TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
   623    629   $(TEST_EXTENSION): $(TOP)/src/test_loadext.c
   624    630   	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)

Changes to src/attach.c.

   146    146       if( !aNew->pSchema ){
   147    147         rc = SQLITE_NOMEM;
   148    148       }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){
   149    149         zErrDyn = sqlite3MPrintf(db, 
   150    150           "attached databases must use the same text encoding as main database");
   151    151         rc = SQLITE_ERROR;
   152    152       }
          153  +    sqlite3BtreeEnter(aNew->pBt);
   153    154       pPager = sqlite3BtreePager(aNew->pBt);
   154    155       sqlite3PagerLockingMode(pPager, db->dfltLockMode);
   155    156       sqlite3BtreeSecureDelete(aNew->pBt,
   156    157                                sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) );
   157    158   #ifndef SQLITE_OMIT_PAGER_PRAGMAS
   158    159       sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK));
   159    160   #endif
          161  +    sqlite3BtreeLeave(aNew->pBt);
   160    162     }
   161    163     aNew->safety_level = 3;
   162    164     aNew->zName = sqlite3DbStrDup(db, zName);
   163    165     if( rc==SQLITE_OK && aNew->zName==0 ){
   164    166       rc = SQLITE_NOMEM;
   165    167     }
   166    168   

Changes to src/btree.c.

  3936   3936       if( pCur->pNext ){
  3937   3937         pCur->pNext->pPrev = pCur->pPrev;
  3938   3938       }
  3939   3939       for(i=0; i<=pCur->iPage; i++){
  3940   3940         releasePage(pCur->apPage[i]);
  3941   3941       }
  3942   3942       unlockBtreeIfUnused(pBt);
  3943         -    sqlite3DbFree(pBtree->db, pCur->aOverflow);
         3943  +    sqlite3_free(pCur->aOverflow);
  3944   3944       /* sqlite3_free(pCur); */
  3945   3945       sqlite3BtreeLeave(pBtree);
  3946   3946     }
  3947   3947     return SQLITE_OK;
  3948   3948   }
  3949   3949   
  3950   3950   /*
................................................................................
  4230   4230       rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage);
  4231   4231       offset = 0;
  4232   4232       pBuf += a;
  4233   4233       amt -= a;
  4234   4234     }else{
  4235   4235       offset -= pCur->info.nLocal;
  4236   4236     }
         4237  +
  4237   4238   
  4238   4239     if( rc==SQLITE_OK && amt>0 ){
  4239   4240       const u32 ovflSize = pBt->usableSize - 4;  /* Bytes content per ovfl page */
  4240   4241       Pgno nextPage;
  4241   4242   
  4242   4243       nextPage = get4byte(&aPayload[pCur->info.nLocal]);
  4243   4244   
................................................................................
  4248   4249       ** in the overflow chain. The page number of the first overflow page is
  4249   4250       ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array
  4250   4251       ** means "not yet known" (the cache is lazily populated).
  4251   4252       */
  4252   4253       if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){
  4253   4254         int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize;
  4254   4255         if( nOvfl>pCur->nOvflAlloc ){
  4255         -        Pgno *aNew = (Pgno*)sqlite3DbRealloc(
  4256         -            pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno)
         4256  +        Pgno *aNew = (Pgno*)sqlite3Realloc(
         4257  +            pCur->aOverflow, nOvfl*2*sizeof(Pgno)
  4257   4258           );
  4258   4259           if( aNew==0 ){
  4259   4260             rc = SQLITE_NOMEM;
  4260   4261           }else{
  4261   4262             pCur->nOvflAlloc = nOvfl*2;
  4262   4263             pCur->aOverflow = aNew;
  4263   4264           }
................................................................................
  4296   4297           ** function.
  4297   4298           **
  4298   4299           ** Note that the aOverflow[] array must be allocated because eOp!=2
  4299   4300           ** here.  If eOp==2, then offset==0 and this branch is never taken.
  4300   4301           */
  4301   4302           assert( eOp!=2 );
  4302   4303           assert( pCur->curFlags & BTCF_ValidOvfl );
         4304  +        assert( pCur->pBtree->db==pBt->db );
  4303   4305           if( pCur->aOverflow[iIdx+1] ){
  4304   4306             nextPage = pCur->aOverflow[iIdx+1];
  4305   4307           }else{
  4306   4308             rc = getOverflowPage(pBt, nextPage, 0, &nextPage);
  4307   4309           }
  4308   4310           offset -= ovflSize;
  4309   4311         }else{
................................................................................
  8299   8301       ** caller.
  8300   8302       */
  8301   8303       if( pPage->leaf ){
  8302   8304         do {
  8303   8305           if( pCur->iPage==0 ){
  8304   8306             /* All pages of the b-tree have been visited. Return successfully. */
  8305   8307             *pnEntry = nEntry;
  8306         -          return SQLITE_OK;
         8308  +          return moveToRoot(pCur);
  8307   8309           }
  8308   8310           moveToParent(pCur);
  8309   8311         }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell );
  8310   8312   
  8311   8313         pCur->aiIdx[pCur->iPage]++;
  8312   8314         pPage = pCur->apPage[pCur->iPage];
  8313   8315       }

Changes to src/build.c.

   431    431   /*
   432    432   ** Reclaim the memory used by an index
   433    433   */
   434    434   static void freeIndex(sqlite3 *db, Index *p){
   435    435   #ifndef SQLITE_OMIT_ANALYZE
   436    436     sqlite3DeleteIndexSamples(db, p);
   437    437   #endif
   438         -  if( db==0 || db->pnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo);
   439    438     sqlite3ExprDelete(db, p->pPartIdxWhere);
   440    439     sqlite3DbFree(db, p->zColAff);
   441    440     if( p->isResized ) sqlite3DbFree(db, p->azColl);
   442    441   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   443    442     sqlite3_free(p->aiRowEst);
   444    443   #endif
   445    444     sqlite3DbFree(db, p);
................................................................................
  4186   4185   ** So there might be multiple references to the returned pointer.  The
  4187   4186   ** caller should not try to modify the KeyInfo object.
  4188   4187   **
  4189   4188   ** The caller should invoke sqlite3KeyInfoUnref() on the returned object
  4190   4189   ** when it has finished using it.
  4191   4190   */
  4192   4191   KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){
         4192  +  int i;
         4193  +  int nCol = pIdx->nColumn;
         4194  +  int nKey = pIdx->nKeyCol;
         4195  +  KeyInfo *pKey;
  4193   4196     if( pParse->nErr ) return 0;
  4194         -#ifndef SQLITE_OMIT_SHARED_CACHE
  4195         -  if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){
  4196         -    sqlite3KeyInfoUnref(pIdx->pKeyInfo);
  4197         -    pIdx->pKeyInfo = 0;
         4197  +  if( pIdx->uniqNotNull ){
         4198  +    pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey);
         4199  +  }else{
         4200  +    pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0);
  4198   4201     }
  4199         -#endif
  4200         -  if( pIdx->pKeyInfo==0 ){
  4201         -    int i;
  4202         -    int nCol = pIdx->nColumn;
  4203         -    int nKey = pIdx->nKeyCol;
  4204         -    KeyInfo *pKey;
  4205         -    if( pIdx->uniqNotNull ){
  4206         -      pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey);
  4207         -    }else{
  4208         -      pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0);
         4202  +  if( pKey ){
         4203  +    assert( sqlite3KeyInfoIsWriteable(pKey) );
         4204  +    for(i=0; i<nCol; i++){
         4205  +      char *zColl = pIdx->azColl[i];
         4206  +      assert( zColl!=0 );
         4207  +      pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 :
         4208  +                        sqlite3LocateCollSeq(pParse, zColl);
         4209  +      pKey->aSortOrder[i] = pIdx->aSortOrder[i];
  4209   4210       }
  4210         -    if( pKey ){
  4211         -      assert( sqlite3KeyInfoIsWriteable(pKey) );
  4212         -      for(i=0; i<nCol; i++){
  4213         -        char *zColl = pIdx->azColl[i];
  4214         -        assert( zColl!=0 );
  4215         -        pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 :
  4216         -                          sqlite3LocateCollSeq(pParse, zColl);
  4217         -        pKey->aSortOrder[i] = pIdx->aSortOrder[i];
  4218         -      }
  4219         -      if( pParse->nErr ){
  4220         -        sqlite3KeyInfoUnref(pKey);
  4221         -      }else{
  4222         -        pIdx->pKeyInfo = pKey;
  4223         -      }
         4211  +    if( pParse->nErr ){
         4212  +      sqlite3KeyInfoUnref(pKey);
         4213  +      pKey = 0;
  4224   4214       }
  4225   4215     }
  4226         -  return sqlite3KeyInfoRef(pIdx->pKeyInfo);
         4216  +  return pKey;
  4227   4217   }
  4228   4218   
  4229   4219   #ifndef SQLITE_OMIT_CTE
  4230   4220   /* 
  4231   4221   ** This routine is invoked once per CTE by the parser while parsing a 
  4232   4222   ** WITH clause. 
  4233   4223   */

Changes to src/loadext.c.

    30     30   #ifndef SQLITE_ENABLE_COLUMN_METADATA
    31     31   # define sqlite3_column_database_name   0
    32     32   # define sqlite3_column_database_name16 0
    33     33   # define sqlite3_column_table_name      0
    34     34   # define sqlite3_column_table_name16    0
    35     35   # define sqlite3_column_origin_name     0
    36     36   # define sqlite3_column_origin_name16   0
    37         -# define sqlite3_table_column_metadata  0
    38     37   #endif
    39     38   
    40     39   #ifdef SQLITE_OMIT_AUTHORIZATION
    41     40   # define sqlite3_set_authorizer         0
    42     41   #endif
    43     42   
    44     43   #ifdef SQLITE_OMIT_UTF16

Changes to src/main.c.

  1031   1031     /* Free any outstanding Savepoint structures. */
  1032   1032     sqlite3CloseSavepoints(db);
  1033   1033   
  1034   1034     /* Close all database connections */
  1035   1035     for(j=0; j<db->nDb; j++){
  1036   1036       struct Db *pDb = &db->aDb[j];
  1037   1037       if( pDb->pBt ){
  1038         -      if( pDb->pSchema ){
  1039         -        /* Must clear the KeyInfo cache.  See ticket [e4a18565a36884b00edf] */
  1040         -        sqlite3BtreeEnter(pDb->pBt);
  1041         -        for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){
  1042         -          Index *pIdx = sqliteHashData(i);
  1043         -          sqlite3KeyInfoUnref(pIdx->pKeyInfo);
  1044         -          pIdx->pKeyInfo = 0;
  1045         -        }
  1046         -        sqlite3BtreeLeave(pDb->pBt);
  1047         -      }
  1048   1038         sqlite3BtreeClose(pDb->pBt);
  1049   1039         pDb->pBt = 0;
  1050   1040         if( j!=1 ){
  1051   1041           pDb->pSchema = 0;
  1052   1042         }
  1053   1043       }
  1054   1044     }
................................................................................
  2172   2162   ** argument.  For now, this simply calls the internal sqlite3ErrStr()
  2173   2163   ** function.
  2174   2164   */
  2175   2165   const char *sqlite3_errstr(int rc){
  2176   2166     return sqlite3ErrStr(rc);
  2177   2167   }
  2178   2168   
  2179         -/*
  2180         -** Invalidate all cached KeyInfo objects for database connection "db"
  2181         -*/
  2182         -static void invalidateCachedKeyInfo(sqlite3 *db){
  2183         -  Db *pDb;                    /* A single database */
  2184         -  int iDb;                    /* The database index number */
  2185         -  HashElem *k;                /* For looping over tables in pDb */
  2186         -  Table *pTab;                /* A table in the database */
  2187         -  Index *pIdx;                /* Each index */
  2188         -
  2189         -  for(iDb=0, pDb=db->aDb; iDb<db->nDb; iDb++, pDb++){
  2190         -    if( pDb->pBt==0 ) continue;
  2191         -    sqlite3BtreeEnter(pDb->pBt);
  2192         -    for(k=sqliteHashFirst(&pDb->pSchema->tblHash);  k; k=sqliteHashNext(k)){
  2193         -      pTab = (Table*)sqliteHashData(k);
  2194         -      for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
  2195         -        if( pIdx->pKeyInfo && pIdx->pKeyInfo->db==db ){
  2196         -          sqlite3KeyInfoUnref(pIdx->pKeyInfo);
  2197         -          pIdx->pKeyInfo = 0;
  2198         -        }
  2199         -      }
  2200         -    }
  2201         -    sqlite3BtreeLeave(pDb->pBt);
  2202         -  }
  2203         -}
  2204         -
  2205   2169   /*
  2206   2170   ** Create a new collating function for database "db".  The name is zName
  2207   2171   ** and the encoding is enc.
  2208   2172   */
  2209   2173   static int createCollation(
  2210   2174     sqlite3* db,
  2211   2175     const char *zName, 
................................................................................
  2241   2205     if( pColl && pColl->xCmp ){
  2242   2206       if( db->nVdbeActive ){
  2243   2207         sqlite3ErrorWithMsg(db, SQLITE_BUSY, 
  2244   2208           "unable to delete/modify collation sequence due to active statements");
  2245   2209         return SQLITE_BUSY;
  2246   2210       }
  2247   2211       sqlite3ExpirePreparedStatements(db);
  2248         -    invalidateCachedKeyInfo(db);
  2249   2212   
  2250   2213       /* If collation sequence pColl was created directly by a call to
  2251   2214       ** sqlite3_create_collation, and not generated by synthCollSeq(),
  2252   2215       ** then any copies made by synthCollSeq() need to be invalidated.
  2253   2216       ** Also, collation destructor - CollSeq.xDel() - function may need
  2254   2217       ** to be called.
  2255   2218       */ 
................................................................................
  2838   2801   #endif
  2839   2802   #if SQLITE_DEFAULT_RECURSIVE_TRIGGERS
  2840   2803                    | SQLITE_RecTriggers
  2841   2804   #endif
  2842   2805   #if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS
  2843   2806                    | SQLITE_ForeignKeys
  2844   2807   #endif
         2808  +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS)
         2809  +                 | SQLITE_ReverseOrder
         2810  +#endif
  2845   2811         ;
  2846   2812     sqlite3HashInit(&db->aCollSeq);
  2847   2813   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2848   2814     sqlite3HashInit(&db->aModule);
  2849   2815   #endif
  2850   2816   
  2851   2817     /* Add the default collation sequence BINARY. BINARY works for both UTF-8
................................................................................
  2887   2853         rc = SQLITE_NOMEM;
  2888   2854       }
  2889   2855       sqlite3Error(db, rc);
  2890   2856       goto opendb_out;
  2891   2857     }
  2892   2858     sqlite3BtreeEnter(db->aDb[0].pBt);
  2893   2859     db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt);
         2860  +  if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db);
  2894   2861     sqlite3BtreeLeave(db->aDb[0].pBt);
  2895   2862     db->aDb[1].pSchema = sqlite3SchemaGet(db, 0);
  2896   2863   
  2897   2864     /* The default safety_level for the main database is 'full'; for the temp
  2898   2865     ** database it is 'NONE'. This matches the pager layer defaults.  
  2899   2866     */
  2900   2867     db->aDb[0].zName = "main";
................................................................................
  3068   3035     sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC);
  3069   3036     zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8);
  3070   3037     if( zFilename8 ){
  3071   3038       rc = openDatabase(zFilename8, ppDb,
  3072   3039                         SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0);
  3073   3040       assert( *ppDb || rc==SQLITE_NOMEM );
  3074   3041       if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){
  3075         -      ENC(*ppDb) = SQLITE_UTF16NATIVE;
         3042  +      SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE;
  3076   3043       }
  3077   3044     }else{
  3078   3045       rc = SQLITE_NOMEM;
  3079   3046     }
  3080   3047     sqlite3ValueFree(pVal);
  3081   3048   
  3082   3049     return sqlite3ApiExit(0, rc);
................................................................................
  3264   3231   }
  3265   3232   #endif
  3266   3233   
  3267   3234   /*
  3268   3235   ** Return meta information about a specific column of a database table.
  3269   3236   ** See comment in sqlite3.h (sqlite.h.in) for details.
  3270   3237   */
  3271         -#ifdef SQLITE_ENABLE_COLUMN_METADATA
  3272   3238   int sqlite3_table_column_metadata(
  3273   3239     sqlite3 *db,                /* Connection handle */
  3274   3240     const char *zDbName,        /* Database name or NULL */
  3275   3241     const char *zTableName,     /* Table name */
  3276   3242     const char *zColumnName,    /* Column name */
  3277   3243     char const **pzDataType,    /* OUTPUT: Declared data type */
  3278   3244     char const **pzCollSeq,     /* OUTPUT: Collation sequence name */
................................................................................
  3304   3270     pTab = sqlite3FindTable(db, zTableName, zDbName);
  3305   3271     if( !pTab || pTab->pSelect ){
  3306   3272       pTab = 0;
  3307   3273       goto error_out;
  3308   3274     }
  3309   3275   
  3310   3276     /* Find the column for which info is requested */
  3311         -  if( sqlite3IsRowid(zColumnName) ){
  3312         -    iCol = pTab->iPKey;
  3313         -    if( iCol>=0 ){
  3314         -      pCol = &pTab->aCol[iCol];
  3315         -    }
         3277  +  if( zColumnName==0 ){
         3278  +    /* Query for existance of table only */
  3316   3279     }else{
  3317   3280       for(iCol=0; iCol<pTab->nCol; iCol++){
  3318   3281         pCol = &pTab->aCol[iCol];
  3319   3282         if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){
  3320   3283           break;
  3321   3284         }
  3322   3285       }
  3323   3286       if( iCol==pTab->nCol ){
  3324         -      pTab = 0;
  3325         -      goto error_out;
         3287  +      if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){
         3288  +        iCol = pTab->iPKey;
         3289  +        pCol = iCol>=0 ? &pTab->aCol[iCol] : 0;
         3290  +      }else{
         3291  +        pTab = 0;
         3292  +        goto error_out;
         3293  +      }
  3326   3294       }
  3327   3295     }
  3328   3296   
  3329   3297     /* The following block stores the meta information that will be returned
  3330   3298     ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey
  3331   3299     ** and autoinc. At this point there are two possibilities:
  3332   3300     ** 
................................................................................
  3371   3339     }
  3372   3340     sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg);
  3373   3341     sqlite3DbFree(db, zErrMsg);
  3374   3342     rc = sqlite3ApiExit(db, rc);
  3375   3343     sqlite3_mutex_leave(db->mutex);
  3376   3344     return rc;
  3377   3345   }
  3378         -#endif
  3379   3346   
  3380   3347   /*
  3381   3348   ** Sleep for a little while.  Return the amount of time slept.
  3382   3349   */
  3383   3350   int sqlite3_sleep(int ms){
  3384   3351     sqlite3_vfs *pVfs;
  3385   3352     int rc;

Changes to src/os_win.c.

  1199   1199   ** the sqlite3_memory_used() function does not return zero, SQLITE_BUSY will
  1200   1200   ** be returned and no changes will be made to the Win32 native heap.
  1201   1201   */
  1202   1202   int sqlite3_win32_reset_heap(){
  1203   1203     int rc;
  1204   1204     MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */
  1205   1205     MUTEX_LOGIC( sqlite3_mutex *pMem; )    /* The memsys static mutex */
  1206         -  MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); )
  1207         -  MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); )
         1206  +  MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); )
         1207  +  MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); )
  1208   1208     sqlite3_mutex_enter(pMaster);
  1209   1209     sqlite3_mutex_enter(pMem);
  1210   1210     winMemAssertMagic();
  1211   1211     if( winMemGetHeap()!=NULL && winMemGetOwned() && sqlite3_memory_used()==0 ){
  1212   1212       /*
  1213   1213       ** At this point, there should be no outstanding memory allocations on
  1214   1214       ** the heap.  Also, since both the master and memsys locks are currently

Changes to src/pragma.c.

  2092   2092         */
  2093   2093         if( 
  2094   2094           !(DbHasProperty(db, 0, DB_SchemaLoaded)) || 
  2095   2095           DbHasProperty(db, 0, DB_Empty) 
  2096   2096         ){
  2097   2097           for(pEnc=&encnames[0]; pEnc->zName; pEnc++){
  2098   2098             if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){
  2099         -            ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE;
         2099  +            SCHEMA_ENC(db) = ENC(db) =
         2100  +                pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE;
  2100   2101               break;
  2101   2102             }
  2102   2103           }
  2103   2104           if( !pEnc->zName ){
  2104   2105             sqlite3ErrorMsg(pParse, "unsupported encoding: %s", zRight);
  2105   2106           }
  2106   2107         }

Changes to src/prepare.c.

   393    393   ** file was of zero-length, then the DB_Empty flag is also set.
   394    394   */
   395    395   int sqlite3Init(sqlite3 *db, char **pzErrMsg){
   396    396     int i, rc;
   397    397     int commit_internal = !(db->flags&SQLITE_InternChanges);
   398    398     
   399    399     assert( sqlite3_mutex_held(db->mutex) );
          400  +  assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) );
   400    401     assert( db->init.busy==0 );
   401    402     rc = SQLITE_OK;
   402    403     db->init.busy = 1;
          404  +  ENC(db) = SCHEMA_ENC(db);
   403    405     for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
   404    406       if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue;
   405    407       rc = sqlite3InitOne(db, i, pzErrMsg);
   406    408       if( rc ){
   407    409         sqlite3ResetOneSchema(db, i);
   408    410       }
   409    411     }

Changes to src/sqlite.h.in.

   192    192   ** the desired setting of the [SQLITE_THREADSAFE] macro.
   193    193   **
   194    194   ** This interface only reports on the compile-time mutex setting
   195    195   ** of the [SQLITE_THREADSAFE] flag.  If SQLite is compiled with
   196    196   ** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but
   197    197   ** can be fully or partially disabled using a call to [sqlite3_config()]
   198    198   ** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD],
   199         -** or [SQLITE_CONFIG_MUTEX].  ^(The return value of the
          199  +** or [SQLITE_CONFIG_SERIALIZED].  ^(The return value of the
   200    200   ** sqlite3_threadsafe() function shows only the compile-time setting of
   201    201   ** thread safety, not any run-time changes to that setting made by
   202    202   ** sqlite3_config(). In other words, the return value from sqlite3_threadsafe()
   203    203   ** is unchanged by calls to sqlite3_config().)^
   204    204   **
   205    205   ** See the [threading mode] documentation for additional information.
   206    206   */
................................................................................
  5158   5158   */
  5159   5159   SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N);
  5160   5160   
  5161   5161   
  5162   5162   /*
  5163   5163   ** CAPI3REF: Extract Metadata About A Column Of A Table
  5164   5164   **
  5165         -** ^This routine returns metadata about a specific column of a specific
  5166         -** database table accessible using the [database connection] handle
  5167         -** passed as the first function argument.
         5165  +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns
         5166  +** information about column C of table T in database D
         5167  +** on [database connection] X.)^  ^The sqlite3_table_column_metadata()
         5168  +** interface returns SQLITE_OK and fills in the non-NULL pointers in
         5169  +** the final five arguments with appropriate values if the specified
         5170  +** column exists.  ^The sqlite3_table_column_metadata() interface returns
         5171  +** SQLITE_ERROR and if the specified column does not exist.
         5172  +** ^If the column-name parameter to sqlite3_table_column_metadata() is a
         5173  +** NULL pointer, then this routine simply checks for the existance of the
         5174  +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it
         5175  +** does not.
  5168   5176   **
  5169   5177   ** ^The column is identified by the second, third and fourth parameters to
  5170         -** this function. ^The second parameter is either the name of the database
         5178  +** this function. ^(The second parameter is either the name of the database
  5171   5179   ** (i.e. "main", "temp", or an attached database) containing the specified
  5172         -** table or NULL. ^If it is NULL, then all attached databases are searched
         5180  +** table or NULL.)^ ^If it is NULL, then all attached databases are searched
  5173   5181   ** for the table using the same algorithm used by the database engine to
  5174   5182   ** resolve unqualified table references.
  5175   5183   **
  5176   5184   ** ^The third and fourth parameters to this function are the table and column
  5177         -** name of the desired column, respectively. Neither of these parameters
  5178         -** may be NULL.
         5185  +** name of the desired column, respectively.
  5179   5186   **
  5180   5187   ** ^Metadata is returned by writing to the memory locations passed as the 5th
  5181   5188   ** and subsequent parameters to this function. ^Any of these arguments may be
  5182   5189   ** NULL, in which case the corresponding element of metadata is omitted.
  5183   5190   **
  5184   5191   ** ^(<blockquote>
  5185   5192   ** <table border="1">
................................................................................
  5190   5197   ** <tr><td> 7th <td> int         <td> True if column has a NOT NULL constraint
  5191   5198   ** <tr><td> 8th <td> int         <td> True if column is part of the PRIMARY KEY
  5192   5199   ** <tr><td> 9th <td> int         <td> True if column is [AUTOINCREMENT]
  5193   5200   ** </table>
  5194   5201   ** </blockquote>)^
  5195   5202   **
  5196   5203   ** ^The memory pointed to by the character pointers returned for the
  5197         -** declaration type and collation sequence is valid only until the next
         5204  +** declaration type and collation sequence is valid until the next
  5198   5205   ** call to any SQLite API function.
  5199   5206   **
  5200   5207   ** ^If the specified table is actually a view, an [error code] is returned.
  5201   5208   **
  5202         -** ^If the specified column is "rowid", "oid" or "_rowid_" and an
         5209  +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table 
         5210  +** is not a [WITHOUT ROWID] table and an
  5203   5211   ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output
  5204   5212   ** parameters are set for the explicitly declared column. ^(If there is no
  5205         -** explicitly declared [INTEGER PRIMARY KEY] column, then the output
  5206         -** parameters are set as follows:
         5213  +** [INTEGER PRIMARY KEY] column, then the outputs
         5214  +** for the [rowid] are set as follows:
  5207   5215   **
  5208   5216   ** <pre>
  5209   5217   **     data type: "INTEGER"
  5210   5218   **     collation sequence: "BINARY"
  5211   5219   **     not null: 0
  5212   5220   **     primary key: 1
  5213   5221   **     auto increment: 0
  5214   5222   ** </pre>)^
  5215   5223   **
  5216         -** ^(This function may load one or more schemas from database files. If an
  5217         -** error occurs during this process, or if the requested table or column
  5218         -** cannot be found, an [error code] is returned and an error message left
  5219         -** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^
  5220         -**
  5221         -** ^This API is only available if the library was compiled with the
  5222         -** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined.
         5224  +** ^This function causes all database schemas to be read from disk and
         5225  +** parsed, if that has not already been done, and returns an error if
         5226  +** any errors are encountered while loading the schema.
  5223   5227   */
  5224   5228   int sqlite3_table_column_metadata(
  5225   5229     sqlite3 *db,                /* Connection handle */
  5226   5230     const char *zDbName,        /* Database name or NULL */
  5227   5231     const char *zTableName,     /* Table name */
  5228   5232     const char *zColumnName,    /* Column name */
  5229   5233     char const **pzDataType,    /* OUTPUT: Declared data type */
................................................................................
  7182   7186   */
  7183   7187   void sqlite3_log(int iErrCode, const char *zFormat, ...);
  7184   7188   
  7185   7189   /*
  7186   7190   ** CAPI3REF: Write-Ahead Log Commit Hook
  7187   7191   **
  7188   7192   ** ^The [sqlite3_wal_hook()] function is used to register a callback that
  7189         -** will be invoked each time a database connection commits data to a
  7190         -** [write-ahead log] (i.e. whenever a transaction is committed in
  7191         -** [journal_mode | journal_mode=WAL mode]). 
         7193  +** is invoked each time data is committed to a database in wal mode.
  7192   7194   **
  7193         -** ^The callback is invoked by SQLite after the commit has taken place and 
  7194         -** the associated write-lock on the database released, so the implementation 
         7195  +** ^(The callback is invoked by SQLite after the commit has taken place and 
         7196  +** the associated write-lock on the database released)^, so the implementation 
  7195   7197   ** may read, write or [checkpoint] the database as required.
  7196   7198   **
  7197   7199   ** ^The first parameter passed to the callback function when it is invoked
  7198   7200   ** is a copy of the third parameter passed to sqlite3_wal_hook() when
  7199   7201   ** registering the callback. ^The second is a copy of the database handle.
  7200   7202   ** ^The third parameter is the name of the database that was written to -
  7201   7203   ** either "main" or the name of an [ATTACH]-ed database. ^The fourth parameter

Changes to src/sqliteInt.h.

  1065   1065     int flags;                    /* Miscellaneous flags. See below */
  1066   1066     i64 lastRowid;                /* ROWID of most recent insert (see above) */
  1067   1067     i64 szMmap;                   /* Default mmap_size setting */
  1068   1068     unsigned int openFlags;       /* Flags passed to sqlite3_vfs.xOpen() */
  1069   1069     int errCode;                  /* Most recent error code (SQLITE_*) */
  1070   1070     int errMask;                  /* & result codes with this before returning */
  1071   1071     u16 dbOptFlags;               /* Flags to enable/disable optimizations */
         1072  +  u8 enc;                       /* Text encoding */
  1072   1073     u8 autoCommit;                /* The auto-commit flag. */
  1073   1074     u8 temp_store;                /* 1: file 2: memory 0: default */
  1074   1075     u8 mallocFailed;              /* True if we have seen a malloc failure */
  1075   1076     u8 dfltLockMode;              /* Default locking-mode for attached dbs */
  1076   1077     signed char nextAutovac;      /* Autovac setting after VACUUM if >=0 */
  1077   1078     u8 suppressErr;               /* Do not issue error messages if true */
  1078   1079     u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
................................................................................
  1166   1167     sqlite3_userauth auth;        /* User authentication information */
  1167   1168   #endif
  1168   1169   };
  1169   1170   
  1170   1171   /*
  1171   1172   ** A macro to discover the encoding of a database.
  1172   1173   */
  1173         -#define ENC(db) ((db)->aDb[0].pSchema->enc)
         1174  +#define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc)
         1175  +#define ENC(db)        ((db)->enc)
  1174   1176   
  1175   1177   /*
  1176   1178   ** Possible values for the sqlite3.flags.
  1177   1179   */
  1178   1180   #define SQLITE_VdbeTrace      0x00000001  /* True to trace VDBE execution */
  1179   1181   #define SQLITE_InternChanges  0x00000002  /* Uncommitted Hash table changes */
  1180   1182   #define SQLITE_FullFSync      0x00000004  /* Use full fsync on the backend */
................................................................................
  1790   1792     Table *pTable;           /* The SQL table being indexed */
  1791   1793     char *zColAff;           /* String defining the affinity of each column */
  1792   1794     Index *pNext;            /* The next index associated with the same table */
  1793   1795     Schema *pSchema;         /* Schema containing this index */
  1794   1796     u8 *aSortOrder;          /* for each column: True==DESC, False==ASC */
  1795   1797     char **azColl;           /* Array of collation sequence names for index */
  1796   1798     Expr *pPartIdxWhere;     /* WHERE clause for partial indices */
  1797         -  KeyInfo *pKeyInfo;       /* A KeyInfo object suitable for this index */
  1798   1799     int tnum;                /* DB Page containing root of this index */
  1799   1800     LogEst szIdxRow;         /* Estimated average row size in bytes */
  1800   1801     u16 nKeyCol;             /* Number of columns forming the key */
  1801   1802     u16 nColumn;             /* Number of columns stored in the index */
  1802   1803     u8 onError;              /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
  1803   1804     unsigned idxType:2;      /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */
  1804   1805     unsigned bUnordered:1;   /* Use this index for == or IN queries only */

Changes to src/tclsqlite.c.

   631    631   ){
   632    632     int ret = SQLITE_OK;
   633    633     Tcl_Obj *p;
   634    634     SqliteDb *pDb = (SqliteDb*)clientData;
   635    635     Tcl_Interp *interp = pDb->interp;
   636    636     assert(pDb->pWalHook);
   637    637   
          638  +  assert( db==pDb->db );
   638    639     p = Tcl_DuplicateObj(pDb->pWalHook);
   639    640     Tcl_IncrRefCount(p);
   640    641     Tcl_ListObjAppendElement(interp, p, Tcl_NewStringObj(zDb, -1));
   641    642     Tcl_ListObjAppendElement(interp, p, Tcl_NewIntObj(nEntry));
   642    643     if( TCL_OK!=Tcl_EvalObjEx(interp, p, 0) 
   643    644      || TCL_OK!=Tcl_GetIntFromObj(interp, Tcl_GetObjResult(interp), &ret)
   644    645     ){

Changes to src/test1.c.

  1565   1565     return TCL_OK;
  1566   1566   }
  1567   1567   
  1568   1568   /*
  1569   1569   ** Usage: sqlite3_table_column_metadata DB dbname tblname colname
  1570   1570   **
  1571   1571   */
  1572         -#ifdef SQLITE_ENABLE_COLUMN_METADATA
  1573   1572   static int test_table_column_metadata(
  1574   1573     ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
  1575   1574     Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
  1576   1575     int objc,              /* Number of arguments */
  1577   1576     Tcl_Obj *CONST objv[]  /* Command arguments */
  1578   1577   ){
  1579   1578     sqlite3 *db;
................................................................................
  1585   1584   
  1586   1585     const char *zDatatype;
  1587   1586     const char *zCollseq;
  1588   1587     int notnull;
  1589   1588     int primarykey;
  1590   1589     int autoincrement;
  1591   1590   
  1592         -  if( objc!=5 ){
         1591  +  if( objc!=5 && objc!=4 ){
  1593   1592       Tcl_WrongNumArgs(interp, 1, objv, "DB dbname tblname colname");
  1594   1593       return TCL_ERROR;
  1595   1594     }
  1596   1595     if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
  1597   1596     zDb = Tcl_GetString(objv[2]);
  1598   1597     zTbl = Tcl_GetString(objv[3]);
  1599         -  zCol = Tcl_GetString(objv[4]);
         1598  +  zCol = objc==5 ? Tcl_GetString(objv[4]) : 0;
  1600   1599   
  1601   1600     if( strlen(zDb)==0 ) zDb = 0;
  1602   1601   
  1603   1602     rc = sqlite3_table_column_metadata(db, zDb, zTbl, zCol, 
  1604   1603         &zDatatype, &zCollseq, &notnull, &primarykey, &autoincrement);
  1605   1604   
  1606   1605     if( rc!=SQLITE_OK ){
................................................................................
  1614   1613     Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(notnull));
  1615   1614     Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(primarykey));
  1616   1615     Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(autoincrement));
  1617   1616     Tcl_SetObjResult(interp, pRet);
  1618   1617   
  1619   1618     return TCL_OK;
  1620   1619   }
  1621         -#endif
  1622   1620   
  1623   1621   #ifndef SQLITE_OMIT_INCRBLOB
  1624   1622   
  1625   1623   static int blobHandleFromObj(
  1626   1624     Tcl_Interp *interp, 
  1627   1625     Tcl_Obj *pObj,
  1628   1626     sqlite3_blob **ppBlob
................................................................................
  5902   5900     )){
  5903   5901       return TCL_ERROR;
  5904   5902     }
  5905   5903   
  5906   5904     rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt);
  5907   5905     if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){
  5908   5906       const char *zErrCode = sqlite3ErrName(rc);
         5907  +    Tcl_ResetResult(interp);
  5909   5908       Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0);
  5910   5909       return TCL_ERROR;
  5911   5910     }
  5912   5911   
  5913   5912     pRet = Tcl_NewObj();
  5914   5913     Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(rc==SQLITE_BUSY?1:0));
  5915   5914     Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nLog));
  5916   5915     Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nCkpt));
  5917   5916     Tcl_SetObjResult(interp, pRet);
  5918   5917   
  5919   5918     return TCL_OK;
  5920   5919   }
         5920  +
         5921  +/*
         5922  +** tclcmd:  sqlite3_wal_autocheckpoint db VALUE
         5923  +*/
         5924  +static int test_wal_autocheckpoint(
         5925  +  ClientData clientData, /* Unused */
         5926  +  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
         5927  +  int objc,              /* Number of arguments */
         5928  +  Tcl_Obj *CONST objv[]  /* Command arguments */
         5929  +){
         5930  +  sqlite3 *db;
         5931  +  int rc;
         5932  +  int iVal;
         5933  +
         5934  +
         5935  +  if( objc!=3 ){
         5936  +    Tcl_WrongNumArgs(interp, 1, objv, "DB VALUE");
         5937  +    return TCL_ERROR;
         5938  +  }
         5939  +
         5940  +  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) 
         5941  +   || Tcl_GetIntFromObj(0, objv[2], &iVal)
         5942  +  ){
         5943  +    return TCL_ERROR;
         5944  +  }
         5945  +
         5946  +  rc = sqlite3_wal_autocheckpoint(db, iVal);
         5947  +  Tcl_ResetResult(interp);
         5948  +  if( rc!=SQLITE_OK ){
         5949  +    const char *zErrCode = sqlite3ErrName(rc);
         5950  +    Tcl_SetObjResult(interp, Tcl_NewStringObj(zErrCode, -1));
         5951  +    return TCL_ERROR;
         5952  +  }
         5953  +
         5954  +  return TCL_OK;
         5955  +}
         5956  +
  5921   5957   
  5922   5958   /*
  5923   5959   ** tclcmd:  test_sqlite3_log ?SCRIPT?
  5924   5960   */
  5925   5961   static struct LogCallback {
  5926   5962     Tcl_Interp *pInterp;
  5927   5963     Tcl_Obj *pObj;
................................................................................
  6973   7009        { "sqlite3_test_errstr",     test_errstr, 0             },
  6974   7010        { "tcl_variable_type",       tcl_variable_type, 0       },
  6975   7011   #ifndef SQLITE_OMIT_SHARED_CACHE
  6976   7012        { "sqlite3_enable_shared_cache", test_enable_shared, 0  },
  6977   7013        { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0},
  6978   7014   #endif
  6979   7015        { "sqlite3_libversion_number", test_libversion_number, 0  },
  6980         -#ifdef SQLITE_ENABLE_COLUMN_METADATA
  6981   7016        { "sqlite3_table_column_metadata", test_table_column_metadata, 0  },
  6982         -#endif
  6983   7017   #ifndef SQLITE_OMIT_INCRBLOB
  6984   7018        { "sqlite3_blob_reopen", test_blob_reopen, 0  },
  6985   7019   #endif
  6986   7020        { "pcache_stats",       test_pcache_stats, 0  },
  6987   7021   #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
  6988   7022        { "sqlite3_unlock_notify", test_unlock_notify, 0  },
  6989   7023   #endif
  6990   7024        { "sqlite3_wal_checkpoint",   test_wal_checkpoint, 0  },
  6991   7025        { "sqlite3_wal_checkpoint_v2",test_wal_checkpoint_v2, 0  },
         7026  +     { "sqlite3_wal_autocheckpoint",test_wal_autocheckpoint, 0  },
  6992   7027        { "test_sqlite3_log",         test_sqlite3_log, 0  },
  6993   7028   #ifndef SQLITE_OMIT_EXPLAIN
  6994   7029        { "print_explain_query_plan", test_print_eqp, 0  },
  6995   7030   #endif
  6996   7031        { "sqlite3_test_control", test_test_control },
  6997   7032   #if SQLITE_OS_UNIX
  6998   7033        { "getrusage", test_getrusage },

Changes to src/vdbeapi.c.

   412    412   static int doWalCallbacks(sqlite3 *db){
   413    413     int rc = SQLITE_OK;
   414    414   #ifndef SQLITE_OMIT_WAL
   415    415     int i;
   416    416     for(i=0; i<db->nDb; i++){
   417    417       Btree *pBt = db->aDb[i].pBt;
   418    418       if( pBt ){
   419         -      int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt));
          419  +      int nEntry;
          420  +      sqlite3BtreeEnter(pBt);
          421  +      nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt));
          422  +      sqlite3BtreeLeave(pBt);
   420    423         if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){
   421    424           rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry);
   422    425         }
   423    426       }
   424    427     }
   425    428   #endif
   426    429     return rc;
................................................................................
   592    595       ** into the database handle. This block copies the error message 
   593    596       ** from the database handle into the statement and sets the statement
   594    597       ** program counter to 0 to ensure that when the statement is 
   595    598       ** finalized or reset the parser error message is available via
   596    599       ** sqlite3_errmsg() and sqlite3_errcode().
   597    600       */
   598    601       const char *zErr = (const char *)sqlite3_value_text(db->pErr); 
   599         -    assert( zErr!=0 || db->mallocFailed );
   600    602       sqlite3DbFree(db, v->zErrMsg);
   601    603       if( !db->mallocFailed ){
   602    604         v->zErrMsg = sqlite3DbStrDup(db, zErr);
   603    605         v->rc = rc2;
   604    606       } else {
   605    607         v->zErrMsg = 0;
   606    608         v->rc = rc = SQLITE_NOMEM;

Changes to src/vdbesort.c.

   447    447   **   void *SRVAL(SorterRecord *p) { return (void*)&p[1]; }
   448    448   */
   449    449   #define SRVAL(p) ((void*)((SorterRecord*)(p) + 1))
   450    450   
   451    451   /* The minimum PMA size is set to this value multiplied by the database
   452    452   ** page size in bytes.  */
   453    453   #ifndef SQLITE_SORTER_PMASZ
   454         -# define SQLITE_SORTER_PMASZ 250
          454  +# define SQLITE_SORTER_PMASZ 10
   455    455   #endif
   456    456   
   457    457   /* Maximum number of PMAs that a single MergeEngine can merge */
   458    458   #define SORTER_MAX_MERGE_COUNT 16
   459    459   
   460    460   static int vdbeIncrSwap(IncrMerger*);
   461    461   static void vdbeIncrFree(IncrMerger *);

Changes to src/where.c.

  3937   3937     if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
  3938   3938       if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
  3939   3939         sqlite3_free(p->u.vtab.idxStr);
  3940   3940         p->u.vtab.needFree = 0;
  3941   3941         p->u.vtab.idxStr = 0;
  3942   3942       }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
  3943   3943         sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
  3944         -      sqlite3KeyInfoUnref(p->u.btree.pIndex->pKeyInfo);
  3945   3944         sqlite3DbFree(db, p->u.btree.pIndex);
  3946   3945         p->u.btree.pIndex = 0;
  3947   3946       }
  3948   3947     }
  3949   3948   }
  3950   3949   
  3951   3950   /*

Changes to test/colmeta.test.

    13     13   # focus of this script is the sqlite3_table_column_metadata() API.
    14     14   #
    15     15   # $Id: colmeta.test,v 1.4 2008/01/23 12:52:41 drh Exp $
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   
    20         -ifcapable !columnmetadata {
    21         -  finish_test
    22         -  return
    23         -}
    24         -
    25     20   # Set up a schema in the main and temp test databases.
    26     21   do_test colmeta-0 {
    27     22     execsql {
    28     23       CREATE TABLE abc(a, b, c);
    29     24       CREATE TABLE abc2(a PRIMARY KEY COLLATE NOCASE, b VARCHAR(32), c);
    30     25       CREATE TABLE abc3(a NOT NULL, b INTEGER PRIMARY KEY, c);
           26  +    CREATE TABLE abc5(w,x,y,z,PRIMARY KEY(x,z)) WITHOUT ROWID;
           27  +    CREATE TABLE abc6(rowid TEXT COLLATE rtrim, oid REAL, _rowid_ BLOB);
    31     28     }
    32     29     ifcapable autoinc {
    33     30       execsql {
    34     31         CREATE TABLE abc4(a, b INTEGER PRIMARY KEY AUTOINCREMENT, c);
    35     32       }
    36     33     }
    37     34     ifcapable view {
................................................................................
    53     50     4  {main abc2 b}               {0 {VARCHAR(32) BINARY 0 0 0}}
    54     51     5  {{} abc2 a}                 {0 {{} NOCASE 0 1 0}}
    55     52     6  {{} abc3 a}                 {0 {{} BINARY 1 0 0}}
    56     53     7  {{} abc3 b}                 {0 {INTEGER BINARY 0 1 0}}
    57     54     13 {main abc rowid}            {0 {INTEGER BINARY 0 1 0}}
    58     55     14 {main abc3 rowid}           {0 {INTEGER BINARY 0 1 0}}
    59     56     16 {main abc d}                {1 {no such table column: abc.d}}
           57  +  20 {main abc5 w}               {0 {{} BINARY 0 0 0}}
           58  +  21 {main abc5 x}               {0 {{} BINARY 1 1 0}}
           59  +  22 {main abc5 y}               {0 {{} BINARY 0 0 0}}
           60  +  23 {main abc5 z}               {0 {{} BINARY 1 1 0}}
           61  +  24 {main abc5 rowid}           {1 {no such table column: abc5.rowid}}
           62  +  30 {main abc6 rowid}           {0 {TEXT rtrim 0 0 0}}
           63  +  31 {main abc6 oid}             {0 {REAL BINARY 0 0 0}}
           64  +  32 {main abc6 _rowid_}         {0 {BLOB BINARY 0 0 0}}
    60     65   }
    61         -ifcapable view {
           66  +ifcapable autoinc {
    62     67     set tests [concat $tests {
    63         -    8  {{} abc4 b}                 {0 {INTEGER BINARY 0 1 1}}
    64         -    15 {main abc4 rowid}           {0 {INTEGER BINARY 0 1 1}}
           68  +    100 {{} abc4 b}              {0 {INTEGER BINARY 0 1 1}}
           69  +    101 {main abc4 rowid}        {0 {INTEGER BINARY 0 1 1}}
    65     70     }]
    66     71   }
    67     72   ifcapable view {
    68     73     set tests [concat $tests {
    69         -    9  {{} v1 a}                   {1 {no such table column: v1.a}}
    70         -    10 {main v1 b}                 {1 {no such table column: v1.b}}
    71         -    11 {main v1 badname}           {1 {no such table column: v1.badname}}
    72         -    12 {main v1 rowid}             {1 {no such table column: v1.rowid}}
           74  +    200 {{} v1 a}                {1 {no such table column: v1.a}}
           75  +    201 {main v1 b}              {1 {no such table column: v1.b}}
           76  +    202 {main v1 badname}        {1 {no such table column: v1.badname}}
           77  +    203 {main v1 rowid}          {1 {no such table column: v1.rowid}}
    73     78     }]
    74     79   }
    75     80   
    76     81   foreach {tn params results} $tests {
    77     82     set ::DB [sqlite3_connection_pointer db]
    78     83   
    79     84     set tstbody [concat sqlite3_table_column_metadata $::DB $params] 
................................................................................
    86     91   
    87     92     set ::DB [sqlite3_connection_pointer db]
    88     93     set tstbody [concat sqlite3_table_column_metadata $::DB $params] 
    89     94     do_test colmeta-$tn.2 {
    90     95       list [catch $tstbody msg] [set msg]
    91     96     } $results
    92     97   }
           98  +
           99  +# Calling sqlite3_table_column_metadata with a NULL column name merely
          100  +# checks for the existance of the table.
          101  +#
          102  +do_test colmeta-300 {
          103  +  catch {sqlite3_table_column_metadata $::DB main xyzzy} res
          104  +} {1}
          105  +do_test colmeta-301 {
          106  +  catch {sqlite3_table_column_metadata $::DB main abc} res
          107  +} {0}
          108  +
    93    109   
    94    110   finish_test

Added test/e_walauto.test.

            1  +# 2014 December 04
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +source $testdir/wal_common.tcl
           16  +set testprefix e_walauto
           17  +
           18  +
           19  +proc read_nbackfill {} {
           20  +  seek $::shmfd 96
           21  +  binary scan [read $::shmfd 4] i nBackfill
           22  +  set nBackfill
           23  +}
           24  +proc read_mxframe {} {
           25  +  seek $::shmfd 16
           26  +  binary scan [read $::shmfd 4] i mxFrame
           27  +  set mxFrame
           28  +}
           29  +
           30  +# Assuming that the main db for database handle
           31  +#
           32  +proc do_autocommit_threshold_test {tn value} {
           33  +
           34  +  set nBackfillSaved [read_nbackfill]
           35  +  while {1} {
           36  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
           37  +    if {[read_mxframe] >= $value} break
           38  +  }
           39  +  
           40  +  set nBackfillNew [read_nbackfill]
           41  +  uplevel [list do_test $tn "expr $nBackfillNew > $nBackfillSaved" 1]
           42  +}
           43  +
           44  +# EVIDENCE-OF: R-30135-06439 The wal_autocheckpoint pragma can be used
           45  +# to invoke this interface from SQL.
           46  +#
           47  +#   All tests in this file are run twice - once using the
           48  +#   sqlite3_wal_autocheckpoint() API, and once using "PRAGMA
           49  +#   wal_autocheckpoint".
           50  +#
           51  +foreach {tn code} {
           52  +  1 {
           53  +    proc autocheckpoint {db value} {
           54  +      uplevel [list $db eval "PRAGMA wal_autocheckpoint = $value"]
           55  +    }
           56  +  }
           57  +
           58  +  2 {
           59  +    proc autocheckpoint {db value} {
           60  +      uplevel [list sqlite3_wal_autocheckpoint $db $value]
           61  +      return $value
           62  +    }
           63  +  }
           64  +} {
           65  +
           66  +  eval $code
           67  +
           68  +  reset_db
           69  +  do_execsql_test 1.$tn.0 { PRAGMA journal_mode = WAL } {wal}
           70  +  do_execsql_test 1.$tn.1 { CREATE TABLE t1(a, b) }
           71  +  set shmfd [open "test.db-shm"]
           72  +
           73  +  # EVIDENCE-OF: R-41531-51083 Every new database connection defaults to
           74  +  # having the auto-checkpoint enabled with a threshold of 1000 or
           75  +  # SQLITE_DEFAULT_WAL_AUTOCHECKPOINT pages.
           76  +  #
           77  +  do_autocommit_threshold_test 1.$tn.2 1000
           78  +  db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
           79  +  do_autocommit_threshold_test 1.$tn.3 1000
           80  +
           81  +  # EVIDENCE-OF: R-38128-34102 The sqlite3_wal_autocheckpoint(D,N) is a
           82  +  # wrapper around sqlite3_wal_hook() that causes any database on database
           83  +  # connection D to automatically checkpoint after committing a
           84  +  # transaction if there are N or more frames in the write-ahead log file.
           85  +  #
           86  +  do_test 1.$tn.4 {
           87  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
           88  +    autocheckpoint db 100
           89  +  } {100}
           90  +  do_autocommit_threshold_test 1.$tn.5 100
           91  +
           92  +  do_test 1.$tn.6 {
           93  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
           94  +    autocheckpoint db 500
           95  +  } {500}
           96  +  do_autocommit_threshold_test 1.$tn.7 500
           97  +
           98  +  # EVIDENCE-OF: R-26993-43540 Passing zero or a negative value as the
           99  +  # nFrame parameter disables automatic checkpoints entirely.
          100  +  #
          101  +  do_test 1.$tn.7 {
          102  +    autocheckpoint db 0    ;# Set to zero
          103  +    for {set i 0} {$i < 10000} {incr i} {
          104  +      db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          105  +    }
          106  +    expr {[file size test.db-wal] > (5 * 1024 * 1024)}
          107  +  } 1
          108  +  do_test 1.$tn.8 {
          109  +    sqlite3_wal_checkpoint_v2 db truncate
          110  +    file size test.db-wal
          111  +  } 0
          112  +  do_test 1.$tn.9 {
          113  +    autocheckpoint db -4    ;# Set to a negative value
          114  +    for {set i 0} {$i < 10000} {incr i} {
          115  +      db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          116  +    }
          117  +    expr {[file size test.db-wal] > (5 * 1024 * 1024)}
          118  +  } 1
          119  +
          120  +  # EVIDENCE-OF: R-10203-42688 The callback registered by this function
          121  +  # replaces any existing callback registered using sqlite3_wal_hook().
          122  +  #
          123  +  set ::wal_hook_callback 0
          124  +  proc wal_hook_callback {args} { incr ::wal_hook_callback ; return 0 }
          125  +  do_test 1.$tn.10.1 {
          126  +    db wal_hook wal_hook_callback
          127  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          128  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          129  +    set ::wal_hook_callback
          130  +  } 2
          131  +  do_test 1.$tn.10.2 {
          132  +    autocheckpoint db 100
          133  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          134  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          135  +    set ::wal_hook_callback
          136  +  } 2
          137  +
          138  +  # EVIDENCE-OF: R-17497-43474 Likewise, registering a callback using
          139  +  # sqlite3_wal_hook() disables the automatic checkpoint mechanism
          140  +  # configured by this function.
          141  +  do_test 1.$tn.11.1 {
          142  +    sqlite3_wal_checkpoint_v2 db truncate
          143  +    file size test.db-wal
          144  +  } 0
          145  +  do_test 1.$tn.11.2 {
          146  +    autocheckpoint db 100 
          147  +    for {set i 0} {$i < 1000} {incr i} {
          148  +      db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          149  +    }
          150  +    expr {[file size test.db-wal] < (1 * 1024 * 1024)}
          151  +  } 1
          152  +  do_test 1.$tn.11.3 {
          153  +    db wal_hook wal_hook_callback
          154  +    for {set i 0} {$i < 1000} {incr i} {
          155  +      db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          156  +    }
          157  +    expr {[file size test.db-wal] < (1 * 1024 * 1024)}
          158  +  } 0
          159  +
          160  +  # EVIDENCE-OF: R-33080-59193 Checkpoints initiated by this mechanism 
          161  +  # are PASSIVE.
          162  +  #
          163  +  set ::busy_callback_count 0
          164  +  proc busy_callback {args} {
          165  +  puts Hello
          166  +    incr ::busy_callback_count
          167  +    return 0
          168  +  }
          169  +  do_test 1.$tn.12.1 {
          170  +    sqlite3_wal_checkpoint_v2 db truncate
          171  +    autocheckpoint db 100 
          172  +    db busy busy_callback
          173  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          174  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          175  +  } {}
          176  +  do_test 1.$tn.12.2 {
          177  +    sqlite3 db2 test.db
          178  +    db2 eval { BEGIN; SELECT * FROM t1 LIMIT 10; }
          179  +    read_nbackfill
          180  +  } {0}
          181  +  do_test 1.$tn.12.3 {
          182  +    for {set i 0} {$i < 1000} {incr i} {
          183  +      db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          184  +    }
          185  +    read_nbackfill
          186  +  } {2}
          187  +  do_test 1.$tn.12.4 {
          188  +    set ::busy_callback_count
          189  +  } {0}
          190  +  db2 close
          191  +
          192  +  do_test 1.$tn.12.5 {
          193  +    db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) }
          194  +    read_nbackfill
          195  +  } {1559}
          196  +
          197  +  db close
          198  +  close $shmfd
          199  +}
          200  +
          201  +finish_test

Changes to test/e_walckpt.test.

    38     38       set expect 0
    39     39       catch { set expect [md5file $f] }
    40     40       if {$H($f) != $expect} { lappend ret $f }
    41     41     }
    42     42     set ret
    43     43   }
    44     44   
           45  +#-------------------------------------------------------------------------
           46  +# All calls to the [sqlite3_wal_checkpoint_v2] command made within this
           47  +# file use this wrapper. It's sole purpose is to throw an error if the
           48  +# following requirement is violated:
           49  +#
           50  +# EVIDENCE-OF: R-60567-47780 Unless it returns SQLITE_MISUSE, the
           51  +# sqlite3_wal_checkpoint_v2() interface sets the error information that
           52  +# is queried by sqlite3_errcode() and sqlite3_errmsg().
           53  +#
           54  +proc wal_checkpoint_v2 {db args} {
           55  +  set rc [catch {
           56  +    uplevel sqlite3_wal_checkpoint_v2 $db $args
           57  +  } msg]
           58  +
           59  +  set errcode "SQLITE_OK"
           60  +  if {$rc} {
           61  +    set errcode [lindex [split $msg " "] 0]
           62  +  } elseif { [lindex $msg 0] } {
           63  +    set errcode "SQLITE_BUSY"
           64  +  }
           65  +
           66  +  if {$errcode != "SQLITE_MISUSE" && [sqlite3_errcode $db] != $errcode} {
           67  +    error "sqlite3_errcode mismatch! (1) $errcode!=[sqlite3_errcode $db]"
           68  +  }
           69  +
           70  +  if {$rc==0} {
           71  +    return $msg
           72  +  } else {
           73  +    error $msg
           74  +  }
           75  +}
           76  +
    45     77   
    46     78   # The following tests are run 3 times, each using a different method of 
    47     79   # invoking a checkpoint:
    48     80   #
    49     81   #   1) Using sqlite3_wal_checkpoint_v2()
    50     82   #   2) Using "PRAGMA wal_checkpoint"
    51     83   #   3) Using sqlite3_wal_checkpoint() in place of checkpoint_v2(PASSIVE)
................................................................................
    59     91   # EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is
    60     92   # equivalent to
    61     93   # sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0).
    62     94   # 
    63     95   foreach {tn script} {
    64     96     1 {
    65     97       proc checkpoint {db mode args} {
    66         -      eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args
           98  +      eval wal_checkpoint_v2 [list $db] [list $mode] $args
    67     99       }
    68    100     }
    69    101   
    70    102     2 {
    71    103       proc checkpoint {db mode args} {
    72    104         set sql "PRAGMA wal_checkpoint = $mode"
    73    105         if {[llength $args] && [lindex $args 0]!=""} {
................................................................................
    86    118       proc checkpoint {db mode args} {
    87    119         if {$mode == "passive"} {
    88    120           set rc [eval sqlite3_wal_checkpoint [list $db] $args]
    89    121           if {$rc != "SQLITE_OK"} {
    90    122             error "$rc - [sqlite3_errmsg $db]"
    91    123           }
    92    124         } else {
    93         -        eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args
          125  +        eval wal_checkpoint_v2 [list $db] [list $mode] $args
    94    126         }
    95    127       }
    96    128     }
    97    129   
    98    130   } {
    99    131   
   100    132     eval $script
................................................................................
   266    298       switch -- $busy_handler_mode {
   267    299         1 {
   268    300           # Do nothing. Do not block.
   269    301           return 1
   270    302         }
   271    303   
   272    304         2 {
   273         -        # Close first the reader, then later the writer.
          305  +        # Close first the reader, then later the writer. Give up before
          306  +        # closing the [db6] reader.
   274    307           if {$n==5}  { catch {db2 eval commit} }
   275    308           if {$n==10} { catch {db3 eval commit} }
          309  +        if {$n==15} { return 1 }
   276    310           return 0
   277    311         }
   278    312   
   279    313         3 {
   280         -        # Close first the writer, then later the reader.
          314  +        # Close first the writer, then later the reader. And finally the 
          315  +        # [db6] reader.
   281    316           if {$n==5}  { catch {db2 eval commit} }
   282    317           if {$n==10} { catch {db3 eval commit} }
          318  +        if {$n==15} { catch {db6 eval commit} }
   283    319           return 0
   284    320         }
   285    321       }
   286    322     }
   287    323   
   288    324     foreach {mode busy_handler_mode} { 
   289         -    passive 1
   290         -    full    1
   291         -    full    2
   292         -    full    3
          325  +    passive  1
          326  +    full     1       full     2       full    3
          327  +    restart  1       restart  2       restart  3
          328  +    truncate 1       truncate 2       truncate 3
   293    329     } {
          330  +    set tp "$tn.$mode.$busy_handler_mode"
   294    331   
   295    332       set ::sync_counter 0
   296    333   
          334  +    # Set up a callback function for xSync and xWrite calls made during
          335  +    # the checkpoint.
          336  +    #
          337  +    set ::checkpoint_ongoing 0
   297    338       proc tvfs_callback {method args} {
          339  +      if {$::checkpoint_ongoing==0} return
          340  +
   298    341         set tail [file tail [lindex $args 0]]
   299    342         if {$method == "xSync" && $tail == "test.db"} {
   300    343           incr ::sync_counter
   301    344         }
   302         -
   303    345         if {$method == "xWrite" && $tail=="test.db"} {
   304    346           if {$::write_ok < 0} {
   305    347             set ::write_ok [expr ![catch {db5 eval { BEGIN IMMEDIATE }}]]
   306    348             catch { db5 eval ROLLBACK }
   307    349           }
   308    350           if {$::read_ok < 0} {
   309    351             set ::read_ok [expr ![catch {db5 eval { SELECT * FROM t1 }}]]
   310    352           }
          353  +
          354  +        # If one has not already been opened, open a read-transaction using
          355  +        # connection [db6]
          356  +        catch { db6 eval { BEGIN ; SELECT * FROM sqlite_master } } msg
          357  +      }
          358  +      if {$method == "xShmLock" } {
          359  +        set details [lindex $args 2]
          360  +        if {$details == "0 1 lock exclusive"} { set ::seen_writer_lock 1 }
   311    361         }
   312    362       }
   313    363   
   314    364       catch { db close }
   315    365       forcedelete test.db
   316    366       testvfs tvfs
   317    367       sqlite3 db test.db -vfs tvfs
   318    368       #tvfs filter xSync
   319    369       tvfs script tvfs_callback
   320    370   
   321         -    do_execsql_test $tn.4.$mode.0 {
          371  +    do_execsql_test $tp.0 {
   322    372         CREATE TABLE t1(a, b);
   323    373         CREATE TABLE t2(a, b);
   324    374         PRAGMA journal_mode = wal;
   325    375         INSERT INTO t1 VALUES(1, 2);
   326    376         INSERT INTO t1 VALUES(3, 4);
   327    377         INSERT INTO t1 VALUES(5, 6);
   328    378       } {wal}
   329    379   
   330    380       # Open a reader on the current database snapshot.
   331         -    do_test $tn.4.$mode.1 {
          381  +    do_test $tp.1 {
   332    382         sqlite3 db2 test.db -vfs tvfs
   333    383         execsql {
   334    384           BEGIN;
   335    385             SELECT * FROM t1 UNION ALL SELECT * FROM t2;
   336    386         } db2
   337    387       } {1 2 3 4 5 6}
   338    388   
   339    389       # Open a writer. Write a transaction. Then begin, but do not commit,
   340    390       # a second transaction.
   341         -    do_test $tn.4.$mode.2 {
          391  +    do_test $tp.2 {
   342    392         sqlite3 db3 test.db -vfs tvfs
   343    393         execsql {
   344    394           INSERT INTO t2 VALUES(7, 8);
   345    395           BEGIN;
   346    396             INSERT INTO t2 VALUES(9, 10);
   347    397             SELECT * FROM t1 UNION ALL SELECT * FROM t2;
   348    398         } db3
   349    399       } {1 2 3 4 5 6 7 8 9 10}
   350    400   
   351    401       sqlite3 db5 test.db -vfs tvfs
          402  +    sqlite3 db6 test.db -vfs tvfs
   352    403   
   353    404       # Register a busy-handler with connection [db].
   354    405       #
   355    406       db busy [list busy_handler $mode $busy_handler_mode]
   356    407       set ::sync_counter 0
   357    408       set ::busy_handler_counter 0
   358    409       set ::read_ok -1
   359    410       set ::write_ok -1
          411  +    set ::seen_writer_lock 0
   360    412       
   361         -    do_test $tn.4.$mode.3 {
          413  +    set ::checkpoint_ongoing 1
          414  +    do_test $tp.3 {
   362    415         checkpoint db $mode main
   363    416         set {} {}
   364    417       } {}
          418  +    set ::checkpoint_ongoing 0
          419  +    set ::did_restart_blocking [expr {[catch {db6 eval commit}]}]
   365    420   
   366    421       if { $mode=="passive" } {
   367    422         # EVIDENCE-OF: R-16333-64433 Checkpoint as many frames as possible
   368    423         # without waiting for any database readers or writers to finish, then
   369    424         # sync the database file if all frames in the log were checkpointed.
   370    425         #
   371    426         #   "As many frames as possible" means all but the last two transactions
................................................................................
   377    432         #   writer are still active - so the checkpointer did not wait for either
   378    433         #   readers or writers. As a result the checkpoint was not finished and
   379    434         #   so the db file is not synced.
   380    435         #
   381    436         # EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
   382    437         # in the SQLITE_CHECKPOINT_PASSIVE mode.
   383    438         #
   384         -      #   It's not. Test case "$tn.4.$mode.6".
          439  +      #   It's not. Test case "$tp.6".
   385    440         #
   386         -      do_test $tn.4.$mode.4 {
          441  +      do_test $tp.4 {
   387    442           forcecopy test.db abc.db
   388    443           sqlite3 db4 abc.db
   389    444           db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 }
   390    445         } {1 2 3 4 5 6}
   391         -      do_test $tn.4.$mode.5 { set ::sync_counter } 0
   392         -      do_test $tn.4.$mode.6 { set ::busy_handler_counter } 0
          446  +      do_test $tp.5 { set ::sync_counter } 0
          447  +      do_test $tp.6 { set ::busy_handler_counter } 0
   393    448         db4 close
   394    449     
   395    450         db2 eval COMMIT
   396    451         db3 eval COMMIT
   397    452     
   398    453         # EVIDENCE-OF: R-65499-53765 On the other hand, passive mode might leave
   399    454         # the checkpoint unfinished if there are concurrent readers or writers.
................................................................................
   401    456         #   The reader and writer have now dropped their locks. And so a 
   402    457         #   checkpoint now is able to checkpoint more frames. Showing that the
   403    458         #   attempt above was left "unfinished".
   404    459         #
   405    460         #   Also, because the checkpoint finishes this time, the db is synced.
   406    461         #   Which is part of R-16333-64433 above.
   407    462         #
   408         -      do_test $tn.4.$mode.7 {
          463  +      set ::checkpoint_ongoing 1
          464  +      do_test $tp.7 {
   409    465           checkpoint db $mode main
   410    466           forcecopy test.db abc.db
   411    467           sqlite3 db4 abc.db
   412    468           db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 }
   413    469         } {1 2 3 4 5 6 7 8 9 10}
   414         -      do_test $tn.4.$mode.6 { set ::sync_counter } 1
   415         -      do_test $tn.4.$mode.7 { set ::busy_handler_counter } 0
          470  +      set ::checkpoint_ongoing 0
          471  +      do_test $tp.7 { set ::sync_counter } 1
          472  +      do_test $tp.8 { set ::busy_handler_counter } 0
   416    473         db4 close
   417    474       }
   418    475   
   419         -    if { $mode=="full" } {
          476  +    if { $mode=="full" || $mode=="restart" || $mode=="truncate" } {
          477  +
          478  +      # EVIDENCE-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and
          479  +      # TRUNCATE modes also obtain the exclusive "writer" lock on the 
          480  +      # database file.
          481  +      #
          482  +      #   Or at least attempts to obtain.
          483  +      #
          484  +      do_test $tp.9 {
          485  +        set ::seen_writer_lock
          486  +      } {1}
          487  +
   420    488         if {$busy_handler_mode==2 || $busy_handler_mode==3} {
   421    489           # EVIDENCE-OF: R-59171-47567 This mode blocks (it invokes the
   422    490           # busy-handler callback) until there is no database writer and all
   423    491           # readers are reading from the most recent database snapshot.
   424    492           #
   425         -        #   Show that both the reader and writer have finished:
          493  +        #   The test below shows that both the reader and writer have 
          494  +        #   finished:
          495  +        #
          496  +        #   Also restated by the following two. That both busy_handler_mode
          497  +        #   values 2 and 3 work show that both of the following are true - as
          498  +        #   they release the reader and writer transactions in different
          499  +        #   orders.
          500  +        #
          501  +        # EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained
          502  +        # immediately, and a busy-handler is configured, it is invoked and the
          503  +        # writer lock retried until either the busy-handler returns 0 or the
          504  +        # lock is successfully obtained.
          505  +        #
          506  +        # EVIDENCE-OF: R-48107-00250 The busy-handler is also invoked while
          507  +        # waiting for database readers as described above.
   426    508           #
   427         -        do_test $tn.4.$mode.7 {
          509  +        do_test $tp.7 {
   428    510             list [catchsql COMMIT db2] [catchsql COMMIT db3]
   429    511           } [list                                             \
   430    512               {1 {cannot commit - no transaction is active}}  \
   431    513               {1 {cannot commit - no transaction is active}}  \
   432    514           ]
   433    515   
   434    516           # EVIDENCE-OF: R-29177-48281 It then checkpoints all frames in the log
   435    517           # file and syncs the database file.
   436    518           #
   437         -        do_test $tn.4.$mode.8 {
          519  +        do_test $tp.8 {
   438    520             forcecopy test.db abc.db
   439    521             sqlite3 db4 abc.db
   440    522             db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 }
   441    523           } {1 2 3 4 5 6 7 8 9 10}
   442         -        do_test $tn.4.$mode.9 { set ::sync_counter } 1
          524  +        do_test $tp.9 { set ::sync_counter } 1
   443    525           db4 close
   444    526   
   445    527           # EVIDENCE-OF: R-51867-44713 This mode blocks new database writers
   446    528           # while it is pending, but new database readers are allowed to continue
   447    529           # unimpeded.
   448         -        do_test $tn.4.$mode.10 {
          530  +        #
          531  +        # EVIDENCE-OF: R-47276-58266 Like SQLITE_CHECKPOINT_FULL, this mode
          532  +        # blocks new database writer attempts while it is pending, but does not
          533  +        # impede readers.
          534  +        #
          535  +        #   The first of the above two refers to "full" mode. The second
          536  +        #   to "restart".
          537  +        #
          538  +        do_test $tp.10.1 {
   449    539             list $::write_ok $::read_ok
   450    540           } {0 1}
   451    541   
          542  +        # EVIDENCE-OF: R-12410-31217 This mode works the same way as
          543  +        # SQLITE_CHECKPOINT_FULL with the addition that after checkpointing the
          544  +        # log file it blocks (calls the busy-handler callback) until all
          545  +        # readers are reading from the database file only.
          546  +        #
          547  +        #     The stuff above passed, so the first part of this requirement
          548  +        #     is met. The second part is tested below. If the checkpoint mode
          549  +        #     was "restart" or "truncate", then the busy-handler will have
          550  +        #     been called to block on wal-file readers.
          551  +        #
          552  +        do_test $tp.11 {
          553  +          set ::did_restart_blocking
          554  +        } [expr {($mode=="restart"||$mode=="truncate")&&$busy_handler_mode==3}]
          555  +
          556  +        # EVIDENCE-OF: R-44699-57140 This mode works the same way as
          557  +        # SQLITE_CHECKPOINT_RESTART with the addition that it also truncates
          558  +        # the log file to zero bytes just prior to a successful return.
          559  +        if {$mode=="truncate" && $busy_handler_mode==3} {
          560  +          do_test $tp.12 {
          561  +            file size test.db-wal
          562  +          } 0
          563  +        }
          564  +      } elseif {$busy_handler_mode==1} {
          565  +
          566  +        # EVIDENCE-OF: R-34519-06271 SQLITE_BUSY is returned in this case.
          567  +        if {$tn!=2} {
          568  +          # ($tn==2) is the loop that uses "PRAGMA wal_checkpoint"
          569  +          do_test $tp.13 { sqlite3_errcode db } {SQLITE_BUSY}
          570  +        }
          571  +
          572  +        # EVIDENCE-OF: R-49155-63541 If the busy-handler returns 0 before the
          573  +        # writer lock is obtained or while waiting for database readers, the
          574  +        # checkpoint operation proceeds from that point in the same way as
          575  +        # SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible
          576  +        # without blocking any further.
          577  +        do_test $tp.14 {
          578  +          forcecopy test.db abc.db
          579  +            sqlite3 db4 abc.db
          580  +            db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 }
          581  +        } {1 2 3 4 5 6}
          582  +        do_test $tp.15 { set ::sync_counter } 0
          583  +        do_test $tp.16 { set ::busy_handler_counter } 1
          584  +        db4 close
   452    585         }
   453    586       }
   454    587   
   455    588       db2 close
   456    589       db3 close
   457    590       db5 close
          591  +    db6 close
   458    592     }
   459    593   
   460    594     db close
   461    595     tvfs delete
   462    596   }
   463    597   
   464    598   #-----------------------------------------------------------------------
................................................................................
   476    610     4  2       {0 {0 -1 -1}}
   477    611     5  3       {0 {0 -1 -1}}
   478    612     6  4       {1 {SQLITE_MISUSE - not an error}}
   479    613     7  114     {1 {SQLITE_MISUSE - not an error}}
   480    614     8  1000000 {1 {SQLITE_MISUSE - not an error}}
   481    615   } {
   482    616     do_test 4.$tn {
   483         -    list [catch "sqlite3_wal_checkpoint_v2 db $mode" msg] $msg
          617  +    list [catch "wal_checkpoint_v2 db $mode" msg] $msg
   484    618     } $res
   485    619   }
          620  +db close
          621  +
          622  +foreach tn {1 2 3} {
          623  +  forcedelete test.db test.db2 test.db3
          624  +  testvfs tvfs
          625  +
          626  +  sqlite3 db test.db -vfs tvfs
          627  +  execsql {
          628  +    ATTACH 'test.db2' AS aux2;
          629  +    ATTACH 'test.db3' AS aux3;
          630  +    PRAGMA main.journal_mode = WAL;
          631  +    PRAGMA aux2.journal_mode = WAL;
          632  +    PRAGMA aux3.journal_mode = WAL;
          633  +
          634  +    CREATE TABLE main.t1(x,y);
          635  +    CREATE TABLE aux2.t2(x,y);
          636  +    CREATE TABLE aux3.t3(x,y);
          637  +
          638  +    INSERT INTO t1 VALUES('a', 'b');
          639  +    INSERT INTO t2 VALUES('a', 'b');
          640  +    INSERT INTO t3 VALUES('a', 'b');
          641  +  }
          642  +  sqlite3 db2 test.db2 -vfs tvfs
          643  +
          644  +  switch -- $tn {
          645  +    1 {
          646  +      # EVIDENCE-OF: R-41299-52117 If no error (SQLITE_BUSY or otherwise) is
          647  +      # encountered while processing the attached databases, SQLITE_OK is
          648  +      # returned.
          649  +      do_test 5.$tn.1 {
          650  +        lindex [wal_checkpoint_v2 db truncate] 0
          651  +      } {0}    ;# 0 -> SQLITE_OK
          652  +      do_test 5.$tn.2 {
          653  +        list [expr [file size test.db-wal]==0]  \
          654  +             [expr [file size test.db2-wal]==0] \
          655  +             [expr [file size test.db3-wal]==0]
          656  +      } {1 1 1}
          657  +    }
          658  +
          659  +    2 {
          660  +      # EVIDENCE-OF: R-38578-34175 If an SQLITE_BUSY error is encountered when
          661  +      # processing one or more of the attached WAL databases, the operation is
          662  +      # still attempted on any remaining attached databases and SQLITE_BUSY is
          663  +      # returned at the end.
          664  +      db2 eval { BEGIN; INSERT INTO t2 VALUES('d', 'e'); }
          665  +      do_test 5.$tn.1 {
          666  +        lindex [wal_checkpoint_v2 db truncate] 0
          667  +      } {1}    ;# 1 -> SQLITE_BUSY
          668  +      do_test 5.$tn.2 {
          669  +        list [expr [file size test.db-wal]==0]  \
          670  +             [expr [file size test.db2-wal]==0] \
          671  +             [expr [file size test.db3-wal]==0]
          672  +      } {1 0 1}
          673  +      db2 eval ROLLBACK
          674  +    }
          675  +
          676  +    3 {
          677  +      # EVIDENCE-OF: R-38049-07913 If any other error occurs while processing
          678  +      # an attached database, processing is abandoned and the error code is
          679  +      # returned to the caller immediately.
          680  +      tvfs filter xWrite
          681  +      tvfs script inject_ioerr
          682  +      proc inject_ioerr {method file args} {
          683  +        if {[file tail $file]=="test.db2"} {
          684  +          return "SQLITE_IOERR"
          685  +        }
          686  +        return 0
          687  +      }
          688  +      do_test 5.$tn.1 {
          689  +        list [catch { wal_checkpoint_v2 db truncate } msg] $msg
          690  +      } {1 {SQLITE_IOERR - disk I/O error}}
          691  +      do_test 5.$tn.2 {
          692  +        list [expr [file size test.db-wal]==0]  \
          693  +             [expr [file size test.db2-wal]==0] \
          694  +             [expr [file size test.db3-wal]==0]
          695  +      } {1 0 0}
          696  +      tvfs script ""
          697  +    }
          698  +  }
          699  +
          700  +  db close
          701  +  db2 close
          702  +}
          703  +
          704  +reset_db
          705  +sqlite3 db2 test.db
          706  +
          707  +do_test 6.1 {
          708  +  execsql {
          709  +    PRAGMA journal_mode = WAL;
          710  +    CREATE TABLE t1(a, b);
          711  +    INSERT INTO t1 VALUES(1, 2);
          712  +  }
          713  +  file size test.db-wal
          714  +} [wal_file_size 3 1024]
          715  +
          716  +do_test 6.2 {
          717  +  db2 eval { BEGIN; SELECT * FROM t1; }
          718  +  db  eval { INSERT INTO t1 VALUES(3, 4) }
          719  +  file size test.db-wal
          720  +} [wal_file_size 4 1024]
          721  +
          722  +#   At this point the log file contains 4 frames. 3 of which it should
          723  +#   be possible to checkpoint.
          724  +#
          725  +# EVIDENCE-OF: R-16642-42503 If pnLog is not NULL, then *pnLog is set to
          726  +# the total number of frames in the log file or to -1 if the checkpoint
          727  +# could not run because of an error or because the database is not in
          728  +# WAL mode.
          729  +#
          730  +# EVIDENCE-OF: R-10514-25250 If pnCkpt is not NULL,then *pnCkpt is set
          731  +# to the total number of checkpointed frames in the log file (including
          732  +# any that were already checkpointed before the function was called) or
          733  +# to -1 if the checkpoint could not run due to an error or because the
          734  +# database is not in WAL mode.
          735  +#
          736  +do_test 6.4 {
          737  +  lrange [wal_checkpoint_v2 db passive] 1 2
          738  +} {4 3} 
          739  +
          740  +# EVIDENCE-OF: R-37257-17813 Note that upon successful completion of an
          741  +# SQLITE_CHECKPOINT_TRUNCATE, the log file will have been truncated to
          742  +# zero bytes and so both *pnLog and *pnCkpt will be set to zero.
          743  +#
          744  +do_test 6.5 {
          745  +  db2 eval COMMIT
          746  +  wal_checkpoint_v2 db truncate
          747  +} {0 0 0}
          748  +
   486    749   
   487    750   
   488    751   finish_test
          752  +

Added test/e_walhook.test.

            1  +# 2014 December 04
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +source $testdir/wal_common.tcl
           16  +set testprefix e_walhook
           17  +
           18  +
           19  +# EVIDENCE-OF: R-00752-43975 The sqlite3_wal_hook() function is used to
           20  +# register a callback that is invoked each time data is committed to a
           21  +# database in wal mode.
           22  +#
           23  +#   1.1: shows that the wal-hook is not invoked in rollback mode.
           24  +#   1.2: but is invoked in wal mode.
           25  +#
           26  +set ::wal_hook_count 0
           27  +proc my_wal_hook {args} {
           28  +  incr ::wal_hook_count
           29  +  return 0
           30  +}
           31  +
           32  +do_test 1.1.1 {
           33  +  db wal_hook my_wal_hook
           34  +  execsql {
           35  +    CREATE TABLE t1(x);
           36  +    INSERT INTO t1 VALUES(1);
           37  +  }
           38  +  set ::wal_hook_count
           39  +} 0
           40  +do_test 1.1.2 {
           41  +  execsql { PRAGMA journal_mode = wal }
           42  +  set ::wal_hook_count
           43  +} 0
           44  +
           45  +do_test 1.3 {
           46  +  execsql { INSERT INTO t1 VALUES(2) }
           47  +  set wal_hook_count
           48  +} 1
           49  +
           50  +do_test 1.4 {
           51  +  execsql { 
           52  +    BEGIN;
           53  +      INSERT INTO t1 VALUES(3);
           54  +      INSERT INTO t1 VALUES(4);
           55  +    COMMIT;
           56  +  }
           57  +  set wal_hook_count
           58  +} 2
           59  +
           60  +# EVIDENCE-OF: R-65366-15139 The callback is invoked by SQLite after the
           61  +# commit has taken place and the associated write-lock on the database
           62  +# released
           63  +#
           64  +set ::read_ok 0
           65  +proc my_wal_hook {args} {
           66  +  sqlite3 db2 test.db
           67  +  if {[db2 eval { SELECT * FROM t1 }] == "1 2 3 4 5"} {
           68  +    set ::read_ok 1
           69  +  }
           70  +  db2 close
           71  +}
           72  +do_test 2.1 {
           73  +  execsql { INSERT INTO t1 VALUES(5) }
           74  +  set ::read_ok
           75  +} 1
           76  +
           77  +# EVIDENCE-OF: R-44294-52863 The third parameter is the name of the
           78  +# database that was written to - either "main" or the name of an
           79  +# ATTACH-ed database.
           80  +#
           81  +# EVIDENCE-OF: R-18913-19355 The fourth parameter is the number of pages
           82  +# currently in the write-ahead log file, including those that were just
           83  +# committed.
           84  +#
           85  +set ::wal_hook_args [list]
           86  +proc my_wal_hook {dbname nEntry} {
           87  +  set ::wal_hook_args [list $dbname $nEntry]
           88  +}
           89  +forcedelete test.db2
           90  +do_test 3.0 {
           91  +  execsql {
           92  +    ATTACH 'test.db2' AS aux;
           93  +    CREATE TABLE aux.t2(x);
           94  +    PRAGMA aux.journal_mode = wal;
           95  +  }
           96  +} {wal}
           97  +
           98  +# Database "aux"
           99  +do_test 3.1.1 {
          100  +  set wal_hook_args [list]
          101  +  execsql { INSERT INTO t2 VALUES('a') }
          102  +} {}
          103  +do_test 3.1.2 {
          104  +  set wal_hook_args
          105  +} [list aux [wal_frame_count test.db2-wal 1024]]
          106  +
          107  +# Database "main"
          108  +do_test 3.2.1 {
          109  +  set wal_hook_args [list]
          110  +  execsql { INSERT INTO t1 VALUES(6) }
          111  +} {}
          112  +do_test 3.1.2 {
          113  +  set wal_hook_args
          114  +} [list main [wal_frame_count test.db-wal 1024]]
          115  +
          116  +# EVIDENCE-OF: R-14034-00929 If an error code is returned, that error
          117  +# will propagate back up through the SQLite code base to cause the
          118  +# statement that provoked the callback to report an error, though the
          119  +# commit will have still occurred.
          120  +#
          121  +proc my_wal_hook {args} { return 1 ;# SQLITE_ERROR }
          122  +do_catchsql_test 4.1 {
          123  +  INSERT INTO t1 VALUES(7)
          124  +} {1 {SQL logic error or missing database}}
          125  +
          126  +proc my_wal_hook {args} { return 5 ;# SQLITE_BUSY }
          127  +do_catchsql_test 4.2 {
          128  +  INSERT INTO t1 VALUES(8)
          129  +} {1 {database is locked}}
          130  +
          131  +proc my_wal_hook {args} { return 14 ;# SQLITE_CANTOPEN }
          132  +do_catchsql_test 4.3 {
          133  +  INSERT INTO t1 VALUES(9)
          134  +} {1 {unable to open database file}}
          135  +
          136  +do_execsql_test 4.4 {
          137  +  SELECT * FROM t1
          138  +} {1 2 3 4 5 6 7 8 9}
          139  +
          140  +# EVIDENCE-OF: R-10466-53920 Calling sqlite3_wal_hook() replaces any
          141  +# previously registered write-ahead log callback.
          142  +set ::old_wal_hook 0
          143  +proc my_old_wal_hook {args} {
          144  +  incr ::old_wal_hook 
          145  +  return 0
          146  +}
          147  +db wal_hook my_old_wal_hook
          148  +do_test 5.1 {
          149  +  execsql { INSERT INTO t1 VALUES(10) }
          150  +  set ::old_wal_hook
          151  +} {1}
          152  +
          153  +# Replace old_wal_hook. Observe that it is not invoked after it has 
          154  +# been replaced.
          155  +proc my_new_wal_hook {args} { return 0 }
          156  +db wal_hook my_new_wal_hook
          157  +do_test 5.2 {
          158  +  execsql { INSERT INTO t1 VALUES(11) }
          159  +  set ::old_wal_hook
          160  +} {1}
          161  +
          162  +
          163  +
          164  +# EVIDENCE-OF: R-42842-27162 Note that the sqlite3_wal_autocheckpoint()
          165  +# interface and the wal_autocheckpoint pragma both invoke
          166  +# sqlite3_wal_hook() and will those overwrite any prior
          167  +# sqlite3_wal_hook() settings.
          168  +#
          169  +set ::old_wal_hook 0
          170  +proc my_old_wal_hook {args} { incr ::old_wal_hook ; return 0 }
          171  +db wal_hook my_old_wal_hook
          172  +do_test 6.1.1 {
          173  +  execsql { INSERT INTO t1 VALUES(12) }
          174  +  set ::old_wal_hook
          175  +} {1}
          176  +do_test 6.1.2 {
          177  +  execsql { PRAGMA wal_autocheckpoint = 1000 }
          178  +  execsql { INSERT INTO t1 VALUES(12) }
          179  +  set ::old_wal_hook
          180  +} {1}
          181  +
          182  +# EVIDENCE-OF: R-52629-38967 The first parameter passed to the callback
          183  +# function when it is invoked is a copy of the third parameter passed to
          184  +# sqlite3_wal_hook() when registering the callback.
          185  +#
          186  +#    This is tricky to test using the tcl interface. However, the
          187  +#    mechanism used to invoke the tcl script registered as a wal-hook
          188  +#    depends on the context pointer being correctly passed through. And
          189  +#    since multiple different wal-hook scripts have been successfully
          190  +#    invoked by this test script, consider this tested.
          191  +#
          192  +# EVIDENCE-OF: R-23378-42536 The second is a copy of the database
          193  +# handle.
          194  +#
          195  +#    There is an assert() in the C wal-hook used by tclsqlite.c to
          196  +#    prove this. And that hook has been invoked multiple times when
          197  +#    running this script. So consider this requirement tested as well.
          198  +#
          199  +
          200  +finish_test

Changes to test/lock5.test.

   172    172       execsql { 
   173    173         BEGIN;
   174    174         SELECT * FROM t1;
   175    175       } db2
   176    176     } {1 2}
   177    177     do_test lock5-none.5 {
   178    178       execsql COMMIT
   179         -  }
          179  +  } {}
   180    180     do_test lock5-none.6 {
   181    181       sqlite3_release_memory 1000000
   182    182       execsql {SELECT * FROM t1} db2
   183    183     } {1 2}
   184    184   
   185    185     ifcapable memorymanage {
   186    186       do_test lock5-none.6 {

Changes to test/pragma.test.

   450    450   } {{non-unique entry in index t1a} {NULL value in t1x.a} {non-unique entry in index t1a} {NULL value in t1x.a}}
   451    451   do_execsql_test pragma-3.21 {
   452    452     PRAGMA integrity_check(3);
   453    453   } {{non-unique entry in index t1a} {NULL value in t1x.a} {non-unique entry in index t1a}}
   454    454   do_execsql_test pragma-3.22 {
   455    455     PRAGMA integrity_check(2);
   456    456   } {{non-unique entry in index t1a} {NULL value in t1x.a}}
   457         -do_execsql_test pragma-3.21 {
          457  +do_execsql_test pragma-3.23 {
   458    458     PRAGMA integrity_check(1);
   459    459   } {{non-unique entry in index t1a}}
          460  +
          461  +# PRAGMA integrity check (or more specifically the sqlite3BtreeCount()
          462  +# interface) used to leave index cursors in an inconsistent state
          463  +# which could result in an assertion fault in sqlite3BtreeKey()
          464  +# called from saveCursorPosition() if content is removed from the
          465  +# index while the integrity_check is still running.  This test verifies
          466  +# that problem has been fixed.
          467  +#
          468  +do_test pragma-3.30 {
          469  +  db close
          470  +  delete_file test.db
          471  +  sqlite3 db test.db
          472  +  db eval {
          473  +    CREATE TABLE t1(a,b,c);
          474  +    WITH RECURSIVE
          475  +      c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<100)
          476  +    INSERT INTO t1(a,b,c) SELECT i, printf('xyz%08x',i), 2000-i FROM c;
          477  +    CREATE INDEX t1a ON t1(a);
          478  +    CREATE INDEX t1bc ON t1(b,c);
          479  +  }
          480  +  db eval {PRAGMA integrity_check} {
          481  +     db eval {DELETE FROM t1}
          482  +  }
          483  +} {}
   460    484   
   461    485   # Test modifying the cache_size of an attached database.
   462    486   ifcapable pager_pragmas&&attach {
   463    487   do_test pragma-4.1 {
   464    488     execsql {
   465    489       ATTACH 'test2.db' AS aux;
   466    490       pragma aux.cache_size;

Changes to test/threadtest3.c.

    43     43   
    44     44   /* Functions to execute SQL */
    45     45   #define sql_script(x,y,z)       (SEL(x), sql_script_x(x,y,z))
    46     46   #define integrity_check(x,y)    (SEL(x), integrity_check_x(x,y))
    47     47   #define execsql_i64(x,y,...)    (SEL(x), execsql_i64_x(x,y,__VA_ARGS__))
    48     48   #define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__))
    49     49   #define execsql(x,y,...)        (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__))
           50  +#define sql_script_printf(x,y,z,...) (                \
           51  +    SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__)    \
           52  +) 
    50     53   
    51     54   /* Thread functions */
    52         -#define launch_thread(w,x,y,z)  (SEL(w), launch_thread_x(w,x,y,z))
    53         -#define join_all_threads(y,z)   (SEL(y), join_all_threads_x(y,z))
           55  +#define launch_thread(w,x,y,z)     (SEL(w), launch_thread_x(w,x,y,z))
           56  +#define join_all_threads(y,z)      (SEL(y), join_all_threads_x(y,z))
    54     57   
    55     58   /* Timer functions */
    56     59   #define setstoptime(y,z)        (SEL(y), setstoptime_x(y,z))
    57     60   #define timetostop(z)           (SEL(z), timetostop_x(z))
    58     61   
    59     62   /* Report/clear errors. */
    60     63   #define test_error(z, ...)      test_error_x(z, sqlite3_mprintf(__VA_ARGS__))
    61     64   #define clear_error(y,z)        clear_error_x(y, z)
    62     65   
    63     66   /* File-system operations */
    64     67   #define filesize(y,z)           (SEL(y), filesize_x(y,z))
    65     68   #define filecopy(x,y,z)         (SEL(x), filecopy_x(x,y,z))
    66     69   
           70  +#define PTR2INT(x) ((int)((intptr_t)x))
           71  +#define INT2PTR(x) ((void*)((intptr_t)x))
           72  +
    67     73   /*
    68     74   ** End of test code/infrastructure interface macros.
    69     75   *************************************************************************/
    70     76   
    71     77   
    72     78   
    73     79   
................................................................................
   111    117   #  define uint32 unsigned int
   112    118   #endif
   113    119   
   114    120   struct MD5Context {
   115    121     int isInit;
   116    122     uint32 buf[4];
   117    123     uint32 bits[2];
   118         -  unsigned char in[64];
          124  +  union {
          125  +    unsigned char in[64];
          126  +    uint32 in32[16];
          127  +  } u;
   119    128   };
   120    129   typedef struct MD5Context MD5Context;
   121    130   
   122    131   /*
   123    132    * Note: this code is harmless on little-endian machines.
   124    133    */
   125    134   static void byteReverse (unsigned char *buf, unsigned longs){
................................................................................
   260    269     ctx->bits[1] += len >> 29;
   261    270   
   262    271     t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */
   263    272   
   264    273     /* Handle any leading odd-sized chunks */
   265    274   
   266    275     if ( t ) {
   267         -    unsigned char *p = (unsigned char *)ctx->in + t;
          276  +    unsigned char *p = (unsigned char *)ctx->u.in + t;
   268    277   
   269    278       t = 64-t;
   270    279       if (len < t) {
   271    280         memcpy(p, buf, len);
   272    281         return;
   273    282       }
   274    283       memcpy(p, buf, t);
   275         -    byteReverse(ctx->in, 16);
   276         -    MD5Transform(ctx->buf, (uint32 *)ctx->in);
          284  +    byteReverse(ctx->u.in, 16);
          285  +    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
   277    286       buf += t;
   278    287       len -= t;
   279    288     }
   280    289   
   281    290     /* Process data in 64-byte chunks */
   282    291   
   283    292     while (len >= 64) {
   284         -    memcpy(ctx->in, buf, 64);
   285         -    byteReverse(ctx->in, 16);
   286         -    MD5Transform(ctx->buf, (uint32 *)ctx->in);
          293  +    memcpy(ctx->u.in, buf, 64);
          294  +    byteReverse(ctx->u.in, 16);
          295  +    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
   287    296       buf += 64;
   288    297       len -= 64;
   289    298     }
   290    299   
   291    300     /* Handle any remaining bytes of data. */
   292    301   
   293         -  memcpy(ctx->in, buf, len);
          302  +  memcpy(ctx->u.in, buf, len);
   294    303   }
   295    304   
   296    305   /*
   297    306    * Final wrapup - pad to 64-byte boundary with the bit pattern 
   298    307    * 1 0* (64-bit count of bits processed, MSB-first)
   299    308    */
   300    309   static void MD5Final(unsigned char digest[16], MD5Context *ctx){
................................................................................
   302    311     unsigned char *p;
   303    312   
   304    313     /* Compute number of bytes mod 64 */
   305    314     count = (ctx->bits[0] >> 3) & 0x3F;
   306    315   
   307    316     /* Set the first char of padding to 0x80.  This is safe since there is
   308    317        always at least one byte free */
   309         -  p = ctx->in + count;
          318  +  p = ctx->u.in + count;
   310    319     *p++ = 0x80;
   311    320   
   312    321     /* Bytes of padding needed to make 64 bytes */
   313    322     count = 64 - 1 - count;
   314    323   
   315    324     /* Pad out to 56 mod 64 */
   316    325     if (count < 8) {
   317    326       /* Two lots of padding:  Pad the first block to 64 bytes */
   318    327       memset(p, 0, count);
   319         -    byteReverse(ctx->in, 16);
   320         -    MD5Transform(ctx->buf, (uint32 *)ctx->in);
          328  +    byteReverse(ctx->u.in, 16);
          329  +    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
   321    330   
   322    331       /* Now fill the next block with 56 bytes */
   323         -    memset(ctx->in, 0, 56);
          332  +    memset(ctx->u.in, 0, 56);
   324    333     } else {
   325    334       /* Pad block to 56 bytes */
   326    335       memset(p, 0, count-8);
   327    336     }
   328         -  byteReverse(ctx->in, 14);
          337  +  byteReverse(ctx->u.in, 14);
   329    338   
   330    339     /* Append length in bits and transform */
   331         -  ((uint32 *)ctx->in)[ 14 ] = ctx->bits[0];
   332         -  ((uint32 *)ctx->in)[ 15 ] = ctx->bits[1];
          340  +  ctx->u.in32[14] = ctx->bits[0];
          341  +  ctx->u.in32[15] = ctx->bits[1];
   333    342   
   334         -  MD5Transform(ctx->buf, (uint32 *)ctx->in);
          343  +  MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
   335    344     byteReverse((unsigned char *)ctx->buf, 4);
   336    345     memcpy(digest, ctx->buf, 16);
   337         -  memset(ctx, 0, sizeof(ctx));    /* In case it is sensitive */
          346  +  memset(ctx, 0, sizeof(*ctx));    /* In case it is sensitive */
   338    347   }
   339    348   
   340    349   /*
   341    350   ** Convert a 128-bit MD5 digest into a 32-digit base-16 number.
   342    351   */
   343    352   static void MD5DigestToBase16(unsigned char *digest, char *zBuf){
   344    353     static char const zEncode[] = "0123456789abcdef";
................................................................................
   394    403   
   395    404   typedef struct Threadset Threadset;
   396    405   typedef struct Thread Thread;
   397    406   
   398    407   /* Total number of errors in this process so far. */
   399    408   static int nGlobalErr = 0;
   400    409   
   401         -/* Set to true to run in "process" instead of "thread" mode. */
   402         -static int bProcessMode = 0;
   403         -
   404    410   struct Error {
   405    411     int rc;
   406    412     int iLine;
   407    413     char *zErr;
   408    414   };
   409    415   
   410    416   struct Sqlite {
................................................................................
   417    423   struct Statement {
   418    424     sqlite3_stmt *pStmt;            /* Pre-compiled statement handle */
   419    425     Statement *pNext;               /* Next statement in linked-list */
   420    426   };
   421    427   
   422    428   struct Thread {
   423    429     int iTid;                       /* Thread number within test */
   424         -  int iArg;                       /* Integer argument passed by caller */
          430  +  void* pArg;                     /* Pointer argument passed by caller */
   425    431   
   426    432     pthread_t tid;                  /* Thread id */
   427         -  char *(*xProc)(int, int);       /* Thread main proc */
          433  +  char *(*xProc)(int, void*);     /* Thread main proc */
   428    434     Thread *pNext;                  /* Next in this list of threads */
   429    435   };
   430    436   
   431    437   struct Threadset {
   432    438     int iMaxTid;                    /* Largest iTid value allocated so far */
   433    439     Thread *pThread;                /* Linked list of threads */
   434    440   };
................................................................................
   502    508     Error *pErr,                    /* IN/OUT: Error code */
   503    509     Sqlite *pDb,                    /* OUT: Database handle */
   504    510     const char *zFile,              /* Database file name */
   505    511     int bDelete                     /* True to delete db file before opening */
   506    512   ){
   507    513     if( pErr->rc==SQLITE_OK ){
   508    514       int rc;
          515  +    int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI;
   509    516       if( bDelete ) unlink(zFile);
   510         -    rc = sqlite3_open(zFile, &pDb->db);
          517  +    rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0);
   511    518       if( rc ){
   512    519         sqlite_error(pErr, pDb, "open");
   513    520         sqlite3_close(pDb->db);
   514    521         pDb->db = 0;
   515    522       }else{
   516    523         sqlite3_create_function(
   517    524             pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize
................................................................................
   551    558     Sqlite *pDb,                    /* Database handle */
   552    559     const char *zSql                /* SQL script to execute */
   553    560   ){
   554    561     if( pErr->rc==SQLITE_OK ){
   555    562       pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr);
   556    563     }
   557    564   }
          565  +
          566  +static void sql_script_printf_x(
          567  +  Error *pErr,                    /* IN/OUT: Error code */
          568  +  Sqlite *pDb,                    /* Database handle */
          569  +  const char *zFormat,            /* SQL printf format string */
          570  +  ...                             /* Printf args */
          571  +){
          572  +  va_list ap;                     /* ... printf arguments */
          573  +  va_start(ap, zFormat);
          574  +  if( pErr->rc==SQLITE_OK ){
          575  +    char *zSql = sqlite3_vmprintf(zFormat, ap);
          576  +    pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr);
          577  +    sqlite3_free(zSql);
          578  +  }
          579  +  va_end(ap);
          580  +}
   558    581   
   559    582   static Statement *getSqlStatement(
   560    583     Error *pErr,                    /* IN/OUT: Error code */
   561    584     Sqlite *pDb,                    /* Database handle */
   562    585     const char *zSql                /* SQL statement */
   563    586   ){
   564    587     Statement *pRet;
................................................................................
   620    643     Sqlite *pDb,                    /* Database handle */
   621    644     ...                             /* SQL and pointers to parameter values */
   622    645   ){
   623    646     i64 iRet = 0;
   624    647     if( pErr->rc==SQLITE_OK ){
   625    648       sqlite3_stmt *pStmt;          /* SQL statement to execute */
   626    649       va_list ap;                   /* ... arguments */
   627         -    int i;                        /* Used to iterate through parameters */
   628    650       va_start(ap, pDb);
   629    651       pStmt = getAndBindSqlStatement(pErr, pDb, ap);
   630    652       if( pStmt ){
   631         -      int rc;
   632    653         int first = 1;
   633    654         while( SQLITE_ROW==sqlite3_step(pStmt) ){
   634    655           if( first && sqlite3_column_count(pStmt)>0 ){
   635    656             iRet = sqlite3_column_int64(pStmt, 0);
   636    657           }
   637    658           first = 0;
   638    659         }
................................................................................
   659    680       memset(&pDb->aText[pDb->nText], 0, sizeof(char*)*(iSlot+1-pDb->nText));
   660    681       pDb->nText = iSlot+1;
   661    682     }
   662    683   
   663    684     if( pErr->rc==SQLITE_OK ){
   664    685       sqlite3_stmt *pStmt;          /* SQL statement to execute */
   665    686       va_list ap;                   /* ... arguments */
   666         -    int i;                        /* Used to iterate through parameters */
   667    687       va_start(ap, iSlot);
   668    688       pStmt = getAndBindSqlStatement(pErr, pDb, ap);
   669    689       if( pStmt ){
   670         -      int rc;
   671    690         int first = 1;
   672    691         while( SQLITE_ROW==sqlite3_step(pStmt) ){
   673    692           if( first && sqlite3_column_count(pStmt)>0 ){
   674    693             zRet = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
   675    694             sqlite3_free(pDb->aText[iSlot]);
   676    695             pDb->aText[iSlot] = zRet;
   677    696           }
................................................................................
   689    708   
   690    709   static void integrity_check_x(
   691    710     Error *pErr,                    /* IN/OUT: Error code */
   692    711     Sqlite *pDb                     /* Database handle */
   693    712   ){
   694    713     if( pErr->rc==SQLITE_OK ){
   695    714       Statement *pStatement;        /* Statement to execute */
   696         -    int rc;                       /* Return code */
   697    715       char *zErr = 0;               /* Integrity check error */
   698    716   
   699    717       pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check");
   700    718       if( pStatement ){
   701    719         sqlite3_stmt *pStmt = pStatement->pStmt;
   702    720         while( SQLITE_ROW==sqlite3_step(pStmt) ){
   703         -        const char *z = sqlite3_column_text(pStmt, 0);
          721  +        const char *z = (const char*)sqlite3_column_text(pStmt, 0);
   704    722           if( strcmp(z, "ok") ){
   705    723             if( zErr==0 ){
   706    724               zErr = sqlite3_mprintf("%s", z);
   707    725             }else{
   708    726               zErr = sqlite3_mprintf("%z\n%s", zErr, z);
   709    727             }
   710    728           }
................................................................................
   717    735         }
   718    736       }
   719    737     }
   720    738   }
   721    739   
   722    740   static void *launch_thread_main(void *pArg){
   723    741     Thread *p = (Thread *)pArg;
   724         -  return (void *)p->xProc(p->iTid, p->iArg);
          742  +  return (void *)p->xProc(p->iTid, p->pArg);
   725    743   }
   726    744   
   727    745   static void launch_thread_x(
   728    746     Error *pErr,                    /* IN/OUT: Error code */
   729    747     Threadset *pThreads,            /* Thread set */
   730         -  char *(*xProc)(int, int),       /* Proc to run */
   731         -  int iArg                        /* Argument passed to thread proc */
          748  +  char *(*xProc)(int, void*),     /* Proc to run */
          749  +  void *pArg                      /* Argument passed to thread proc */
   732    750   ){
   733    751     if( pErr->rc==SQLITE_OK ){
   734    752       int iTid = ++pThreads->iMaxTid;
   735    753       Thread *p;
   736    754       int rc;
   737    755   
   738    756       p = (Thread *)sqlite3_malloc(sizeof(Thread));
   739    757       memset(p, 0, sizeof(Thread));
   740    758       p->iTid = iTid;
   741         -    p->iArg = iArg;
          759  +    p->pArg = pArg;
   742    760       p->xProc = xProc;
   743    761   
   744    762       rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p);
   745    763       if( rc!=0 ){
   746    764         system_error(pErr, rc);
   747    765         sqlite3_free(p);
   748    766       }else{
................................................................................
   891    909   **************************************************************************
   892    910   ** End infrastructure. Begin tests.
   893    911   */
   894    912   
   895    913   #define WALTHREAD1_NTHREAD  10
   896    914   #define WALTHREAD3_NTHREAD  6
   897    915   
   898         -static char *walthread1_thread(int iTid, int iArg){
          916  +static char *walthread1_thread(int iTid, void *pArg){
   899    917     Error err = {0};                /* Error code and message */
   900    918     Sqlite db = {0};                /* SQLite database connection */
   901    919     int nIter = 0;                  /* Iterations so far */
   902    920   
   903    921     opendb(&err, &db, "test.db", 0);
   904    922     while( !timetostop(&err) ){
   905    923       const char *azSql[] = {
................................................................................
   930    948     }
   931    949     closedb(&err, &db);
   932    950   
   933    951     print_and_free_err(&err);
   934    952     return sqlite3_mprintf("%d iterations", nIter);
   935    953   }
   936    954   
   937         -static char *walthread1_ckpt_thread(int iTid, int iArg){
          955  +static char *walthread1_ckpt_thread(int iTid, void *pArg){
   938    956     Error err = {0};                /* Error code and message */
   939    957     Sqlite db = {0};                /* SQLite database connection */
   940    958     int nCkpt = 0;                  /* Checkpoints so far */
   941    959   
   942    960     opendb(&err, &db, "test.db", 0);
   943    961     while( !timetostop(&err) ){
   944    962       usleep(500*1000);
................................................................................
   973    991     }
   974    992     launch_thread(&err, &threads, walthread1_ckpt_thread, 0);
   975    993     join_all_threads(&err, &threads);
   976    994   
   977    995     print_and_free_err(&err);
   978    996   }
   979    997   
   980         -static char *walthread2_thread(int iTid, int iArg){
          998  +static char *walthread2_thread(int iTid, void *pArg){
   981    999     Error err = {0};                /* Error code and message */
   982   1000     Sqlite db = {0};                /* SQLite database connection */
   983   1001     int anTrans[2] = {0, 0};        /* Number of WAL and Rollback transactions */
         1002  +  int iArg = PTR2INT(pArg);
   984   1003   
   985   1004     const char *zJournal = "PRAGMA journal_mode = WAL";
   986   1005     if( iArg ){ zJournal = "PRAGMA journal_mode = DELETE"; }
   987   1006   
   988   1007     while( !timetostop(&err) ){
   989   1008       int journal_exists = 0;
   990   1009       int wal_exists = 0;
................................................................................
  1022   1041     opendb(&err, &db, "test.db", 1);
  1023   1042     sql_script(&err, &db, "CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE)");
  1024   1043     closedb(&err, &db);
  1025   1044   
  1026   1045     setstoptime(&err, nMs);
  1027   1046     launch_thread(&err, &threads, walthread2_thread, 0);
  1028   1047     launch_thread(&err, &threads, walthread2_thread, 0);
  1029         -  launch_thread(&err, &threads, walthread2_thread, 1);
  1030         -  launch_thread(&err, &threads, walthread2_thread, 1);
         1048  +  launch_thread(&err, &threads, walthread2_thread, (void*)1);
         1049  +  launch_thread(&err, &threads, walthread2_thread, (void*)1);
  1031   1050     join_all_threads(&err, &threads);
  1032   1051   
  1033   1052     print_and_free_err(&err);
  1034   1053   }
  1035   1054   
  1036         -static char *walthread3_thread(int iTid, int iArg){
         1055  +static char *walthread3_thread(int iTid, void *pArg){
  1037   1056     Error err = {0};                /* Error code and message */
  1038   1057     Sqlite db = {0};                /* SQLite database connection */
  1039   1058     i64 iNextWrite;                 /* Next value this thread will write */
         1059  +  int iArg = PTR2INT(pArg);
  1040   1060   
  1041   1061     opendb(&err, &db, "test.db", 0);
  1042   1062     sql_script(&err, &db, "PRAGMA wal_autocheckpoint = 10");
  1043   1063   
  1044   1064     iNextWrite = iArg+1;
  1045   1065     while( 1 ){
  1046   1066       i64 sum1;
................................................................................
  1083   1103         "CREATE INDEX i2 ON t1(sum2);"
  1084   1104         "INSERT INTO t1 VALUES(0, 0, 0);"
  1085   1105     );
  1086   1106     closedb(&err, &db);
  1087   1107   
  1088   1108     setstoptime(&err, nMs);
  1089   1109     for(i=0; i<WALTHREAD3_NTHREAD; i++){
  1090         -    launch_thread(&err, &threads, walthread3_thread, i);
         1110  +    launch_thread(&err, &threads, walthread3_thread, INT2PTR(i));
  1091   1111     }
  1092   1112     join_all_threads(&err, &threads);
  1093   1113   
  1094   1114     print_and_free_err(&err);
  1095   1115   }
  1096   1116   
  1097         -static char *walthread4_reader_thread(int iTid, int iArg){
         1117  +static char *walthread4_reader_thread(int iTid, void *pArg){
  1098   1118     Error err = {0};                /* Error code and message */
  1099   1119     Sqlite db = {0};                /* SQLite database connection */
  1100   1120   
  1101   1121     opendb(&err, &db, "test.db", 0);
  1102   1122     while( !timetostop(&err) ){
  1103   1123       integrity_check(&err, &db);
  1104   1124     }
  1105   1125     closedb(&err, &db);
  1106   1126   
  1107   1127     print_and_free_err(&err);
  1108   1128     return 0;
  1109   1129   }
  1110   1130   
  1111         -static char *walthread4_writer_thread(int iTid, int iArg){
         1131  +static char *walthread4_writer_thread(int iTid, void *pArg){
  1112   1132     Error err = {0};                /* Error code and message */
  1113   1133     Sqlite db = {0};                /* SQLite database connection */
  1114   1134     i64 iRow = 1;
  1115   1135   
  1116   1136     opendb(&err, &db, "test.db", 0);
  1117   1137     sql_script(&err, &db, "PRAGMA wal_autocheckpoint = 15;");
  1118   1138     while( !timetostop(&err) ){
................................................................................
  1144   1164     launch_thread(&err, &threads, walthread4_reader_thread, 0);
  1145   1165     launch_thread(&err, &threads, walthread4_writer_thread, 0);
  1146   1166     join_all_threads(&err, &threads);
  1147   1167   
  1148   1168     print_and_free_err(&err);
  1149   1169   }
  1150   1170   
  1151         -static char *walthread5_thread(int iTid, int iArg){
         1171  +static char *walthread5_thread(int iTid, void *pArg){
  1152   1172     Error err = {0};                /* Error code and message */
  1153   1173     Sqlite db = {0};                /* SQLite database connection */
  1154   1174     i64 nRow;
  1155   1175   
  1156   1176     opendb(&err, &db, "test.db", 0);
  1157   1177     nRow = execsql_i64(&err, &db, "SELECT count(*) FROM t1");
  1158   1178     closedb(&err, &db);
................................................................................
  1276   1296   ** Test case "dynamic_triggers"
  1277   1297   **
  1278   1298   **   Two threads executing statements that cause deeply nested triggers
  1279   1299   **   to fire. And one thread busily creating and deleting triggers. This
  1280   1300   **   is an attempt to find a bug reported to us.
  1281   1301   */
  1282   1302   
  1283         -static char *dynamic_triggers_1(int iTid, int iArg){
         1303  +static char *dynamic_triggers_1(int iTid, void *pArg){
  1284   1304     Error err = {0};                /* Error code and message */
  1285   1305     Sqlite db = {0};                /* SQLite database connection */
  1286   1306     int nDrop = 0;
  1287   1307     int nCreate = 0;
  1288   1308   
  1289   1309     opendb(&err, &db, "test.db", 0);
  1290   1310     while( !timetostop(&err) ){
................................................................................
  1322   1342       for(i=1; i<9; i++){
  1323   1343         char *zSql = sqlite3_mprintf("DROP TRIGGER dtr%d", i);
  1324   1344         execsql(&err, &db, zSql);
  1325   1345         sqlite3_free(zSql);
  1326   1346         nDrop++;
  1327   1347       }
  1328   1348     }
         1349  +  closedb(&err, &db);
  1329   1350   
  1330   1351     print_and_free_err(&err);
  1331   1352     return sqlite3_mprintf("%d created, %d dropped", nCreate, nDrop);
  1332   1353   }
  1333   1354   
  1334         -static char *dynamic_triggers_2(int iTid, int iArg){
         1355  +static char *dynamic_triggers_2(int iTid, void *pArg){
  1335   1356     Error err = {0};                /* Error code and message */
  1336   1357     Sqlite db = {0};                /* SQLite database connection */
  1337   1358     i64 iVal = 0;
  1338   1359     int nInsert = 0;
  1339   1360     int nDelete = 0;
  1340   1361   
  1341   1362     opendb(&err, &db, "test.db", 0);
................................................................................
  1348   1369   
  1349   1370       do {
  1350   1371         iVal = (iVal+1)%100;
  1351   1372         execsql(&err, &db, "DELETE FROM t1 WHERE x = :iX", &iVal);
  1352   1373         nDelete++;
  1353   1374       } while( iVal );
  1354   1375     }
         1376  +  closedb(&err, &db);
  1355   1377   
  1356   1378     print_and_free_err(&err);
  1357   1379     return sqlite3_mprintf("%d inserts, %d deletes", nInsert, nDelete);
  1358   1380   }
  1359   1381   
  1360   1382   static void dynamic_triggers(int nMs){
  1361   1383     Error err = {0};
................................................................................
  1372   1394         "CREATE TABLE t4(x, y);"
  1373   1395         "CREATE TABLE t5(x, y);"
  1374   1396         "CREATE TABLE t6(x, y);"
  1375   1397         "CREATE TABLE t7(x, y);"
  1376   1398         "CREATE TABLE t8(x, y);"
  1377   1399         "CREATE TABLE t9(x, y);"
  1378   1400     );
         1401  +  closedb(&err, &db);
  1379   1402   
  1380   1403     setstoptime(&err, nMs);
  1381   1404   
  1382   1405     sqlite3_enable_shared_cache(1);
  1383   1406     launch_thread(&err, &threads, dynamic_triggers_2, 0);
  1384   1407     launch_thread(&err, &threads, dynamic_triggers_2, 0);
  1385         -  sqlite3_enable_shared_cache(0);
  1386   1408   
  1387   1409     sleep(2);
         1410  +  sqlite3_enable_shared_cache(0);
  1388   1411   
  1389   1412     launch_thread(&err, &threads, dynamic_triggers_2, 0);
  1390   1413     launch_thread(&err, &threads, dynamic_triggers_1, 0);
  1391   1414   
  1392   1415     join_all_threads(&err, &threads);
  1393   1416   
  1394   1417     print_and_free_err(&err);
  1395   1418   }
  1396   1419   
  1397   1420   
  1398   1421   
  1399   1422   #include "tt3_checkpoint.c"
  1400   1423   #include "tt3_index.c"
         1424  +#include "tt3_lookaside1.c"
         1425  +#include "tt3_vacuum.c"
         1426  +#include "tt3_stress.c"
  1401   1427   
  1402   1428   int main(int argc, char **argv){
  1403   1429     struct ThreadTest {
  1404   1430       void (*xTest)(int);
  1405   1431       const char *zTest;
  1406   1432       int nMs;
  1407   1433     } aTest[] = {
................................................................................
  1415   1441       { cgt_pager_1,      "cgt_pager_1", 0 },
  1416   1442       { dynamic_triggers, "dynamic_triggers", 20000 },
  1417   1443   
  1418   1444       { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
  1419   1445       { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },
  1420   1446   
  1421   1447       { create_drop_index_1, "create_drop_index_1", 10000 },
         1448  +    { lookaside1,          "lookaside1", 10000 },
         1449  +    { vacuum1,             "vacuum1", 10000 },
         1450  +    { stress1,             "stress1", 10000 },
         1451  +    { stress2,             "stress2", 60000 },
  1422   1452     };
  1423   1453   
  1424   1454     int i;
  1425         -  char *zTest = 0;
  1426         -  int nTest = 0;
  1427   1455     int bTestfound = 0;
  1428         -  int bPrefix = 0;
  1429   1456   
  1430         -  if( argc>2 ) goto usage;
  1431         -  if( argc==2 ){
  1432         -    zTest = argv[1];
  1433         -    nTest = strlen(zTest);
  1434         -    if( zTest[nTest-1]=='*' ){
  1435         -      nTest--;
  1436         -      bPrefix = 1;
  1437         -    }
  1438         -  }
  1439         -
         1457  +  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  1440   1458     sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  1441   1459   
  1442   1460     for(i=0; i<sizeof(aTest)/sizeof(aTest[0]); i++){
  1443   1461       char const *z = aTest[i].zTest;
  1444         -    int n = strlen(z);
  1445         -    if( !zTest || ((bPrefix || n==nTest) && 0==strncmp(zTest, z, nTest)) ){
  1446         -      printf("Running %s for %d seconds...\n", z, aTest[i].nMs/1000);
  1447         -      aTest[i].xTest(aTest[i].nMs);
  1448         -      bTestfound++;
         1462  +    if( argc>1 ){
         1463  +      int iArg;
         1464  +      for(iArg=1; iArg<argc; iArg++){
         1465  +        if( 0==sqlite3_strglob(argv[iArg], z) ) break;
         1466  +      }
         1467  +      if( iArg==argc ) continue;
  1449   1468       }
         1469  +
         1470  +    printf("Running %s for %d seconds...\n", z, aTest[i].nMs/1000);
         1471  +    aTest[i].xTest(aTest[i].nMs);
         1472  +    bTestfound++;
  1450   1473     }
  1451   1474     if( bTestfound==0 ) goto usage;
  1452   1475   
  1453   1476     printf("Total of %d errors across all tests\n", nGlobalErr);
  1454   1477     return (nGlobalErr>0 ? 255 : 0);
  1455   1478   
  1456   1479    usage:
  1457         -  printf("Usage: %s [testname|testprefix*]\n", argv[0]);
         1480  +  printf("Usage: %s [testname|testprefix*]...\n", argv[0]);
  1458   1481     printf("Available tests are:\n");
  1459   1482     for(i=0; i<sizeof(aTest)/sizeof(aTest[0]); i++){
  1460   1483       printf("   %s\n", aTest[i].zTest);
  1461   1484     }
  1462   1485   
  1463   1486     return 254;
  1464   1487   }
  1465   1488   
  1466   1489   

Added test/threadtest4.c.

            1  +/*
            2  +** 2014-12-11
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** This file implements a simple standalone program used to stress the
           13  +** SQLite library when accessing the same set of databases simultaneously
           14  +** from multiple threads in shared-cache mode.
           15  +**
           16  +** This test program runs on unix-like systems only.  It uses pthreads.
           17  +** To compile:
           18  +**
           19  +**     gcc -g -Wall -I. threadtest4.c sqlite3.c -ldl -lpthread
           20  +**
           21  +** To run:
           22  +**
           23  +**     ./a.out 10
           24  +**
           25  +** The argument is the number of threads.  There are also options, such
           26  +** as -wal and -multithread and -serialized.
           27  +**
           28  +** Consider also compiling with clang instead of gcc and adding the
           29  +** -fsanitize=thread option.
           30  +*/
           31  +#include "sqlite3.h"
           32  +#include <pthread.h>
           33  +#include <sched.h>
           34  +#include <stdio.h>
           35  +#include <stdlib.h>
           36  +#include <string.h>
           37  +#include <unistd.h>
           38  +#include <stdarg.h>
           39  +
           40  +/*
           41  +** An instance of the following structure is passed into each worker
           42  +** thread.
           43  +*/
           44  +typedef struct WorkerInfo WorkerInfo;
           45  +struct WorkerInfo {
           46  +  int tid;                    /* Thread ID */
           47  +  int nWorker;                /* Total number of workers */
           48  +  unsigned wkrFlags;          /* Flags */
           49  +  sqlite3 *mainDb;            /* Database connection of the main thread */
           50  +  sqlite3 *db;                /* Database connection of this thread */
           51  +  int nErr;                   /* Number of errors seen by this thread */
           52  +  int nTest;                  /* Number of tests run by this thread */
           53  +  char *zMsg;                 /* Message returned by this thread */
           54  +  pthread_t id;               /* Thread id */
           55  +  pthread_mutex_t *pWrMutex;  /* Hold this mutex while writing */
           56  +};
           57  +
           58  +/*
           59  +** Allowed values for WorkerInfo.wkrFlags
           60  +*/
           61  +#define TT4_SERIALIZED    0x0000001   /* The --serialized option is used */
           62  +#define TT4_WAL           0x0000002   /* WAL mode in use */
           63  +#define TT4_TRACE         0x0000004   /* Trace activity */
           64  +
           65  +
           66  +/*
           67  +** Report an OOM error and die if the argument is NULL
           68  +*/
           69  +static void check_oom(void *x){
           70  +  if( x==0 ){
           71  +    fprintf(stderr, "out of memory\n");
           72  +    exit(1);
           73  +  }
           74  +}
           75  +
           76  +/*
           77  +** Allocate memory.  If the allocation fails, print an error message and
           78  +** kill the process.
           79  +*/
           80  +static void *safe_malloc(int sz){
           81  +  void *x = sqlite3_malloc(sz>0?sz:1);
           82  +  check_oom(x);
           83  +  return x;
           84  +}
           85  +
           86  +/*
           87  +** Print a trace message for a worker
           88  +*/
           89  +static void worker_trace(WorkerInfo *p, const char *zFormat, ...){
           90  +  va_list ap;
           91  +  char *zMsg;
           92  +  if( (p->wkrFlags & TT4_TRACE)==0 ) return;
           93  +  va_start(ap, zFormat);
           94  +  zMsg = sqlite3_vmprintf(zFormat, ap);
           95  +  check_oom(zMsg);
           96  +  va_end(ap);
           97  +  fprintf(stderr, "TRACE(%02d): %s\n", p->tid, zMsg);
           98  +  sqlite3_free(zMsg);
           99  +}
          100  +
          101  +/*
          102  +** Prepare a single SQL query
          103  +*/
          104  +static sqlite3_stmt *prep_sql(sqlite3 *db, const char *zFormat, ...){
          105  +  va_list ap;
          106  +  char *zSql;
          107  +  int rc;
          108  +  sqlite3_stmt *pStmt = 0;
          109  +
          110  +  va_start(ap, zFormat);
          111  +  zSql = sqlite3_vmprintf(zFormat, ap);
          112  +  va_end(ap);
          113  +  check_oom(zSql);
          114  +  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
          115  +  if( rc!=SQLITE_OK ){
          116  +    fprintf(stderr, "SQL error (%d,%d): %s\nWhile preparing: [%s]\n",
          117  +            rc, sqlite3_extended_errcode(db), sqlite3_errmsg(db), zSql);
          118  +    exit(1);
          119  +  }
          120  +  sqlite3_free(zSql);
          121  +  return pStmt;
          122  +}
          123  +
          124  +/*
          125  +** Run a SQL statements.  Panic if unable.
          126  +*/
          127  +static void run_sql(WorkerInfo *p, const char *zFormat, ...){
          128  +  va_list ap;
          129  +  char *zSql;
          130  +  int rc;
          131  +  sqlite3_stmt *pStmt = 0;
          132  +  int nRetry = 0;
          133  +
          134  +  va_start(ap, zFormat);
          135  +  zSql = sqlite3_vmprintf(zFormat, ap);
          136  +  va_end(ap);
          137  +  check_oom(zSql);
          138  +  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
          139  +  if( rc!=SQLITE_OK ){
          140  +    fprintf(stderr, "SQL error (%d,%d): %s\nWhile preparing: [%s]\n",
          141  +            rc, sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zSql);
          142  +    exit(1);
          143  +  }
          144  +  worker_trace(p, "running [%s]", zSql);
          145  +  while( (rc = sqlite3_step(pStmt))!=SQLITE_DONE ){
          146  +    if( (rc&0xff)==SQLITE_BUSY || (rc&0xff)==SQLITE_LOCKED ){
          147  +      sqlite3_reset(pStmt);
          148  +      nRetry++;
          149  +      if( nRetry<10 ){
          150  +        worker_trace(p, "retry %d for [%s]", nRetry, zSql);
          151  +        sched_yield();
          152  +        continue;
          153  +      }else{
          154  +        fprintf(stderr, "Deadlock in thread %d while running [%s]\n",
          155  +                p->tid, zSql);
          156  +        exit(1);
          157  +      }
          158  +    }
          159  +    if( rc!=SQLITE_ROW ){
          160  +      fprintf(stderr, "SQL error (%d,%d): %s\nWhile running [%s]\n",
          161  +              rc, sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zSql);
          162  +      exit(1);
          163  +    }
          164  +  }
          165  +  sqlite3_free(zSql);
          166  +  sqlite3_finalize(pStmt);
          167  +}
          168  +
          169  +
          170  +/*
          171  +** Open the database connection for WorkerInfo.  The order in which
          172  +** the files are opened is a function of the tid value.
          173  +*/
          174  +static void worker_open_connection(WorkerInfo *p, int iCnt){
          175  +  char *zFile;
          176  +  int x;
          177  +  int rc;
          178  +  static const unsigned char aOrder[6][3] = {
          179  +    { 1, 2, 3},
          180  +    { 1, 3, 2},
          181  +    { 2, 1, 3},
          182  +    { 2, 3, 1},
          183  +    { 3, 1, 2},
          184  +    { 3, 2, 1}
          185  +  };
          186  +  x = (p->tid + iCnt) % 6;
          187  +  zFile = sqlite3_mprintf("tt4-test%d.db", aOrder[x][0]);
          188  +  check_oom(zFile);
          189  +  worker_trace(p, "open %s", zFile);
          190  +  rc = sqlite3_open_v2(zFile, &p->db,
          191  +                       SQLITE_OPEN_READWRITE|SQLITE_OPEN_SHAREDCACHE, 0);
          192  +  if( rc!=SQLITE_OK ){
          193  +    fprintf(stderr, "sqlite_open_v2(%s) failed on thread %d\n",
          194  +            zFile, p->tid);
          195  +    exit(1);
          196  +  }
          197  +  sqlite3_free(zFile);
          198  +  run_sql(p, "PRAGMA read_uncommitted=ON;");
          199  +  sqlite3_busy_timeout(p->db, 10000);
          200  +  run_sql(p, "PRAGMA synchronous=OFF;");
          201  +  run_sql(p, "ATTACH 'tt4-test%d.db' AS aux1", aOrder[x][1]);
          202  +  run_sql(p, "ATTACH 'tt4-test%d.db' AS aux2", aOrder[x][2]);
          203  +}
          204  +
          205  +/*
          206  +** Close the worker database connection
          207  +*/
          208  +static void worker_close_connection(WorkerInfo *p){
          209  +  if( p->db ){
          210  +    worker_trace(p, "close");
          211  +    sqlite3_close(p->db);
          212  +    p->db = 0;
          213  +  }
          214  +}
          215  +
          216  +/*
          217  +** Delete all content in the three databases associated with a
          218  +** single thread.  Make this happen all in a single transaction if
          219  +** inTrans is true, or separately for each database if inTrans is
          220  +** false.
          221  +*/
          222  +static void worker_delete_all_content(WorkerInfo *p, int inTrans){
          223  +  if( inTrans ){
          224  +    pthread_mutex_lock(p->pWrMutex);
          225  +    run_sql(p, "BEGIN");
          226  +    run_sql(p, "DELETE FROM t1 WHERE tid=%d", p->tid);
          227  +    run_sql(p, "DELETE FROM t2 WHERE tid=%d", p->tid);
          228  +    run_sql(p, "DELETE FROM t3 WHERE tid=%d", p->tid);
          229  +    run_sql(p, "COMMIT");
          230  +    pthread_mutex_unlock(p->pWrMutex);
          231  +    p->nTest++;
          232  +  }else{
          233  +    pthread_mutex_lock(p->pWrMutex);
          234  +    run_sql(p, "DELETE FROM t1 WHERE tid=%d", p->tid);
          235  +    pthread_mutex_unlock(p->pWrMutex);
          236  +    p->nTest++;
          237  +    pthread_mutex_lock(p->pWrMutex);
          238  +    run_sql(p, "DELETE FROM t2 WHERE tid=%d", p->tid);
          239  +    pthread_mutex_unlock(p->pWrMutex);
          240  +    p->nTest++;
          241  +    pthread_mutex_lock(p->pWrMutex);
          242  +    run_sql(p, "DELETE FROM t3 WHERE tid=%d", p->tid);
          243  +    pthread_mutex_unlock(p->pWrMutex);
          244  +    p->nTest++;
          245  +  }
          246  +}
          247  +
          248  +/*
          249  +** Create rows mn through mx in table iTab for the given worker
          250  +*/
          251  +static void worker_add_content(WorkerInfo *p, int mn, int mx, int iTab){
          252  +  char *zTabDef;
          253  +  switch( iTab ){
          254  +    case 1:  zTabDef = "t1(tid,sp,a,b,c)";  break;
          255  +    case 2:  zTabDef = "t2(tid,sp,d,e,f)";  break;
          256  +    case 3:  zTabDef = "t3(tid,sp,x,y,z)";  break;
          257  +  }
          258  +  pthread_mutex_lock(p->pWrMutex);
          259  +  run_sql(p, 
          260  +     "WITH RECURSIVE\n"
          261  +     " c(i) AS (VALUES(%d) UNION ALL SELECT i+1 FROM c WHERE i<%d)\n"
          262  +     "INSERT INTO %s SELECT %d, zeroblob(3000), i, printf('%%d',i), i FROM c;",
          263  +     mn, mx, zTabDef, p->tid
          264  +  );
          265  +  pthread_mutex_unlock(p->pWrMutex);
          266  +  p->nTest++;
          267  +}
          268  +
          269  +/*
          270  +** Set an error message on a worker
          271  +*/
          272  +static void worker_error(WorkerInfo *p, const char *zFormat, ...){
          273  +  va_list ap;
          274  +  p->nErr++;
          275  +  sqlite3_free(p->zMsg);
          276  +  va_start(ap, zFormat);
          277  +  p->zMsg = sqlite3_vmprintf(zFormat, ap);
          278  +  va_end(ap);
          279  +}
          280  +
          281  +/*
          282  +** Each thread runs the following function.
          283  +*/
          284  +static void *worker_thread(void *pArg){
          285  +  WorkerInfo *p = (WorkerInfo*)pArg;
          286  +  int iOuter;
          287  +  int i;
          288  +  int rc;
          289  +  sqlite3_stmt *pStmt;
          290  +
          291  +  printf("worker %d startup\n", p->tid);  fflush(stdout);
          292  +  for(iOuter=1; iOuter<=p->nWorker; iOuter++){
          293  +    worker_open_connection(p, iOuter);
          294  +    for(i=0; i<4; i++){
          295  +      worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter)%3 + 1);
          296  +      worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter+1)%3 + 1);
          297  +      worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter+2)%3 + 1);
          298  +    }
          299  +
          300  +    pStmt = prep_sql(p->db, "SELECT count(a) FROM t1 WHERE tid=%d", p->tid);
          301  +    worker_trace(p, "query [%s]", sqlite3_sql(pStmt));
          302  +    rc = sqlite3_step(pStmt);
          303  +    if( rc!=SQLITE_ROW ){
          304  +      worker_error(p, "Failed to step: %s", sqlite3_sql(pStmt));
          305  +    }else if( sqlite3_column_int(pStmt, 0)!=400 ){
          306  +      worker_error(p, "Wrong result: %d", sqlite3_column_int(pStmt,0));
          307  +    }
          308  +    sqlite3_finalize(pStmt);
          309  +    if( p->nErr ) break;
          310  +
          311  +    if( ((iOuter+p->tid)%3)==0 ){
          312  +      sqlite3_db_release_memory(p->db);
          313  +      p->nTest++;
          314  +    }
          315  +
          316  +    pthread_mutex_lock(p->pWrMutex);
          317  +    run_sql(p, "BEGIN;");
          318  +    run_sql(p, "UPDATE t1 SET c=NULL WHERE a=55");
          319  +    run_sql(p, "UPDATE t2 SET f=NULL WHERE d=42");
          320  +    run_sql(p, "UPDATE t3 SET z=NULL WHERE x=31");
          321  +    run_sql(p, "ROLLBACK;");
          322  +    p->nTest++;
          323  +    pthread_mutex_unlock(p->pWrMutex);
          324  +
          325  +
          326  +    if( iOuter==p->tid ){
          327  +      pthread_mutex_lock(p->pWrMutex);
          328  +      run_sql(p, "VACUUM");
          329  +      pthread_mutex_unlock(p->pWrMutex);
          330  +    }
          331  +
          332  +    pStmt = prep_sql(p->db,
          333  +       "SELECT t1.rowid, t2.rowid, t3.rowid"
          334  +       "  FROM t1, t2, t3"
          335  +       " WHERE t1.tid=%d AND t2.tid=%d AND t3.tid=%d"
          336  +       "   AND t1.a<>t2.d AND t2.d<>t3.x"
          337  +       " ORDER BY 1, 2, 3"
          338  +       ,p->tid, p->tid, p->tid);
          339  +    worker_trace(p, "query [%s]", sqlite3_sql(pStmt));
          340  +    for(i=0; i<p->nWorker; i++){
          341  +      rc = sqlite3_step(pStmt);
          342  +      if( rc!=SQLITE_ROW ){
          343  +        worker_error(p, "Failed to step: %s", sqlite3_sql(pStmt));
          344  +        break;
          345  +      }
          346  +      sched_yield();
          347  +    }
          348  +    sqlite3_finalize(pStmt);
          349  +    if( p->nErr ) break;
          350  +
          351  +    worker_delete_all_content(p, (p->tid+iOuter)%2);
          352  +    worker_close_connection(p);
          353  +    p->db = 0;
          354  +  }
          355  +  worker_close_connection(p);
          356  +  printf("worker %d finished\n", p->tid); fflush(stdout);
          357  +  return 0;
          358  +}
          359  +
          360  +int main(int argc, char **argv){
          361  +  int nWorker = 0;         /* Number of worker threads */
          362  +  int i;                   /* Loop counter */
          363  +  WorkerInfo *aInfo;       /* Information for each worker */
          364  +  unsigned wkrFlags = 0;   /* Default worker flags */
          365  +  int nErr = 0;            /* Number of errors */
          366  +  int nTest = 0;           /* Number of tests */
          367  +  int rc;                  /* Return code */
          368  +  sqlite3 *db = 0;         /* Main database connection */
          369  +  pthread_mutex_t wrMutex; /* The write serialization mutex */
          370  +  WorkerInfo infoTop;      /* WorkerInfo for the main thread */
          371  +  WorkerInfo *p;           /* Pointer to infoTop */
          372  +
          373  +  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
          374  +  for(i=1; i<argc; i++){
          375  +    const char *z = argv[i];
          376  +    if( z[0]=='-' ){
          377  +      if( z[1]=='-' && z[2]!=0 ) z++;
          378  +      if( strcmp(z,"-multithread")==0 ){
          379  +        sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
          380  +        wkrFlags &= ~TT4_SERIALIZED;
          381  +      }else if( strcmp(z,"-serialized")==0 ){
          382  +        sqlite3_config(SQLITE_CONFIG_SERIALIZED);
          383  +        wkrFlags |= TT4_SERIALIZED;
          384  +      }else if( strcmp(z,"-wal")==0 ){
          385  +        wkrFlags |= TT4_WAL;
          386  +      }else if( strcmp(z,"-trace")==0 ){
          387  +        wkrFlags |= TT4_TRACE;
          388  +      }else{
          389  +        fprintf(stderr, "unknown command-line option: %s\n", argv[i]);
          390  +        exit(1);
          391  +      }
          392  +    }else if( z[0]>='1' && z[0]<='9' && nWorker==0 ){
          393  +      nWorker = atoi(z);
          394  +      if( nWorker<2 ){
          395  +        fprintf(stderr, "minimum of 2 threads\n");
          396  +        exit(1);
          397  +      }
          398  +    }else{
          399  +      fprintf(stderr, "extra command-line argument: \"%s\"\n", argv[i]);
          400  +      exit(1);
          401  +    }
          402  +  }
          403  +  if( nWorker==0 ){ 
          404  +    fprintf(stderr,
          405  +       "usage:  %s ?OPTIONS? N\n"
          406  +       "N is the number of threads and must be at least 2.\n"
          407  +       "Options:\n"
          408  +       "  --serialized\n"
          409  +       "  --multithread\n"
          410  +       "  --wal\n"
          411  +       "  --trace\n"
          412  +       ,argv[0]
          413  +    );
          414  +    exit(1);
          415  +  }
          416  +  if( !sqlite3_threadsafe() ){
          417  +    fprintf(stderr, "requires a threadsafe build of SQLite\n");
          418  +    exit(1);
          419  +  }
          420  +  sqlite3_initialize();
          421  +  sqlite3_enable_shared_cache(1);
          422  +  pthread_mutex_init(&wrMutex, 0);
          423  +
          424  +  /* Initialize the test database files */
          425  +  (void)unlink("tt4-test1.db");
          426  +  (void)unlink("tt4-test2.db");
          427  +  (void)unlink("tt4-test3.db");
          428  +  rc = sqlite3_open("tt4-test1.db", &db);
          429  +  if( rc!=SQLITE_OK ){
          430  +    fprintf(stderr, "Unable to open test database: tt4-test2.db\n");
          431  +    exit(1);
          432  +  }
          433  +  memset(&infoTop, 0, sizeof(infoTop));
          434  +  infoTop.db = db;
          435  +  infoTop.wkrFlags = wkrFlags;
          436  +  p = &infoTop;
          437  +  if( wkrFlags & TT4_WAL ){
          438  +    run_sql(p, "PRAGMA journal_mode=WAL");
          439  +  }
          440  +  run_sql(p, "PRAGMA synchronous=OFF");
          441  +  run_sql(p, "CREATE TABLE IF NOT EXISTS t1(tid INTEGER, sp, a, b, c)");
          442  +  run_sql(p, "CREATE INDEX t1tid ON t1(tid)");
          443  +  run_sql(p, "CREATE INDEX t1ab ON t1(a,b)");
          444  +  run_sql(p, "ATTACH 'tt4-test2.db' AS 'test2'");
          445  +  run_sql(p, "CREATE TABLE IF NOT EXISTS test2.t2(tid INTEGER, sp, d, e, f)");
          446  +  run_sql(p, "CREATE INDEX test2.t2tid ON t2(tid)");
          447  +  run_sql(p, "CREATE INDEX test2.t2de ON t2(d,e)");
          448  +  run_sql(p, "ATTACH 'tt4-test3.db' AS 'test3'");
          449  +  run_sql(p, "CREATE TABLE IF NOT EXISTS test3.t3(tid INTEGER, sp, x, y, z)");
          450  +  run_sql(p, "CREATE INDEX test3.t3tid ON t3(tid)");
          451  +  run_sql(p, "CREATE INDEX test3.t3xy ON t3(x,y)");
          452  +  aInfo = safe_malloc( sizeof(*aInfo)*nWorker );
          453  +  memset(aInfo, 0, sizeof(*aInfo)*nWorker);
          454  +  for(i=0; i<nWorker; i++){
          455  +    aInfo[i].tid = i+1;
          456  +    aInfo[i].nWorker = nWorker;
          457  +    aInfo[i].wkrFlags = wkrFlags;
          458  +    aInfo[i].mainDb = db;
          459  +    aInfo[i].pWrMutex = &wrMutex;
          460  +    rc = pthread_create(&aInfo[i].id, 0, worker_thread, &aInfo[i]);
          461  +    if( rc!=0 ){
          462  +      fprintf(stderr, "thread creation failed for thread %d\n", i+1);
          463  +      exit(1);
          464  +    }
          465  +    sched_yield();
          466  +  }
          467  +  for(i=0; i<nWorker; i++){
          468  +    pthread_join(aInfo[i].id, 0);
          469  +    printf("Joined thread %d: %d errors in %d tests",
          470  +           aInfo[i].tid, aInfo[i].nErr, aInfo[i].nTest);
          471  +    if( aInfo[i].zMsg ){
          472  +      printf(": %s\n", aInfo[i].zMsg);
          473  +    }else{
          474  +      printf("\n");
          475  +    }
          476  +    nErr += aInfo[i].nErr;
          477  +    nTest += aInfo[i].nTest;
          478  +    fflush(stdout);
          479  +  }
          480  +  sqlite3_close(db);
          481  +  sqlite3_free(aInfo);
          482  +  printf("Total %d errors in %d tests\n", nErr, nTest);
          483  +  return nErr;
          484  +}

Changes to test/tt3_checkpoint.c.

    62     62     }
    63     63     if( nFrame>=CHECKPOINT_STARVATION_FRAMELIMIT ){
    64     64       sqlite3_wal_checkpoint_v2(db, zDb, p->eMode, 0, 0);
    65     65     }
    66     66     return SQLITE_OK;
    67     67   }
    68     68   
    69         -static char *checkpoint_starvation_reader(int iTid, int iArg){
           69  +static char *checkpoint_starvation_reader(int iTid, void *pArg){
    70     70     Error err = {0};
    71     71     Sqlite db = {0};
    72     72   
    73     73     opendb(&err, &db, "test.db", 0);
    74     74     while( !timetostop(&err) ){
    75     75       i64 iCount1, iCount2;
    76     76       sql_script(&err, &db, "BEGIN");

Changes to test/tt3_index.c.

    10     10   **
    11     11   *************************************************************************
    12     12   **
    13     13   **     create_drop_index_1
    14     14   */
    15     15   
    16     16   
    17         -static char *create_drop_index_thread(int iTid, int iArg){
           17  +static char *create_drop_index_thread(int iTid, void *pArg){
    18     18     Error err = {0};                /* Error code and message */
    19     19     Sqlite db = {0};                /* SQLite database connection */
    20     20   
    21     21     while( !timetostop(&err) ){
    22     22       opendb(&err, &db, "test.db", 0);
    23     23   
    24     24       sql_script(&err, &db, 
    25         -
    26     25         "DROP INDEX IF EXISTS i1;"
    27     26         "DROP INDEX IF EXISTS i2;"
    28     27         "DROP INDEX IF EXISTS i3;"
    29     28         "DROP INDEX IF EXISTS i4;"
    30     29   
    31         -      "CREATE INDEX IF NOT EXISTS i1 ON t1(a);"
    32         -      "CREATE INDEX IF NOT EXISTS i2 ON t1(b);"
    33         -      "CREATE INDEX IF NOT EXISTS i3 ON t1(c);"
    34         -      "CREATE INDEX IF NOT EXISTS i4 ON t1(d);"
           30  +      "CREATE INDEX IF NOT EXISTS i1 ON t11(a);"
           31  +      "CREATE INDEX IF NOT EXISTS i2 ON t11(b);"
           32  +      "CREATE INDEX IF NOT EXISTS i3 ON t11(c);"
           33  +      "CREATE INDEX IF NOT EXISTS i4 ON t11(d);"
    35     34   
    36         -      "SELECT * FROM t1 ORDER BY a;"
    37         -      "SELECT * FROM t1 ORDER BY b;"
    38         -      "SELECT * FROM t1 ORDER BY c;"
    39         -      "SELECT * FROM t1 ORDER BY d;"
           35  +      "SELECT * FROM t11 ORDER BY a;"
           36  +      "SELECT * FROM t11 ORDER BY b;"
           37  +      "SELECT * FROM t11 ORDER BY c;"
           38  +      "SELECT * FROM t11 ORDER BY d;"
    40     39       );
           40  +    clear_error(&err, SQLITE_LOCKED);
    41     41   
    42     42       closedb(&err, &db);
    43     43     }
    44     44   
    45     45     print_and_free_err(&err);
    46     46     return sqlite3_mprintf("ok");
    47     47   }
................................................................................
    49     49   static void create_drop_index_1(int nMs){
    50     50     Error err = {0};
    51     51     Sqlite db = {0};
    52     52     Threadset threads = {0};
    53     53   
    54     54     opendb(&err, &db, "test.db", 1);
    55     55     sql_script(&err, &db, 
    56         -     "CREATE TABLE t1(a, b, c, d);"
           56  +     "CREATE TABLE t11(a, b, c, d);"
    57     57        "WITH data(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM data WHERE x<100) "
    58         -     "INSERT INTO t1 SELECT x,x,x,x FROM data;"
           58  +     "INSERT INTO t11 SELECT x,x,x,x FROM data;"
    59     59     );
    60     60     closedb(&err, &db);
    61     61   
    62     62     setstoptime(&err, nMs);
    63     63   
    64     64     sqlite3_enable_shared_cache(1);
    65     65     launch_thread(&err, &threads, create_drop_index_thread, 0);
    66     66     launch_thread(&err, &threads, create_drop_index_thread, 0);
    67     67     launch_thread(&err, &threads, create_drop_index_thread, 0);
    68     68     launch_thread(&err, &threads, create_drop_index_thread, 0);
    69     69     launch_thread(&err, &threads, create_drop_index_thread, 0);
    70         -  sqlite3_enable_shared_cache(0);
    71     70   
    72     71     join_all_threads(&err, &threads);
           72  +  sqlite3_enable_shared_cache(0);
    73     73     print_and_free_err(&err);
    74     74   }

Added test/tt3_lookaside1.c.

            1  +/*
            2  +** 2014 December 9
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +**     lookaside1
           14  +*/
           15  +
           16  +/*
           17  +** The test in this file attempts to expose a specific race condition
           18  +** that is suspected to exist at time of writing.
           19  +*/
           20  +
           21  +static char *lookaside1_thread_reader(int iTid, void *pArg){
           22  +  Error err = {0};                /* Error code and message */
           23  +  Sqlite db = {0};                /* SQLite database connection */
           24  +
           25  +  opendb(&err, &db, "test.db", 0);
           26  +
           27  +  while( !timetostop(&err) ){
           28  +    sqlite3_stmt *pStmt = 0;
           29  +    int rc;
           30  +
           31  +    sqlite3_prepare_v2(db.db, "SELECT 1 FROM t1", -1, &pStmt, 0);
           32  +    while( sqlite3_step(pStmt)==SQLITE_ROW ){
           33  +      execsql(&err, &db, "SELECT length(x||y||z) FROM t2");
           34  +    }
           35  +    rc = sqlite3_finalize(pStmt);
           36  +    if( err.rc==SQLITE_OK && rc!=SQLITE_OK ){
           37  +      sqlite_error(&err, &db, "finalize");
           38  +    }
           39  +  }
           40  +
           41  +  closedb(&err, &db);
           42  +  print_and_free_err(&err);
           43  +  return sqlite3_mprintf("ok");
           44  +}
           45  +
           46  +static char *lookaside1_thread_writer(int iTid, void *pArg){
           47  +  Error err = {0};                /* Error code and message */
           48  +  Sqlite db = {0};                /* SQLite database connection */
           49  +
           50  +  opendb(&err, &db, "test.db", 0);
           51  +
           52  +  do{
           53  +    sql_script(&err, &db, 
           54  +      "BEGIN;"
           55  +        "UPDATE t3 SET i=i+1 WHERE x=1;"
           56  +      "ROLLBACK;"
           57  +    );
           58  +  }while( !timetostop(&err) );
           59  +
           60  +  closedb(&err, &db);
           61  +  print_and_free_err(&err);
           62  +  return sqlite3_mprintf("ok");
           63  +}
           64  +
           65  +
           66  +static void lookaside1(int nMs){
           67  +  Error err = {0};
           68  +  Sqlite db = {0};
           69  +  Threadset threads = {0};
           70  +
           71  +  opendb(&err, &db, "test.db", 1);
           72  +  sql_script(&err, &db, 
           73  +     "CREATE TABLE t1(x PRIMARY KEY) WITHOUT ROWID;"
           74  +     "WITH data(x,y) AS ("
           75  +     "  SELECT 1, quote(randomblob(750)) UNION ALL "
           76  +     "  SELECT x*2, y||y FROM data WHERE x<5) "
           77  +     "INSERT INTO t1 SELECT y FROM data;"
           78  +
           79  +     "CREATE TABLE t3(x PRIMARY KEY,i) WITHOUT ROWID;"
           80  +     "INSERT INTO t3 VALUES(1, 1);"
           81  +
           82  +     "CREATE TABLE t2(x,y,z);"
           83  +     "INSERT INTO t2 VALUES(randomblob(50), randomblob(50), randomblob(50));"
           84  +  );
           85  +  closedb(&err, &db);
           86  +
           87  +  setstoptime(&err, nMs);
           88  +
           89  +  sqlite3_enable_shared_cache(1);
           90  +  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
           91  +  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
           92  +  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
           93  +  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
           94  +  launch_thread(&err, &threads, lookaside1_thread_reader, 0);
           95  +  launch_thread(&err, &threads, lookaside1_thread_writer, 0);
           96  +  join_all_threads(&err, &threads);
           97  +  sqlite3_enable_shared_cache(0);
           98  +  print_and_free_err(&err);
           99  +}

Added test/tt3_stress.c.

            1  +/*
            2  +** 2014 December 9
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +**
           14  +*/
           15  +
           16  +
           17  +/*
           18  +** Thread 1. CREATE and DROP a table.
           19  +*/
           20  +static char *stress_thread_1(int iTid, void *pArg){
           21  +  Error err = {0};                /* Error code and message */
           22  +  Sqlite db = {0};                /* SQLite database connection */
           23  +
           24  +  opendb(&err, &db, "test.db", 0);
           25  +  while( !timetostop(&err) ){
           26  +    sql_script(&err, &db, "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b)");
           27  +    clear_error(&err, SQLITE_LOCKED);
           28  +    sql_script(&err, &db, "DROP TABLE IF EXISTS t1");
           29  +    clear_error(&err, SQLITE_LOCKED);
           30  +  }
           31  +  closedb(&err, &db);
           32  +  print_and_free_err(&err);
           33  +  return sqlite3_mprintf("ok");
           34  +}
           35  +
           36  +/*
           37  +** Thread 2. Open and close database connections.
           38  +*/
           39  +static char *stress_thread_2(int iTid, void *pArg){
           40  +  Error err = {0};                /* Error code and message */
           41  +  Sqlite db = {0};                /* SQLite database connection */
           42  +  while( !timetostop(&err) ){
           43  +    opendb(&err, &db, "test.db", 0);
           44  +    sql_script(&err, &db, "SELECT * FROM sqlite_master;");
           45  +    clear_error(&err, SQLITE_LOCKED);
           46  +    closedb(&err, &db);
           47  +  }
           48  +  print_and_free_err(&err);
           49  +  return sqlite3_mprintf("ok");
           50  +}
           51  +
           52  +/*
           53  +** Thread 3. Attempt many small SELECT statements.
           54  +*/
           55  +static char *stress_thread_3(int iTid, void *pArg){
           56  +  Error err = {0};                /* Error code and message */
           57  +  Sqlite db = {0};                /* SQLite database connection */
           58  +
           59  +  int i1 = 0;
           60  +  int i2 = 0;
           61  +
           62  +  opendb(&err, &db, "test.db", 0);
           63  +  while( !timetostop(&err) ){
           64  +    sql_script(&err, &db, "SELECT * FROM t1 ORDER BY a;");
           65  +    i1++;
           66  +    if( err.rc ) i2++;
           67  +    clear_error(&err, SQLITE_LOCKED);
           68  +    clear_error(&err, SQLITE_ERROR);
           69  +  }
           70  +  closedb(&err, &db);
           71  +  print_and_free_err(&err);
           72  +  return sqlite3_mprintf("read t1 %d/%d attempts", i2, i1);
           73  +}
           74  +
           75  +/*
           76  +** Thread 5. Attempt INSERT statements.
           77  +*/
           78  +static char *stress_thread_4(int iTid, void *pArg){
           79  +  Error err = {0};                /* Error code and message */
           80  +  Sqlite db = {0};                /* SQLite database connection */
           81  +  int i1 = 0;
           82  +  int i2 = 0;
           83  +  int iArg = PTR2INT(pArg);
           84  +
           85  +  opendb(&err, &db, "test.db", 0);
           86  +  while( !timetostop(&err) ){
           87  +    if( iArg ){
           88  +      closedb(&err, &db);
           89  +      opendb(&err, &db, "test.db", 0);
           90  +    }
           91  +    sql_script(&err, &db, 
           92  +        "WITH loop(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM loop LIMIT 200) "
           93  +        "INSERT INTO t1 VALUES(randomblob(60), randomblob(60));"
           94  +    );
           95  +    i1++;
           96  +    if( err.rc ) i2++;
           97  +    clear_error(&err, SQLITE_LOCKED);
           98  +    clear_error(&err, SQLITE_ERROR);
           99  +  }
          100  +  closedb(&err, &db);
          101  +  print_and_free_err(&err);
          102  +  return sqlite3_mprintf("wrote t1 %d/%d attempts", i2, i1);
          103  +}
          104  +
          105  +/*
          106  +** Thread 6. Attempt DELETE operations.
          107  +*/
          108  +static char *stress_thread_5(int iTid, void *pArg){
          109  +  Error err = {0};                /* Error code and message */
          110  +  Sqlite db = {0};                /* SQLite database connection */
          111  +  int iArg = PTR2INT(pArg);
          112  +
          113  +  int i1 = 0;
          114  +  int i2 = 0;
          115  +
          116  +  opendb(&err, &db, "test.db", 0);
          117  +  while( !timetostop(&err) ){
          118  +    i64 i = (i1 % 4);
          119  +    if( iArg ){
          120  +      closedb(&err, &db);
          121  +      opendb(&err, &db, "test.db", 0);
          122  +    }
          123  +    execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i);
          124  +    i1++;
          125  +    if( err.rc ) i2++;
          126  +    clear_error(&err, SQLITE_LOCKED);
          127  +  }
          128  +  closedb(&err, &db);
          129  +  print_and_free_err(&err);
          130  +  return sqlite3_mprintf("deleted from t1 %d/%d attempts", i2, i1);
          131  +}
          132  +
          133  +
          134  +static void stress1(int nMs){
          135  +  Error err = {0};
          136  +  Threadset threads = {0};
          137  +
          138  +  setstoptime(&err, nMs);
          139  +  sqlite3_enable_shared_cache(1);
          140  +
          141  +  launch_thread(&err, &threads, stress_thread_1, 0);
          142  +  launch_thread(&err, &threads, stress_thread_1, 0);
          143  +
          144  +  launch_thread(&err, &threads, stress_thread_2, 0);
          145  +  launch_thread(&err, &threads, stress_thread_2, 0);
          146  +
          147  +  launch_thread(&err, &threads, stress_thread_3, 0);
          148  +  launch_thread(&err, &threads, stress_thread_3, 0);
          149  +
          150  +  launch_thread(&err, &threads, stress_thread_4, 0);
          151  +  launch_thread(&err, &threads, stress_thread_4, 0);
          152  +
          153  +  launch_thread(&err, &threads, stress_thread_5, 0);
          154  +  launch_thread(&err, &threads, stress_thread_5, (void*)1);
          155  +
          156  +  join_all_threads(&err, &threads);
          157  +  sqlite3_enable_shared_cache(0);
          158  +
          159  +  print_and_free_err(&err);
          160  +}
          161  +
          162  +/**************************************************************************
          163  +***************************************************************************
          164  +** Start of test case "stress2"
          165  +*/
          166  +
          167  +
          168  +
          169  +/*
          170  +** 1.  CREATE TABLE statements.
          171  +** 2.  DROP TABLE statements.
          172  +** 3.  Small SELECT statements.
          173  +** 4.  Big SELECT statements.
          174  +** 5.  Small INSERT statements.
          175  +** 6.  Big INSERT statements.
          176  +** 7.  Small UPDATE statements.
          177  +** 8.  Big UPDATE statements.
          178  +** 9.  Small DELETE statements.
          179  +** 10. Big DELETE statements.
          180  +** 11. VACUUM.
          181  +** 14. Integrity-check.
          182  +** 17. Switch the journal mode from delete to wal and back again.
          183  +** 19. Open and close database connections rapidly.
          184  +*/
          185  +
          186  +#define STRESS2_TABCNT 5          /* count1 in SDS test */
          187  +
          188  +#define STRESS2_COUNT2 200        /* count2 in SDS test */
          189  +#define STRESS2_COUNT3  57        /* count2 in SDS test */
          190  +
          191  +static void stress2_workload1(Error *pErr, Sqlite *pDb, int i){
          192  +  int iTab = (i % (STRESS2_TABCNT-1)) + 1;
          193  +  sql_script_printf(pErr, pDb, 
          194  +      "CREATE TABLE IF NOT EXISTS t%d(x PRIMARY KEY, y, z);", iTab
          195  +  );
          196  +}
          197  +
          198  +static void stress2_workload2(Error *pErr, Sqlite *pDb, int i){
          199  +  int iTab = (i % (STRESS2_TABCNT-1)) + 1;
          200  +  sql_script_printf(pErr, pDb, "DROP TABLE IF EXISTS t%d;", iTab);
          201  +}
          202  +
          203  +static void stress2_workload3(Error *pErr, Sqlite *pDb, int i){
          204  +  sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'small'");
          205  +}
          206  +
          207  +static void stress2_workload4(Error *pErr, Sqlite *pDb, int i){
          208  +  sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'big'");
          209  +}
          210  +
          211  +static void stress2_workload5(Error *pErr, Sqlite *pDb, int i){
          212  +  sql_script(pErr, pDb,
          213  +      "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(200)), 'small');"
          214  +  );
          215  +}
          216  +
          217  +static void stress2_workload6(Error *pErr, Sqlite *pDb, int i){
          218  +  sql_script(pErr, pDb,
          219  +      "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(57)), 'big');"
          220  +  );
          221  +}
          222  +
          223  +static void stress2_workload7(Error *pErr, Sqlite *pDb, int i){
          224  +  sql_script_printf(pErr, pDb,
          225  +      "UPDATE t0 SET y = hex(randomblob(200)) "
          226  +      "WHERE x LIKE hex((%d %% 5)) AND z='small';"
          227  +      ,i
          228  +  );
          229  +}
          230  +static void stress2_workload8(Error *pErr, Sqlite *pDb, int i){
          231  +  sql_script_printf(pErr, pDb,
          232  +      "UPDATE t0 SET y = hex(randomblob(57)) "
          233  +      "WHERE x LIKE hex(%d %% 5) AND z='big';"
          234  +      ,i
          235  +  );
          236  +}
          237  +
          238  +static void stress2_workload9(Error *pErr, Sqlite *pDb, int i){
          239  +  sql_script_printf(pErr, pDb,
          240  +      "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='small';", i
          241  +  );
          242  +}
          243  +static void stress2_workload10(Error *pErr, Sqlite *pDb, int i){
          244  +  sql_script_printf(pErr, pDb,
          245  +      "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='big';", i
          246  +  );
          247  +}
          248  +
          249  +static void stress2_workload11(Error *pErr, Sqlite *pDb, int i){
          250  +  sql_script(pErr, pDb, "VACUUM");
          251  +}
          252  +
          253  +static void stress2_workload14(Error *pErr, Sqlite *pDb, int i){
          254  +  sql_script(pErr, pDb, "PRAGMA integrity_check");
          255  +}
          256  +
          257  +static void stress2_workload17(Error *pErr, Sqlite *pDb, int i){
          258  +  sql_script_printf(pErr, pDb, 
          259  +      "PRAGMA journal_mode = %q", (i%2) ? "delete" : "wal"
          260  +  );
          261  +}
          262  +
          263  +static char *stress2_workload19(int iTid, void *pArg){
          264  +  Error err = {0};                /* Error code and message */
          265  +  Sqlite db = {0};                /* SQLite database connection */
          266  +  const char *zDb = (const char*)pArg;
          267  +  while( !timetostop(&err) ){
          268  +    opendb(&err, &db, zDb, 0);
          269  +    sql_script(&err, &db, "SELECT * FROM sqlite_master;");
          270  +    clear_error(&err, SQLITE_LOCKED);
          271  +    closedb(&err, &db);
          272  +  }
          273  +  print_and_free_err(&err);
          274  +  return sqlite3_mprintf("ok");
          275  +}
          276  +
          277  +
          278  +typedef struct Stress2Ctx Stress2Ctx;
          279  +struct Stress2Ctx {
          280  +  const char *zDb;
          281  +  void (*xProc)(Error*, Sqlite*, int);
          282  +};
          283  +
          284  +static char *stress2_thread_wrapper(int iTid, void *pArg){
          285  +  Stress2Ctx *pCtx = (Stress2Ctx*)pArg;
          286  +  Error err = {0};                /* Error code and message */
          287  +  Sqlite db = {0};                /* SQLite database connection */
          288  +  int i1 = 0;
          289  +  int i2 = 0;
          290  +
          291  +  while( !timetostop(&err) ){
          292  +    int cnt;
          293  +    opendb(&err, &db, pCtx->zDb, 0);
          294  +    for(cnt=0; err.rc==SQLITE_OK && cnt<STRESS2_TABCNT; cnt++){
          295  +      pCtx->xProc(&err, &db, i1);
          296  +      i2 += (err.rc==SQLITE_OK);
          297  +      clear_error(&err, SQLITE_LOCKED);
          298  +      i1++;
          299  +    }
          300  +    closedb(&err, &db);
          301  +  }
          302  +
          303  +  print_and_free_err(&err);
          304  +  return sqlite3_mprintf("ok %d/%d", i2, i1);
          305  +}
          306  +
          307  +static void stress2_launch_thread_loop(
          308  +  Error *pErr,                    /* IN/OUT: Error code */
          309  +  Threadset *pThreads,            /* Thread set */
          310  +  const char *zDb,                /* Database name */
          311  +  void (*x)(Error*,Sqlite*,int)   /* Run this until error or timeout */
          312  +){
          313  +  Stress2Ctx *pCtx = sqlite3_malloc(sizeof(Stress2Ctx));
          314  +  pCtx->zDb = zDb;
          315  +  pCtx->xProc = x;
          316  +  launch_thread(pErr, pThreads, stress2_thread_wrapper, (void*)pCtx);
          317  +}
          318  +
          319  +static void stress2(int nMs){
          320  +  struct Stress2Task {
          321  +    void (*x)(Error*,Sqlite*,int);
          322  +  } aTask[] = {
          323  +    { stress2_workload1 },
          324  +    { stress2_workload2 },
          325  +    { stress2_workload3 },
          326  +    { stress2_workload4 },
          327  +    { stress2_workload5 },
          328  +    { stress2_workload6 },
          329  +    { stress2_workload7 },
          330  +    { stress2_workload8 },
          331  +    { stress2_workload9 },
          332  +    { stress2_workload10 },
          333  +    { stress2_workload11 },
          334  +    { stress2_workload14 },
          335  +    { stress2_workload17 },
          336  +  };
          337  +  const char *zDb = "test.db";
          338  +
          339  +  int i;
          340  +  Error err = {0};
          341  +  Sqlite db = {0};
          342  +  Threadset threads = {0};
          343  +
          344  +  /* To make sure the db file is empty before commencing */
          345  +  opendb(&err, &db, zDb, 1);
          346  +  sql_script(&err, &db, 
          347  +      "CREATE TABLE IF NOT EXISTS t0(x PRIMARY KEY, y, z);"
          348  +      "CREATE INDEX IF NOT EXISTS i0 ON t0(y);"
          349  +  );
          350  +  closedb(&err, &db);
          351  +
          352  +  setstoptime(&err, nMs);
          353  +  sqlite3_enable_shared_cache(1);
          354  +
          355  +  for(i=0; i<sizeof(aTask)/sizeof(aTask[0]); i++){
          356  +    stress2_launch_thread_loop(&err, &threads, zDb, aTask[i].x);
          357  +  }
          358  +  launch_thread(&err, &threads, stress2_workload19, (void*)zDb);
          359  +  launch_thread(&err, &threads, stress2_workload19, (void*)zDb);
          360  +
          361  +  join_all_threads(&err, &threads);
          362  +  sqlite3_enable_shared_cache(0);
          363  +  print_and_free_err(&err);
          364  +}
          365  +
          366  +
          367  +
          368  +

Added test/tt3_vacuum.c.

            1  +/*
            2  +** 2014 December 9
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file contains multi-threaded tests that use shared-cache and 
           14  +** the VACUUM command.
           15  +**
           16  +** Tests:
           17  +**
           18  +**     vacuum1
           19  +**
           20  +*/
           21  +
           22  +
           23  +static char *vacuum1_thread_writer(int iTid, void *pArg){
           24  +  Error err = {0};                /* Error code and message */
           25  +  Sqlite db = {0};                /* SQLite database connection */
           26  +  opendb(&err, &db, "test.db", 0);
           27  +  i64 i = 0;
           28  +
           29  +  while( !timetostop(&err) ){
           30  +    i++;
           31  +
           32  +    /* Insert lots of rows. Then delete some. */
           33  +    execsql(&err, &db, 
           34  +        "WITH loop(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM loop WHERE i<100) "
           35  +        "INSERT INTO t1 SELECT randomblob(50), randomblob(2500) FROM loop"
           36  +    );
           37  +
           38  +    /* Delete lots of rows */
           39  +    execsql(&err, &db, "DELETE FROM t1 WHERE rowid = :i", &i);
           40  +    clear_error(&err, SQLITE_LOCKED);
           41  +
           42  +    /* Select the rows */
           43  +    execsql(&err, &db, "SELECT * FROM t1 ORDER BY x");
           44  +    clear_error(&err, SQLITE_LOCKED);
           45  +  }
           46  +
           47  +  closedb(&err, &db);
           48  +  print_and_free_err(&err);
           49  +  return sqlite3_mprintf("ok");
           50  +}
           51  +
           52  +static char *vacuum1_thread_vacuumer(int iTid, void *pArg){
           53  +  Error err = {0};                /* Error code and message */
           54  +  Sqlite db = {0};                /* SQLite database connection */
           55  +  opendb(&err, &db, "test.db", 0);
           56  +
           57  +  do{
           58  +    sql_script(&err, &db, "VACUUM");
           59  +    clear_error(&err, SQLITE_LOCKED);
           60  +  }while( !timetostop(&err) );
           61  +
           62  +  closedb(&err, &db);
           63  +  print_and_free_err(&err);
           64  +  return sqlite3_mprintf("ok");
           65  +}
           66  +
           67  +static void vacuum1(int nMs){
           68  +  Error err = {0};
           69  +  Sqlite db = {0};
           70  +  Threadset threads = {0};
           71  +
           72  +  opendb(&err, &db, "test.db", 1);
           73  +  sql_script(&err, &db, 
           74  +     "CREATE TABLE t1(x PRIMARY KEY, y BLOB);"
           75  +     "CREATE INDEX i1 ON t1(y);"
           76  +  );
           77  +  closedb(&err, &db);
           78  +
           79  +  setstoptime(&err, nMs);
           80  +
           81  +  sqlite3_enable_shared_cache(1);
           82  +  launch_thread(&err, &threads, vacuum1_thread_writer, 0);
           83  +  launch_thread(&err, &threads, vacuum1_thread_writer, 0);
           84  +  launch_thread(&err, &threads, vacuum1_thread_writer, 0);
           85  +  launch_thread(&err, &threads, vacuum1_thread_vacuumer, 0);
           86  +  join_all_threads(&err, &threads);
           87  +  sqlite3_enable_shared_cache(0);
           88  +
           89  +  print_and_free_err(&err);
           90  +}