/ Check-in [7f48f6ab]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge in all the latest trunk enhancements.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA3-256: 7f48f6ab2af071ec20204105ed703c60e9fa92602558b910d14b09cc634bed7d
User & Date: drh 2017-07-07 22:58:12
Context
2017-07-17
20:21
Merge all the latest fixes and enhancements from trunk. check-in: e181225d user: drh tags: apple-osx
2017-07-07
22:58
Merge in all the latest trunk enhancements. check-in: 7f48f6ab user: drh tags: apple-osx
22:47
Basic test cases for PRAGMA secure_delete=FAST. check-in: 0c246017 user: drh tags: trunk
22:39
Fix test cases in wal2.test due to the fact that Darwin checks to see if the SHM file is writable before attempting to open in read/write. check-in: 6b618b18 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   177    177            icu.lo insert.lo json1.lo legacy.lo loadext.lo \
   178    178            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   179    179            memjournal.lo \
   180    180            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   181    181            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   182    182            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   183    183            random.lo resolve.lo rowset.lo rtree.lo \
   184         -         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
          184  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
   185    185            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   186    186            update.lo util.lo vacuum.lo \
   187    187            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   188    188            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   189    189            utf.lo vtab.lo
   190    190   
   191    191   # Object files for the amalgamation.
................................................................................
   349    349   SRC += \
   350    350     $(TOP)/ext/session/sqlite3session.c \
   351    351     $(TOP)/ext/session/sqlite3session.h
   352    352   SRC += \
   353    353     $(TOP)/ext/rbu/sqlite3rbu.h \
   354    354     $(TOP)/ext/rbu/sqlite3rbu.c
   355    355   SRC += \
   356         -  $(TOP)/ext/misc/json1.c
          356  +  $(TOP)/ext/misc/json1.c \
          357  +  $(TOP)/ext/misc/stmt.c
   357    358   
   358    359   # Generated source code files
   359    360   #
   360    361   SRC += \
   361    362     keywordhash.h \
   362    363     opcodes.c \
   363    364     opcodes.h \
................................................................................
   574    575   
   575    576   # Extra compiler options for various shell tools
   576    577   #
   577    578   SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
   578    579   # SHELL_OPT += -DSQLITE_ENABLE_FTS5
   579    580   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   580    581   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
          582  +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   581    583   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   582    584   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
   583    585   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
   584    586   FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
   585    587   DBFUZZ_OPT = 
   586    588   
   587    589   # This is the default Makefile target.  The objects listed here
................................................................................
  1043   1045   
  1044   1046   sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
  1045   1047   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
  1046   1048   
  1047   1049   json1.lo:	$(TOP)/ext/misc/json1.c
  1048   1050   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
  1049   1051   
         1052  +stmt.lo:	$(TOP)/ext/misc/stmt.c
         1053  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
         1054  +
  1050   1055   # FTS5 things
  1051   1056   #
  1052   1057   FTS5_SRC = \
  1053   1058      $(TOP)/ext/fts5/fts5.h \
  1054   1059      $(TOP)/ext/fts5/fts5Int.h \
  1055   1060      $(TOP)/ext/fts5/fts5_aux.c \
  1056   1061      $(TOP)/ext/fts5/fts5_buffer.c \
................................................................................
  1092   1097   # hidden when the library is built via the amalgamation).
  1093   1098   #
  1094   1099   TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1095   1100   TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
  1096   1101   TESTFIXTURE_FLAGS += -DBUILD_sqlite
  1097   1102   TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  1098   1103   TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
         1104  +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
  1099   1105   
  1100   1106   TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
  1101   1107   TESTFIXTURE_SRC1 = sqlite3.c
  1102   1108   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
  1103   1109   TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
  1104   1110   
  1105   1111   testfixture$(TEXE):	$(TESTFIXTURE_SRC)

Changes to Makefile.msc.

  1290   1290     $(TOP)\ext\fts3\fts3_unicode.c \
  1291   1291     $(TOP)\ext\fts3\fts3_unicode2.c \
  1292   1292     $(TOP)\ext\fts3\fts3_write.c \
  1293   1293     $(TOP)\ext\icu\icu.c \
  1294   1294     $(TOP)\ext\rtree\rtree.c \
  1295   1295     $(TOP)\ext\session\sqlite3session.c \
  1296   1296     $(TOP)\ext\rbu\sqlite3rbu.c \
  1297         -  $(TOP)\ext\misc\json1.c
         1297  +  $(TOP)\ext\misc\json1.c \
         1298  +  $(TOP)\ext\misc\stmt.c
  1298   1299   
  1299   1300   # Extension header files, part 1.
  1300   1301   #
  1301   1302   SRC08 = \
  1302   1303     $(TOP)\ext\fts1\fts1.h \
  1303   1304     $(TOP)\ext\fts1\fts1_hash.h \
  1304   1305     $(TOP)\ext\fts1\fts1_tokenizer.h \
................................................................................
  1501   1502     $(TOP)\test\fuzzdata5.db
  1502   1503   # <</mark>>
  1503   1504   
  1504   1505   # Additional compiler options for the shell.  These are only effective
  1505   1506   # when the shell is not being dynamically linked.
  1506   1507   #
  1507   1508   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1508         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4
         1509  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
  1509   1510   # SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS5
  1510   1511   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
  1511   1512   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
  1512   1513   !ENDIF
  1513   1514   
  1514   1515   # <<mark>>
  1515   1516   # Extra compiler options for various test tools.
................................................................................
  2088   2089   # hidden when the library is built via the amalgamation).
  2089   2090   #
  2090   2091   TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  2091   2092   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  2092   2093   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
  2093   2094   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  2094   2095   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
         2096  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
  2095   2097   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
  2096   2098   
  2097   2099   TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  2098   2100   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  2099   2101   !IF $(USE_AMALGAMATION)==0
  2100   2102   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  2101   2103   !ELSE
................................................................................
  2241   2243   dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
  2242   2244   	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
  2243   2245   
  2244   2246   rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
  2245   2247   	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
  2246   2248   		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2247   2249   
         2250  +LSMDIR=$(TOP)\ext\lsm1
         2251  +!INCLUDE $(LSMDIR)\Makefile.msc
         2252  +
  2248   2253   moreclean:	clean
  2249   2254   	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
  2250   2255   # <</mark>>
  2251   2256   
  2252   2257   clean:
  2253   2258   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  2254   2259   	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
................................................................................
  2260   2265   	del /Q mkkeywordhash.* keywordhash.h 2>NUL
  2261   2266   	del /Q notasharedlib.* 2>NUL
  2262   2267   	-rmdir /Q/S .deps 2>NUL
  2263   2268   	-rmdir /Q/S .libs 2>NUL
  2264   2269   	-rmdir /Q/S tsrc 2>NUL
  2265   2270   	del /Q .target_source 2>NUL
  2266   2271   	del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL
         2272  +	del /Q lsm.dll lsmtest.exe 2>NUL
  2267   2273   	del /Q testloadext.dll 2>NUL
  2268   2274   	del /Q testfixture.exe test.db 2>NUL
  2269   2275   	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL
  2270   2276   	del /Q changeset.exe 2>NUL
  2271   2277   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  2272   2278   	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2273   2279   	del /Q sqlite3.c sqlite3-*.c 2>NUL

Changes to ext/fts3/fts3.c.

  1702   1702         sqlite3_reset(pCsr->pStmt);
  1703   1703         pCsr->pStmt = 0;
  1704   1704       }
  1705   1705       pCsr->bSeekStmt = 0;
  1706   1706     }
  1707   1707     sqlite3_finalize(pCsr->pStmt);
  1708   1708   }
         1709  +
         1710  +/*
         1711  +** Free all resources currently held by the cursor passed as the only
         1712  +** argument.
         1713  +*/
         1714  +static void fts3ClearCursor(Fts3Cursor *pCsr){
         1715  +  fts3CursorFinalizeStmt(pCsr);
         1716  +  sqlite3Fts3FreeDeferredTokens(pCsr);
         1717  +  sqlite3_free(pCsr->aDoclist);
         1718  +  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
         1719  +  sqlite3Fts3ExprFree(pCsr->pExpr);
         1720  +  memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
         1721  +}
  1709   1722   
  1710   1723   /*
  1711   1724   ** Close the cursor.  For additional information see the documentation
  1712   1725   ** on the xClose method of the virtual table interface.
  1713   1726   */
  1714   1727   static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
  1715   1728     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  1716   1729     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1717         -  fts3CursorFinalizeStmt(pCsr);
  1718         -  sqlite3Fts3ExprFree(pCsr->pExpr);
  1719         -  sqlite3Fts3FreeDeferredTokens(pCsr);
  1720         -  sqlite3_free(pCsr->aDoclist);
  1721         -  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
         1730  +  fts3ClearCursor(pCsr);
  1722   1731     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1723   1732     sqlite3_free(pCsr);
  1724   1733     return SQLITE_OK;
  1725   1734   }
  1726   1735   
  1727   1736   /*
  1728   1737   ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
................................................................................
  1740   1749       char *zSql;
  1741   1750       if( p->pSeekStmt ){
  1742   1751         pCsr->pStmt = p->pSeekStmt;
  1743   1752         p->pSeekStmt = 0;
  1744   1753       }else{
  1745   1754         zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
  1746   1755         if( !zSql ) return SQLITE_NOMEM;
  1747         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         1756  +      rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
  1748   1757         sqlite3_free(zSql);
  1749   1758       }
  1750   1759       if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
  1751   1760     }
  1752   1761     return rc;
  1753   1762   }
  1754   1763   
................................................................................
  3215   3224     if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++];
  3216   3225     if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++];
  3217   3226     if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
  3218   3227     if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
  3219   3228     assert( iIdx==nVal );
  3220   3229   
  3221   3230     /* In case the cursor has been used before, clear it now. */
  3222         -  fts3CursorFinalizeStmt(pCsr);
  3223         -  sqlite3_free(pCsr->aDoclist);
  3224         -  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
  3225         -  sqlite3Fts3ExprFree(pCsr->pExpr);
  3226         -  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
         3231  +  fts3ClearCursor(pCsr);
  3227   3232   
  3228   3233     /* Set the lower and upper bounds on docids to return */
  3229   3234     pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
  3230   3235     pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64);
  3231   3236   
  3232   3237     if( idxStr ){
  3233   3238       pCsr->bDesc = (idxStr[0]=='D');
................................................................................
  3277   3282         );
  3278   3283       }else{
  3279   3284         zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", 
  3280   3285             p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC")
  3281   3286         );
  3282   3287       }
  3283   3288       if( zSql ){
  3284         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         3289  +      rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
  3285   3290         sqlite3_free(zSql);
  3286   3291       }else{
  3287   3292         rc = SQLITE_NOMEM;
  3288   3293       }
  3289   3294     }else if( eSearch==FTS3_DOCID_SEARCH ){
  3290   3295       rc = fts3CursorSeekStmt(pCsr);
  3291   3296       if( rc==SQLITE_OK ){
................................................................................
  3298   3303   }
  3299   3304   
  3300   3305   /* 
  3301   3306   ** This is the xEof method of the virtual table. SQLite calls this 
  3302   3307   ** routine to find out if it has reached the end of a result set.
  3303   3308   */
  3304   3309   static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
  3305         -  return ((Fts3Cursor *)pCursor)->isEof;
         3310  +  Fts3Cursor *pCsr = (Fts3Cursor*)pCursor;
         3311  +  if( pCsr->isEof ){
         3312  +    fts3ClearCursor(pCsr);
         3313  +    pCsr->isEof = 1;
         3314  +  }
         3315  +  return pCsr->isEof;
  3306   3316   }
  3307   3317   
  3308   3318   /* 
  3309   3319   ** This is the xRowid method. The SQLite core calls this routine to
  3310   3320   ** retrieve the rowid for the current row of the result set. fts3
  3311   3321   ** exposes %_content.docid as the rowid for the virtual table. The
  3312   3322   ** rowid should be written to *pRowid.

Changes to ext/fts3/fts3_write.c.

   403    403         zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
   404    404       }else{
   405    405         zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
   406    406       }
   407    407       if( !zSql ){
   408    408         rc = SQLITE_NOMEM;
   409    409       }else{
   410         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
          410  +      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
          411  +                              &pStmt, NULL);
   411    412         sqlite3_free(zSql);
   412    413         assert( rc==SQLITE_OK || pStmt==0 );
   413    414         p->aStmt[eStmt] = pStmt;
   414    415       }
   415    416     }
   416    417     if( apVal ){
   417    418       int i;

Changes to ext/fts5/fts5_index.c.

   724    724   static int fts5IndexPrepareStmt(
   725    725     Fts5Index *p,
   726    726     sqlite3_stmt **ppStmt,
   727    727     char *zSql
   728    728   ){
   729    729     if( p->rc==SQLITE_OK ){
   730    730       if( zSql ){
   731         -      p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
          731  +      p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1,
          732  +                                 SQLITE_PREPARE_PERSISTENT, ppStmt, 0);
   732    733       }else{
   733    734         p->rc = SQLITE_NOMEM;
   734    735       }
   735    736     }
   736    737     sqlite3_free(zSql);
   737    738     return p->rc;
   738    739   }
................................................................................
   773    774       char *zSql = sqlite3_mprintf(
   774    775           "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", 
   775    776             pConfig->zDb, pConfig->zName
   776    777       );
   777    778       if( zSql==0 ){
   778    779         rc = SQLITE_NOMEM;
   779    780       }else{
   780         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0);
          781  +      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
          782  +                              SQLITE_PREPARE_PERSISTENT, &p->pDeleter, 0);
   781    783         sqlite3_free(zSql);
   782    784       }
   783    785       if( rc!=SQLITE_OK ){
   784    786         p->rc = rc;
   785    787         return;
   786    788       }
   787    789     }

Changes to ext/fts5/fts5_main.c.

   879    879     va_list ap;
   880    880   
   881    881     va_start(ap, zFmt);
   882    882     zSql = sqlite3_vmprintf(zFmt, ap);
   883    883     if( zSql==0 ){
   884    884       rc = SQLITE_NOMEM; 
   885    885     }else{
   886         -    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
          886  +    rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 
          887  +                            SQLITE_PREPARE_PERSISTENT, &pRet, 0);
   887    888       if( rc!=SQLITE_OK ){
   888    889         *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
   889    890       }
   890    891       sqlite3_free(zSql);
   891    892     }
   892    893   
   893    894     va_end(ap);
................................................................................
  1015   1016     const char *zRank = pCsr->zRank;
  1016   1017     const char *zRankArgs = pCsr->zRankArgs;
  1017   1018   
  1018   1019     if( zRankArgs ){
  1019   1020       char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
  1020   1021       if( zSql ){
  1021   1022         sqlite3_stmt *pStmt = 0;
  1022         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0);
         1023  +      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
         1024  +                              SQLITE_PREPARE_PERSISTENT, &pStmt, 0);
  1023   1025         sqlite3_free(zSql);
  1024   1026         assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
  1025   1027         if( rc==SQLITE_OK ){
  1026   1028           if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1027   1029             int nByte;
  1028   1030             pCsr->nRankArg = sqlite3_column_count(pStmt);
  1029   1031             nByte = sizeof(sqlite3_value*)*pCsr->nRankArg;

Changes to ext/fts5/fts5_storage.c.

   132    132           zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
   133    133           break;
   134    134       }
   135    135   
   136    136       if( zSql==0 ){
   137    137         rc = SQLITE_NOMEM;
   138    138       }else{
   139         -      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
          139  +      rc = sqlite3_prepare_v3(pC->db, zSql, -1,
          140  +                              SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
   140    141         sqlite3_free(zSql);
   141    142         if( rc!=SQLITE_OK && pzErrMsg ){
   142    143           *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
   143    144         }
   144    145       }
   145    146     }
   146    147   

Changes to ext/fts5/test/fts5aa.test.

   437    437   # exception. But since bm25() can now used the cached structure record,
   438    438   # it never sees the corruption introduced by funk() and so the following 
   439    439   # statement no longer fails.
   440    440   #
   441    441   do_catchsql_test 16.2 {
   442    442     SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
   443    443   } {0 {{} -1e-06 {}}}
   444         -# {1 {SQL logic error or missing database}}
          444  +# {1 {SQL logic error}}
   445    445   
   446    446   #-------------------------------------------------------------------------
   447    447   #
   448    448   reset_db
   449    449   do_execsql_test 17.1 {
   450    450     CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
   451    451     INSERT INTO b2 VALUES('a');
................................................................................
   591    591     SELECT rowid FROM t9('a*')
   592    592   } {1}
   593    593   
   594    594   }
   595    595   
   596    596   
   597    597   finish_test
   598         -
   599         -

Changes to ext/fts5/test/fts5ab.test.

   290    290     INSERT INTO x1 VALUES($doc);
   291    291   }
   292    292   
   293    293   } ;# foreach_detail_mode...
   294    294   
   295    295   
   296    296   finish_test
   297         -

Changes to ext/fts5/test/fts5ac.test.

   272    272   } {
   273    273     do_execsql_test 2.3.$tn {
   274    274       SELECT fts5_expr_tcl($expr, 'N $x')
   275    275     } [list $tclexpr]
   276    276   }
   277    277   
   278    278   finish_test
   279         -

Changes to ext/fts5/test/fts5ad.test.

   238    238   
   239    239     catchsql COMMIT
   240    240   }
   241    241   
   242    242   }
   243    243   
   244    244   finish_test
   245         -

Changes to ext/fts5/test/fts5ae.test.

   305    305       SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
   306    306     } $cnt
   307    307   }
   308    308   
   309    309   }
   310    310   
   311    311   finish_test
   312         -

Changes to ext/fts5/test/fts5af.test.

   174    174   do_execsql_test 5.1 {
   175    175     SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
   176    176   } {{[x] a a a a a...}}
   177    177   
   178    178   } ;# foreach_detail_mode 
   179    179   
   180    180   finish_test
   181         -

Changes to ext/fts5/test/fts5ag.test.

   138    138     }
   139    139   }
   140    140   
   141    141   } ;# foreach_detail_mode
   142    142   
   143    143   
   144    144   finish_test
   145         -

Changes to ext/fts5/test/fts5ah.test.

   163    163   } {10000}
   164    164   
   165    165   } ;# foreach_detail_mode
   166    166   
   167    167   #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
   168    168   
   169    169   finish_test
   170         -

Changes to ext/fts5/test/fts5ai.test.

    51     51   do_execsql_test 1.2 {
    52     52     INSERT INTO t1(t1) VALUES('integrity-check');
    53     53   }
    54     54   }
    55     55   
    56     56   
    57     57   finish_test
    58         -

Changes to ext/fts5/test/fts5aj.test.

    62     62     }
    63     63   }
    64     64   
    65     65   do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }
    66     66   
    67     67   
    68     68   finish_test
    69         -

Changes to ext/fts5/test/fts5ak.test.

   143    143     {[a b c] [c d e]}
   144    144     {[a b c d e]}
   145    145   }
   146    146   
   147    147   }
   148    148   
   149    149   finish_test
   150         -

Changes to ext/fts5/test/fts5al.test.

    73     73     1 ""
    74     74     2 "fname"
    75     75     3 "fname(X'234ab')"
    76     76     4 "myfunc(-1.,'abc')"
    77     77   } {
    78     78     do_test 2.2.$tn {
    79     79       catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
    80         -  } {1 {SQL logic error or missing database}}
           80  +  } {1 {SQL logic error}}
    81     81   }
    82     82   
    83     83   #-------------------------------------------------------------------------
    84     84   # Assorted tests of the tcl interface for creating extension functions.
    85     85   #
    86     86   
    87     87   do_execsql_test 3.1 {
................................................................................
   293    293     SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
   294    294   } {1 {parse error in rank function: }}
   295    295   
   296    296   } ;# foreach_detail_mode
   297    297   
   298    298   
   299    299   finish_test
   300         -

Changes to ext/fts5/test/fts5alter.test.

    96     96   
    97     97   do_execsql_test 3.3 {
    98     98     COMMIT;
    99     99     SELECT rowid FROM abc WHERE abc MATCH 'a';
   100    100   } {1 2}
   101    101   
   102    102   finish_test
   103         -

Changes to ext/fts5/test/fts5auto.test.

   338    338   } {
   339    339     do_auto_test 4.$tn yy $expr
   340    340   }
   341    341   
   342    342   
   343    343   
   344    344   finish_test
   345         -

Changes to ext/fts5/test/fts5aux.test.

   275    275   } {
   276    276     9 10
   277    277   }
   278    278   
   279    279   
   280    280   
   281    281   finish_test
   282         -

Changes to ext/fts5/test/fts5auxdata.test.

   108    108   db eval { 
   109    109     SELECT aux_function_2(f1, 2, 'A'), aux_function_2(f1, 2, 'B') 
   110    110     FROM f1 WHERE f1 MATCH 'a'
   111    111     ORDER BY rowid ASC
   112    112   }
   113    113   
   114    114   finish_test
   115         -

Changes to ext/fts5/test/fts5bigpl.test.

    57     57       set doc [string repeat "$t " 150000000]
    58     58       execsql { INSERT INTO t1 VALUES($doc) }
    59     59     }
    60     60     execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    61     61   } {}
    62     62   
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5bigtok.test.

    60     60       do_execsql_test 2.[string range $v 0 0] {
    61     61         SELECT rowid FROM t1($v) ORDER BY rowid DESC
    62     62       } [lsort -integer -decr $res]
    63     63     }
    64     64   }
    65     65   
    66     66   finish_test
    67         -
    68         -

Changes to ext/fts5/test/fts5colset.test.

    80     80     do_catchsql_test 4.1 {
    81     81       SELECT * FROM t1 WHERE rowid MATCH 'a'
    82     82     } {1 {unable to use function MATCH in the requested context}}
    83     83   }
    84     84   
    85     85   
    86     86   finish_test
    87         -
    88         -

Changes to ext/fts5/test/fts5config.test.

    62     62     5 "f1(x':;')"
    63     63     6 "f1(x'[]')"
    64     64     7 "f1(x'{}')"
    65     65     8 "f1('abc)"
    66     66   } {
    67     67     do_catchsql_test 3.$tn {
    68     68       INSERT INTO t1(t1, rank) VALUES('rank', $val);
    69         -  } {1 {SQL logic error or missing database}}
           69  +  } {1 {SQL logic error}}
    70     70   }
    71     71   
    72     72   #-------------------------------------------------------------------------
    73     73   # The parsing of SQL literals specified as part of 'rank' options.
    74     74   #
    75     75   do_execsql_test 4.0 {
    76     76     CREATE VIRTUAL TABLE zzz USING fts5(one);
................................................................................
   165    165   #   9.5.* 'hashsize' options.
   166    166   #
   167    167   do_execsql_test 9.0 {
   168    168     CREATE VIRTUAL TABLE abc USING fts5(a, b);
   169    169   } {}
   170    170   do_catchsql_test 9.1.1 {
   171    171     INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
   172         -} {1 {SQL logic error or missing database}}
          172  +} {1 {SQL logic error}}
   173    173   do_catchsql_test 9.1.2 {
   174    174     INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
   175         -} {1 {SQL logic error or missing database}}
          175  +} {1 {SQL logic error}}
   176    176   do_catchsql_test 9.1.3 {
   177    177     INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
   178         -} {1 {SQL logic error or missing database}}
          178  +} {1 {SQL logic error}}
   179    179   
   180    180   do_catchsql_test 9.2.1 {
   181    181     INSERT INTO abc(abc, rank) VALUES('automerge', -5);
   182         -} {1 {SQL logic error or missing database}}
          182  +} {1 {SQL logic error}}
   183    183   do_catchsql_test 9.2.2 {
   184    184     INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
   185         -} {1 {SQL logic error or missing database}}
          185  +} {1 {SQL logic error}}
   186    186   do_catchsql_test 9.2.3 {
   187    187     INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
   188         -} {1 {SQL logic error or missing database}}
          188  +} {1 {SQL logic error}}
   189    189   do_execsql_test 9.2.4 {
   190    190     INSERT INTO abc(abc, rank) VALUES('automerge', 1);
   191    191   } {}
   192    192   
   193    193   do_catchsql_test 9.3.1 {
   194    194     INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
   195         -} {1 {SQL logic error or missing database}}
          195  +} {1 {SQL logic error}}
   196    196   do_catchsql_test 9.3.2 {
   197    197     INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
   198         -} {1 {SQL logic error or missing database}}
          198  +} {1 {SQL logic error}}
   199    199   do_execsql_test 9.3.3 {
   200    200     INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
   201    201   } {}
   202    202   do_execsql_test 9.3.4 {
   203    203     INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
   204    204   } {}
   205    205   
   206    206   do_catchsql_test 9.4.1 {
   207    207     INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
   208         -} {1 {SQL logic error or missing database}}
          208  +} {1 {SQL logic error}}
   209    209   
   210    210   do_catchsql_test 9.5.1 {
   211    211     INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
   212         -} {1 {SQL logic error or missing database}}
          212  +} {1 {SQL logic error}}
   213    213   do_catchsql_test 9.5.2 {
   214    214     INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
   215         -} {1 {SQL logic error or missing database}}
          215  +} {1 {SQL logic error}}
   216    216   do_catchsql_test 9.5.3 {
   217    217     INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
   218    218   } {0 {}}
   219    219   
   220    220   #-------------------------------------------------------------------------
   221    221   # Too many prefix indexes. Maximum allowed is 31.
   222    222   #
................................................................................
   241    241   } {
   242    242     set res [list 1 {malformed detail=... directive}]
   243    243     do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
   244    244   }
   245    245   
   246    246   do_catchsql_test 12.1 {
   247    247     INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
   248         -} {1 {SQL logic error or missing database}}
          248  +} {1 {SQL logic error}}
   249    249   
   250    250   #-------------------------------------------------------------------------
   251    251   # errors in the 'usermerge' option
   252    252   #
   253    253   do_execsql_test 13.0 {
   254    254     CREATE VIRTUAL TABLE tt USING fts5(ttt);
   255    255   }
................................................................................
   256    256   foreach {tn val} {
   257    257     1     -1
   258    258     2     4.2
   259    259     3     17
   260    260     4     1
   261    261   } {
   262    262     set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
   263         -  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
          263  +  do_catchsql_test 13.$tn $sql {1 {SQL logic error}}
   264    264   }
   265    265   
   266    266   finish_test
   267         -

Changes to ext/fts5/test/fts5conflict.test.

    62     62     REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1');
    63     63     DELETE FROM tbl WHERE a=100;
    64     64   
    65     65     INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
    66     66   }
    67     67   
    68     68   finish_test
    69         -
    70         -

Changes to ext/fts5/test/fts5content.test.

   251    251   do_execsql_test 6.2 {
   252    252     DROP TABLE xx;
   253    253     SELECT name FROM sqlite_master;
   254    254   } {}
   255    255   
   256    256   
   257    257   finish_test
   258         -

Changes to ext/fts5/test/fts5corrupt.test.

    92     92   
    93     93   do_catchsql_test 3.1 {
    94     94     DELETE FROM t3_content WHERE rowid = 3;
    95     95     SELECT * FROM t3 WHERE t3 MATCH 'o';
    96     96   } {1 {database disk image is malformed}}
    97     97   
    98     98   finish_test
    99         -

Changes to ext/fts5/test/fts5corrupt2.test.

   265    265   do_catchsql_test 6.2 {
   266    266     SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a'
   267    267   } {1 SQLITE_CORRUPT_VTAB}
   268    268   
   269    269   
   270    270   sqlite3_fts5_may_be_corrupt 0
   271    271   finish_test
   272         -

Changes to ext/fts5/test/fts5corrupt3.test.

   405    405   } {}
   406    406   do_catchsql_test 9.2.2 {
   407    407     SELECT * FROM t1('one AND two');
   408    408   } {1 {database disk image is malformed}}
   409    409   
   410    410   sqlite3_fts5_may_be_corrupt 0
   411    411   finish_test
   412         -

Changes to ext/fts5/test/fts5delete.test.

    47     47     for {set i 0} {$i < 5} {incr i} {
    48     48       execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
    49     49       execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    50     50     }
    51     51   } {}
    52     52   
    53     53   finish_test
    54         -

Changes to ext/fts5/test/fts5detail.test.

   237    237       (SELECT sum(length(block)) from t2_data) <
   238    238       (SELECT sum(length(block)) from t3_data)
   239    239   } {1}
   240    240   
   241    241   
   242    242   
   243    243   finish_test
   244         -

Changes to ext/fts5/test/fts5determin.test.

    59     59     } {}
    60     60   
    61     61     do_determin_test 1.4
    62     62   }
    63     63   
    64     64   
    65     65   finish_test
    66         -
    67         -

Changes to ext/fts5/test/fts5dlidx.test.

   193    193   }
   194    194   
   195    195   } ;# foreach_detail_mode
   196    196   
   197    197   
   198    198   
   199    199   finish_test
   200         -

Changes to ext/fts5/test/fts5doclist.test.

    40     40   
    41     41   do_execsql_test 1.2 {
    42     42     INSERT INTO ccc(ccc) VALUES('integrity-check');
    43     43   }
    44     44   
    45     45   
    46     46   finish_test
    47         -

Changes to ext/fts5/test/fts5eb.test.

    77     77   do_execsql_test 3.3 {
    78     78     SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
    79     79   } {1 -1e-06}
    80     80   
    81     81   
    82     82   
    83     83   finish_test
    84         -
    85         -
    86         -

Changes to ext/fts5/test/fts5fault1.test.

   347    347       if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" }
   348    348     }
   349    349   }
   350    350   
   351    351   
   352    352   
   353    353   finish_test
   354         -

Changes to ext/fts5/test/fts5fault2.test.

   133    133       );
   134    134     }
   135    135   } -test {
   136    136     faultsim_test_result {0 {}}
   137    137   }
   138    138   
   139    139   finish_test
   140         -

Changes to ext/fts5/test/fts5fault3.test.

   106    106   } -test {
   107    107     faultsim_test_result [list 0 {}]
   108    108   }
   109    109   
   110    110   
   111    111   
   112    112   finish_test
   113         -

Changes to ext/fts5/test/fts5fault4.test.

   391    391   } -body {
   392    392     db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" }
   393    393   } -test {
   394    394     faultsim_test_result {0 {}}
   395    395   }
   396    396   
   397    397   finish_test
   398         -

Changes to ext/fts5/test/fts5fault5.test.

   126    126         9 x 1 {}
   127    127     ]]
   128    128   }
   129    129   
   130    130   
   131    131   
   132    132   finish_test
   133         -

Changes to ext/fts5/test/fts5fault6.test.

   288    288     faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }}
   289    289     if {$testrc==0} {
   290    290       db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) }
   291    291     }
   292    292     db close
   293    293   }
   294    294   finish_test
   295         -

Changes to ext/fts5/test/fts5fault7.test.

   112    112   do_faultsim_test 2.2 -faults oom-* -body {
   113    113     db eval { SELECT * FROM xy('""') }
   114    114   } -test {
   115    115     faultsim_test_result {0 {}}
   116    116   }
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5fault8.test.

    78     78     execsql { INSERT INTO x2(x2) VALUES('optimize') }
    79     79   } -test {
    80     80     faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
    81     81   }
    82     82   
    83     83   
    84     84   finish_test
    85         -

Changes to ext/fts5/test/fts5fault9.test.

   149    149     faultsim_test_result [list 0 {1 3}]
   150    150   }
   151    151   
   152    152   
   153    153   } ;# foreach_detail_mode...
   154    154   
   155    155   finish_test
   156         -

Changes to ext/fts5/test/fts5faultA.test.

    57     57     sqlite3 db test.db
    58     58   } -body {
    59     59     execsql { SELECT rowid FROM o2('a+b+c NOT xyz') }
    60     60   } -test {
    61     61     faultsim_test_result {0 {1 2}}
    62     62   }
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5faultB.test.

   128    128     execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
   129    129   } -test {
   130    130     faultsim_test_result {0 {2 3}}
   131    131   }
   132    132   
   133    133   
   134    134   finish_test
   135         -

Changes to ext/fts5/test/fts5full.test.

    36     36         execsql { INSERT INTO x8 VALUES( rnddoc(5) ); }
    37     37       }
    38     38     } msg] $msg
    39     39   } {1 {database or disk is full}}
    40     40   
    41     41   
    42     42   finish_test
    43         -

Changes to ext/fts5/test/fts5fuzz1.test.

    86     86   reset_db
    87     87   do_catchsql_test 4.1 {
    88     88     CREATE VIRTUAL TABLE f2 USING fts5(o, t);
    89     89     SELECT * FROM f2('(8 AND 9)`AND 10');
    90     90   } {1 {fts5: syntax error near "`"}}
    91     91   
    92     92   finish_test
    93         -

Changes to ext/fts5/test/fts5hash.test.

   126    126         INSERT INTO t2 VALUES($small || ' ' || $big);
   127    127       }
   128    128     } {}
   129    129   
   130    130   } ;# foreach_detail_mode
   131    131   
   132    132   finish_test
   133         -

Changes to ext/fts5/test/fts5integrity.test.

   206    206         if {$res == [lsort -integer $res2]} { incr ok }
   207    207       }
   208    208       set ok
   209    209     } {1000}
   210    210   }
   211    211   
   212    212   finish_test
   213         -

Changes to ext/fts5/test/fts5lastrowid.test.

    66     66   do_execsql_test 1.6 {
    67     67     INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
    68     68     SELECT last_insert_rowid();
    69     69   } {14}
    70     70   
    71     71   
    72     72   finish_test
    73         -

Changes to ext/fts5/test/fts5leftjoin.test.

    37     37   } {1 1 abc 2 {} {}}
    38     38   
    39     39   do_execsql_test 1.2 {
    40     40     SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc')
    41     41   } {1 abc 2 abc}
    42     42   
    43     43   finish_test
    44         -
    45         -

Changes to ext/fts5/test/fts5matchinfo.test.

   488    488   } {}
   489    489   
   490    490   do_catchsql_test 14.2 {
   491    491     SELECT matchinfo(x1, 'd') FROM x1('a b c');
   492    492   } {1 {unrecognized matchinfo flag: d}}
   493    493   
   494    494   finish_test
   495         -

Changes to ext/fts5/test/fts5merge.test.

   237    237   do_execsql_test 6.3 {
   238    238     INSERT INTO g1(g1) VALUES('integrity-check');
   239    239   }
   240    240   
   241    241   
   242    242   
   243    243   finish_test
   244         -

Changes to ext/fts5/test/fts5merge2.test.

    51     51   do_execsql_test 1.2 {
    52     52     INSERT INTO t1(t1) VALUES('integrity-check');
    53     53   }
    54     54   
    55     55   }
    56     56   
    57     57   finish_test
    58         -

Changes to ext/fts5/test/fts5multiclient.test.

    41     41       sql1 { INSERT INTO t1 VALUES('a b c') }
    42     42       sql3 { INSERT INTO t1(t1) VALUES('integrity-check') }
    43     43     } {}
    44     44   
    45     45   };# do_multiclient_test
    46     46   };# foreach_detail_mode
    47     47   finish_test
    48         -

Changes to ext/fts5/test/fts5near.test.

    64     64   do_near_test 1.23 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 4) } 0
    65     65   
    66     66   do_near_test 1.24 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 5) } 1
    67     67   do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0
    68     68   
    69     69   
    70     70   finish_test
    71         -

Changes to ext/fts5/test/fts5onepass.test.

   174    174       UPDATE ttt SET x = 'A B C' WHERE rowid = 4;
   175    175       INSERT INTO ttt(rowid, x) VALUES(6, 'd e f');
   176    176     COMMIT;
   177    177   } {}
   178    178   do_test 4.2.2 { fts5_level_segs ttt } {3}
   179    179   
   180    180   finish_test
   181         -

Changes to ext/fts5/test/fts5optimize.test.

   102    102     do_execsql_test 2.$tn.5 {
   103    103       INSERT INTO t1(t1) VALUES('integrity-check');
   104    104     }
   105    105   
   106    106     do_test 2.$tn.6 { fts5_segcount t1 } 1
   107    107   }
   108    108   finish_test
   109         -

Changes to ext/fts5/test/fts5phrase.test.

   112    112     FROM t3('a:f+f')
   113    113   } {
   114    114     31 {h *f f*} {i j g e c} {j j f c a i j} 
   115    115     50 {*f f* c} {f f b i i} {f f a j e c i}
   116    116   }
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5plan.test.

    60     60     0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
    61     61   }
    62     62   
    63     63   
    64     64   
    65     65   
    66     66   finish_test
    67         -

Changes to ext/fts5/test/fts5porter.test.

 11799  11799       lindex [sqlite3_fts5_tokenize db porter $in] 0
 11800  11800     } $out
 11801  11801     incr i
 11802  11802   }
 11803  11803   
 11804  11804   
 11805  11805   finish_test
 11806         -

Changes to ext/fts5/test/fts5porter2.test.

    63     63       lindex [sqlite3_fts5_tokenize db porter $in] 0
    64     64     } $out
    65     65     incr i
    66     66   }
    67     67   
    68     68   
    69     69   finish_test
    70         -

Changes to ext/fts5/test/fts5prefix.test.

   337    337       do_execsql_test 7.$tn {
   338    338         SELECT md5sum(id, block) FROM tt_data
   339    339       } [list $::checksum]
   340    340     }
   341    341   }
   342    342   
   343    343   finish_test
   344         -
   345         -

Changes to ext/fts5/test/fts5query.test.

    75     75       } {}
    76     76       incr ret
    77     77     }
    78     78   }
    79     79   
    80     80   
    81     81   finish_test
    82         -
    83         -

Changes to ext/fts5/test/fts5rank.test.

   148    148     VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
   149    149   } {{wrinkle in time} {Bill Smith}}
   150    150   
   151    151   
   152    152   
   153    153   
   154    154   finish_test
   155         -

Changes to ext/fts5/test/fts5rebuild.test.

    60     60     CREATE VIRTUAL TABLE nc USING fts5(doc, content=);
    61     61   }
    62     62   
    63     63   do_catchsql_test 2.2 {
    64     64     INSERT INTO nc(nc) VALUES('rebuild');
    65     65   } {1 {'rebuild' may not be used with a contentless fts5 table}}
    66     66   finish_test
    67         -

Changes to ext/fts5/test/fts5restart.test.

   145    145     }
   146    146     set res
   147    147   } {500 400 300}
   148    148   
   149    149   
   150    150   
   151    151   finish_test
   152         -

Changes to ext/fts5/test/fts5rowid.test.

   212    212   } {36}
   213    213   
   214    214   #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r}
   215    215   
   216    216   
   217    217   
   218    218   finish_test
   219         -

Changes to ext/fts5/test/fts5simple2.test.

   366    366   do_execsql_test 17.6 { 
   367    367     SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55
   368    368   }
   369    369   
   370    370   #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
   371    371     
   372    372   finish_test
   373         -

Changes to ext/fts5/test/fts5simple3.test.

   112    112   } 
   113    113   do_execsql_test 4.6 {
   114    114     SELECT * FROM t2('ab + xyz');
   115    115   }
   116    116   
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5synonym.test.

   417    417   do_execsql_test 7.1.2 {
   418    418     INSERT INTO t2(t2) VALUES('integrity-check');
   419    419   }
   420    420   
   421    421   } ;# foreach_detail_mode
   422    422   
   423    423   finish_test
   424         -

Changes to ext/fts5/test/fts5synonym2.test.

   157    157   
   158    158   }
   159    159   
   160    160   }
   161    161   }
   162    162   
   163    163   finish_test
   164         -

Changes to ext/fts5/test/fts5tok1.test.

   105    105   do_catchsql_test 2.0 {
   106    106     CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer);
   107    107   } {1 {vtable constructor failed: tX}}
   108    108   
   109    109   do_catchsql_test 2.1 {
   110    110     CREATE VIRTUAL TABLE t4 USING fts5tokenize;
   111    111     SELECT * FROM t4;
   112         -} {1 {SQL logic error or missing database}}
          112  +} {1 {SQL logic error}}
   113    113   
   114    114   
   115    115   finish_test

Changes to ext/fts5/test/fts5tokenizer.test.

   298    298   
   299    299   set ::flags [list]
   300    300   do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
   301    301   do_test 9.5.2 { set ::flags } {query}
   302    302   
   303    303   
   304    304   finish_test
   305         -

Changes to ext/fts5/test/fts5unicode.test.

    55     55     SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC';
    56     56     SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC';
    57     57     SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
    58     58   " {t1 t2}
    59     59   
    60     60   
    61     61   finish_test
    62         -

Changes to ext/fts5/test/fts5unicode3.test.

   122    122     }
   123    123     append str {'");}
   124    124     execsql $str
   125    125   } {}
   126    126   
   127    127   
   128    128   finish_test
   129         -

Changes to ext/fts5/test/fts5unindexed.test.

    72     72     INSERT INTO t4(t4, rowid, a, b, c) VALUES('delete', 20, 'j k l', '', 'p q r');
    73     73     DELETE FROM x4 WHERE rowid=20;
    74     74     INSERT INTO t4(t4) VALUES('integrity-check');
    75     75   } {}
    76     76   
    77     77   
    78     78   finish_test
    79         -

Changes to ext/fts5/test/fts5update.test.

   113    113   } {}
   114    114   do_execsql_test 2.2.integrity {
   115    115     INSERT INTO x2(x2) VALUES('integrity-check');
   116    116   }
   117    117   
   118    118   }
   119    119   finish_test
   120         -
   121         -

Changes to ext/fts5/test/fts5version.test.

    57     57     db close
    58     58     sqlite3 db test.db
    59     59     catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
    60     60   } {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
    61     61   
    62     62   
    63     63   finish_test
    64         -

Added ext/lsm1/Makefile.

            1  +#
            2  +# This Makefile is designed for use with main.mk in the root directory of
            3  +# this project. After including main.mk, the users makefile should contain:
            4  +#
            5  +#    LSMDIR=$(TOP)/ext/lsm1/
            6  +#    include $(LSMDIR)/Makefile
            7  +#
            8  +# The most useful targets are [lsmtest] and [lsm.so].
            9  +#
           10  +
           11  +LSMOBJ    = \
           12  +  lsm_ckpt.o \
           13  +  lsm_file.o \
           14  +  lsm_log.o \
           15  +  lsm_main.o \
           16  +  lsm_mem.o \
           17  +  lsm_mutex.o \
           18  +  lsm_shared.o \
           19  +  lsm_sorted.o \
           20  +  lsm_str.o \
           21  +  lsm_tree.o \
           22  +  lsm_unix.o \
           23  +  lsm_win32.o \
           24  +  lsm_varint.o \
           25  +  lsm_vtab.o
           26  +
           27  +LSMHDR   = \
           28  +  $(LSMDIR)/lsm.h \
           29  +  $(LSMDIR)/lsmInt.h
           30  +
           31  +LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c     \
           32  +             $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c     \
           33  +             $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c     \
           34  +             $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c     \
           35  +             $(LSMDIR)/lsm-test/lsmtest9.c                                   \
           36  +             $(LSMDIR)/lsm-test/lsmtest_datasource.c \
           37  +             $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c  \
           38  +             $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \
           39  +             $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \
           40  +             $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c
           41  +
           42  +
           43  +# all: lsm.so
           44  +
           45  +LSMOPTS = -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
           46  +
           47  +lsm.so:	$(LSMOBJ)
           48  +	$(TCCX) -shared -o lsm.so $(LSMOBJ)
           49  +
           50  +%.o:	$(LSMDIR)/%.c $(LSMHDR) sqlite3.h
           51  +	$(TCCX) $(LSMOPTS) -c $<
           52  +	
           53  +lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
           54  +	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
           55  +	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) 
           56  +

Added ext/lsm1/Makefile.msc.

            1  +#
            2  +# This Makefile is designed for use with main.mk in the root directory of
            3  +# this project. After including main.mk, the users makefile should contain:
            4  +#
            5  +#    LSMDIR=$(TOP)\ext\lsm1\
            6  +#    include $(LSMDIR)\Makefile.msc
            7  +#
            8  +# The most useful targets are [lsmtest.exe] and [lsm.dll].
            9  +#
           10  +
           11  +LSMOBJ    = \
           12  +  lsm_ckpt.lo \
           13  +  lsm_file.lo \
           14  +  lsm_log.lo \
           15  +  lsm_main.lo \
           16  +  lsm_mem.lo \
           17  +  lsm_mutex.lo \
           18  +  lsm_shared.lo \
           19  +  lsm_sorted.lo \
           20  +  lsm_str.lo \
           21  +  lsm_tree.lo \
           22  +  lsm_unix.lo \
           23  +  lsm_win32.lo \
           24  +  lsm_varint.lo \
           25  +  lsm_vtab.lo
           26  +
           27  +LSMHDR   = \
           28  +  $(LSMDIR)\lsm.h \
           29  +  $(LSMDIR)\lsmInt.h
           30  +
           31  +LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c     \
           32  +             $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c     \
           33  +             $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c     \
           34  +             $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c     \
           35  +             $(LSMDIR)\lsm-test\lsmtest9.c                                   \
           36  +             $(LSMDIR)\lsm-test\lsmtest_datasource.c \
           37  +             $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c  \
           38  +             $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \
           39  +             $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \
           40  +             $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c
           41  +
           42  +# all: lsm.dll
           43  +
           44  +LSMOPTS = -DLSM_MUTEX_WIN32=1 -I$(LSMDIR)
           45  +
           46  +!IF $(DEBUG)>2
           47  +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1
           48  +!ENDIF
           49  +
           50  +!IF $(MEMDEBUG)!=0
           51  +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1
           52  +!ENDIF
           53  +
           54  +lsm_ckpt.lo:	$(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H)
           55  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c
           56  +
           57  +lsm_file.lo:	$(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H)
           58  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c
           59  +
           60  +lsm_log.lo:	$(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H)
           61  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c
           62  +
           63  +lsm_main.lo:	$(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H)
           64  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c
           65  +
           66  +lsm_mem.lo:	$(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H)
           67  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c
           68  +
           69  +lsm_mutex.lo:	$(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H)
           70  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c
           71  +
           72  +lsm_shared.lo:	$(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H)
           73  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c
           74  +
           75  +lsm_sorted.lo:	$(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H)
           76  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c
           77  +
           78  +lsm_str.lo:	$(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H)
           79  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c
           80  +
           81  +lsm_tree.lo:	$(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H)
           82  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c
           83  +
           84  +lsm_unix.lo:	$(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H)
           85  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c
           86  +
           87  +lsm_win32.lo:	$(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H)
           88  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c
           89  +
           90  +lsm_varint.lo:	$(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H)
           91  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c
           92  +
           93  +lsm_vtab.lo:	$(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H)
           94  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c
           95  +
           96  +lsm.dll:	$(LSMOBJ)
           97  +	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ)
           98  +
           99  +lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJS1)
          100  +	$(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJS1)

Added ext/lsm1/lsm-test/README.

            1  +
            2  +
            3  +Organization of test case files:
            4  +
            5  +  lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a 
            6  +              database file, then verify that the contents of the database can
            7  +              be queried.
            8  +
            9  +  lsmtest2.c: Crash tests. Tests that attempt to verify that the database 
           10  +              recovers correctly following an application or system crash.
           11  +
           12  +  lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of
           13  +              transactions and sub-transactions.
           14  +
           15  +  lsmtest4.c: Multi-client tests.
           16  +
           17  +  lsmtest5.c: Multi-client tests with a different thread for each client.
           18  +
           19  +  lsmtest6.c: OOM injection tests.
           20  +
           21  +  lsmtest7.c: API tests.
           22  +
           23  +  lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that
           24  +              the system recovers and other clients proceed unaffected if
           25  +              a process fails in the middle of a write transaction.
           26  +
           27  +              The difference from lsmtest2.c is that this file tests
           28  +              live-recovery (recovery from a failure that occurs while other
           29  +              clients are still running) whereas lsmtest2.c tests recovery
           30  +              from a system or power failure.
           31  +
           32  +  lsmtest9.c: More data tests. These focus on testing that calling
           33  +              lsm_work(nMerge=1) to compact the database does not corrupt it.
           34  +              In other words, that databases containing block-redirects
           35  +              can be read and written.
           36  +
           37  +
           38  +
           39  +
           40  +

Added ext/lsm1/lsm-test/lsmtest.h.

            1  +
            2  +#ifndef __WRAPPER_INT_H_
            3  +#define __WRAPPER_INT_H_
            4  +
            5  +#include "lsmtest_tdb.h"
            6  +#include "sqlite3.h"
            7  +#include "lsm.h"
            8  +
            9  +#include <assert.h>
           10  +#include <stdlib.h>
           11  +#include <string.h>
           12  +#include <stdio.h>
           13  +
           14  +#ifdef __cplusplus
           15  +extern "C" {
           16  +#endif
           17  +
           18  +#ifndef _O_BINARY
           19  +# define _O_BINARY (0)
           20  +#endif
           21  +
           22  +#ifdef _WIN32
           23  +# include "windows.h"
           24  +# define gettimeofday win32GetTimeOfDay
           25  +# define F_OK  (0)
           26  +# define sleep(sec) Sleep(1000 * (sec))
           27  +# define usleep(usec) Sleep(((usec) + 999) / 1000)
           28  +# ifdef _MSC_VER
           29  +#  include <io.h>
           30  +#  define snprintf _snprintf
           31  +#  define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
           32  +#  define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
           33  +#  define __va_copy(dst,src) ((dst) = (src))
           34  +#  define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1)
           35  +# else
           36  +#  error Unsupported C compiler for Windows.
           37  +# endif
           38  +int win32GetTimeOfDay(struct timeval *, void *);
           39  +#endif
           40  +
           41  +#ifndef _LSM_INT_H
           42  +typedef unsigned int  u32;
           43  +typedef unsigned char u8;
           44  +typedef long long int i64;
           45  +typedef unsigned long long int u64;
           46  +#endif
           47  +
           48  +
           49  +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0])))
           50  +
           51  +#define MIN(x,y) ((x)<(y) ? (x) : (y))
           52  +#define MAX(x,y) ((x)>(y) ? (x) : (y))
           53  +
           54  +#define unused_parameter(x) (void)(x)
           55  +
           56  +#define TESTDB_DEFAULT_PAGE_SIZE   4096
           57  +#define TESTDB_DEFAULT_CACHE_SIZE  2048
           58  +
           59  +/*
           60  +** Ideally, these should be in wrapper.c. But they are here instead so that 
           61  +** they can be used by the C++ database wrappers in wrapper2.cc.
           62  +*/
           63  +typedef struct DatabaseMethods DatabaseMethods;
           64  +struct TestDb {
           65  +  DatabaseMethods const *pMethods;          /* Database methods */
           66  +  const char *zLibrary;                     /* Library name for tdb_open() */
           67  +};
           68  +struct DatabaseMethods {
           69  +  int (*xClose)(TestDb *);
           70  +  int (*xWrite)(TestDb *, void *, int , void *, int);
           71  +  int (*xDelete)(TestDb *, void *, int);
           72  +  int (*xDeleteRange)(TestDb *, void *, int, void *, int);
           73  +  int (*xFetch)(TestDb *, void *, int, void **, int *);
           74  +  int (*xScan)(TestDb *, void *, int, void *, int, void *, int,
           75  +    void (*)(void *, void *, int , void *, int)
           76  +  );
           77  +  int (*xBegin)(TestDb *, int);
           78  +  int (*xCommit)(TestDb *, int);
           79  +  int (*xRollback)(TestDb *, int);
           80  +};
           81  +
           82  +/* 
           83  +** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the
           84  +** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but
           85  +** the primary interface is the C++ API.
           86  +*/
           87  +int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb);
           88  +int test_kc_close(TestDb *);
           89  +int test_kc_write(TestDb *, void *, int , void *, int);
           90  +int test_kc_delete(TestDb *, void *, int);
           91  +int test_kc_delete_range(TestDb *, void *, int, void *, int);
           92  +int test_kc_fetch(TestDb *, void *, int, void **, int *);
           93  +int test_kc_scan(TestDb *, void *, int, void *, int, void *, int,
           94  +  void (*)(void *, void *, int , void *, int)
           95  +);
           96  +
           97  +int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
           98  +int test_mdb_close(TestDb *);
           99  +int test_mdb_write(TestDb *, void *, int , void *, int);
          100  +int test_mdb_delete(TestDb *, void *, int);
          101  +int test_mdb_fetch(TestDb *, void *, int, void **, int *);
          102  +int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
          103  +  void (*)(void *, void *, int , void *, int)
          104  +);
          105  +
          106  +/* 
          107  +** Functions in wrapper3.c. This file contains the tdb wrapper for lsm.
          108  +** The wrapper for lsm is a bit more involved than the others, as it 
          109  +** includes code for a couple of different lsm configurations, and for
          110  +** various types of fault injection and robustness testing.
          111  +*/
          112  +int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          113  +int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
          114  +int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
          115  +int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
          116  +int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
          117  +int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
          118  +
          119  +int tdb_lsm_configure(lsm_db *, const char *);
          120  +
          121  +/* Functions in lsmtest_tdb4.c */
          122  +int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          123  +int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          124  +int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          125  +
          126  +
          127  +/* Functions in testutil.c. */
          128  +int  testPrngInit(void);
          129  +u32  testPrngValue(u32 iVal);
          130  +void testPrngArray(u32 iVal, u32 *aOut, int nOut);
          131  +void testPrngString(u32 iVal, char *aOut, int nOut);
          132  +
          133  +void testErrorInit(int argc, char **);
          134  +void testPrintError(const char *zFormat, ...);
          135  +void testPrintUsage(const char *zArgs);
          136  +void testPrintFUsage(const char *zFormat, ...);
          137  +void testTimeInit(void);
          138  +int  testTimeGet(void);
          139  +
          140  +/* Functions in testmem.c. */
          141  +void testMallocInstall(lsm_env *pEnv);
          142  +void testMallocUninstall(lsm_env *pEnv);
          143  +void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *);
          144  +void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *);
          145  +void testMallocOomEnable(lsm_env *pEnv, int);
          146  +
          147  +/* lsmtest.c */
          148  +TestDb *testOpen(const char *zSystem, int, int *pRc);
          149  +void testReopen(TestDb **ppDb, int *pRc);
          150  +void testClose(TestDb **ppDb);
          151  +
          152  +void testFetch(TestDb *, void *, int, void *, int, int *);
          153  +void testWrite(TestDb *, void *, int, void *, int, int *);
          154  +void testDelete(TestDb *, void *, int, int *);
          155  +void testDeleteRange(TestDb *, void *, int, void *, int, int *);
          156  +void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc);
          157  +void testFetchStr(TestDb *, const char *, const char *, int *pRc);
          158  +
          159  +void testBegin(TestDb *pDb, int iTrans, int *pRc);
          160  +void testCommit(TestDb *pDb, int iTrans, int *pRc);
          161  +
          162  +void test_failed(void);
          163  +
          164  +char *testMallocPrintf(const char *zFormat, ...);
          165  +char *testMallocVPrintf(const char *zFormat, va_list ap);
          166  +int testGlobMatch(const char *zPattern, const char *zStr);
          167  +
          168  +void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *);
          169  +void testFetchCompare(TestDb *, TestDb *, void *, int, int *);
          170  +
          171  +void *testMalloc(int);
          172  +void *testMallocCopy(void *pCopy, int nByte);
          173  +void *testRealloc(void *, int);
          174  +void testFree(void *);
          175  +
          176  +/* lsmtest_bt.c */
          177  +int do_bt(int nArg, char **azArg);
          178  +
          179  +/* testio.c */
          180  +int testVfsConfigureDb(TestDb *pDb);
          181  +
          182  +/* testfunc.c */
          183  +int do_show(int nArg, char **azArg);
          184  +int do_work(int nArg, char **azArg);
          185  +
          186  +/* testio.c */
          187  +int do_io(int nArg, char **azArg);
          188  +
          189  +/* lsmtest2.c */
          190  +void do_crash_test(const char *zPattern, int *pRc);
          191  +int do_rollback_test(int nArg, char **azArg);
          192  +
          193  +/* test3.c */
          194  +void test_rollback(const char *zSystem, const char *zPattern, int *pRc);
          195  +
          196  +/* test4.c */
          197  +void test_mc(const char *zSystem, const char *zPattern, int *pRc);
          198  +
          199  +/* test5.c */
          200  +void test_mt(const char *zSystem, const char *zPattern, int *pRc);
          201  +
          202  +/* lsmtest6.c */
          203  +void test_oom(const char *zPattern, int *pRc);
          204  +void testDeleteLsmdb(const char *zFile);
          205  +
          206  +void testSaveDb(const char *zFile, const char *zAuxExt);
          207  +void testRestoreDb(const char *zFile, const char *zAuxExt);
          208  +void testCopyLsmdb(const char *zFrom, const char *zTo);
          209  +
          210  +/* lsmtest7.c */
          211  +void test_api(const char *zPattern, int *pRc);
          212  +
          213  +/* lsmtest8.c */
          214  +void do_writer_crash_test(const char *zPattern, int *pRc);
          215  +
          216  +/*************************************************************************
          217  +** Interface to functionality in test_datasource.c.
          218  +*/
          219  +typedef struct Datasource Datasource;
          220  +typedef struct DatasourceDefn DatasourceDefn;
          221  +
          222  +struct DatasourceDefn {
          223  +  int eType;                      /* A TEST_DATASOURCE_* value */
          224  +  int nMinKey;                    /* Minimum key size */
          225  +  int nMaxKey;                    /* Maximum key size */
          226  +  int nMinVal;                    /* Minimum value size */
          227  +  int nMaxVal;                    /* Maximum value size */
          228  +};
          229  +
          230  +#define TEST_DATASOURCE_RANDOM    1
          231  +#define TEST_DATASOURCE_SEQUENCE  2
          232  +
          233  +char *testDatasourceName(const DatasourceDefn *);
          234  +Datasource *testDatasourceNew(const DatasourceDefn *);
          235  +void testDatasourceFree(Datasource *);
          236  +void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *);
          237  +/* End of test_datasource.c interface.
          238  +*************************************************************************/
          239  +void testDatasourceFetch(
          240  +  TestDb *pDb,                    /* Database handle */
          241  +  Datasource *pData,
          242  +  int iKey,
          243  +  int *pRc                        /* IN/OUT: Error code */
          244  +);
          245  +
          246  +void testWriteDatasource(TestDb *, Datasource *, int, int *);
          247  +void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *);
          248  +void testDeleteDatasource(TestDb *, Datasource *, int, int *);
          249  +void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *);
          250  +
          251  +
          252  +/* test1.c */
          253  +void test_data_1(const char *, const char *, int *pRc);
          254  +void test_data_2(const char *, const char *, int *pRc);
          255  +void test_data_3(const char *, const char *, int *pRc);
          256  +void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *);
          257  +void testCaseProgress(int, int, int, int *);
          258  +int testCaseNDot(void);
          259  +
          260  +void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *);
          261  +int testControlDb(TestDb **ppDb);
          262  +
          263  +typedef struct CksumDb CksumDb;
          264  +CksumDb *testCksumArrayNew(Datasource *, int, int, int);
          265  +char *testCksumArrayGet(CksumDb *, int);
          266  +void testCksumArrayFree(CksumDb *);
          267  +void testCaseStart(int *pRc, char *zFmt, ...);
          268  +void testCaseFinish(int rc);
          269  +void testCaseSkip(void);
          270  +int testCaseBegin(int *, const char *, const char *, ...);
          271  +
          272  +#define TEST_CKSUM_BYTES 29
          273  +int testCksumDatabase(TestDb *pDb, char *zOut);
          274  +int testCountDatabase(TestDb *pDb);
          275  +void testCompareInt(int, int, int *);
          276  +void testCompareStr(const char *z1, const char *z2, int *pRc);
          277  +
          278  +/* lsmtest9.c */
          279  +void test_data_4(const char *, const char *, int *pRc);
          280  +
          281  +
          282  +/*
          283  +** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function.
          284  +*/
          285  +#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z)
          286  +int testArgSelectX(void *, const char *, int, const char *, int *);
          287  +
          288  +#ifdef __cplusplus
          289  +}  /* End of the 'extern "C"' block */
          290  +#endif
          291  +
          292  +#endif

Added ext/lsm1/lsm-test/lsmtest1.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE
            5  +#define DATA_RANDOM     TEST_DATASOURCE_RANDOM
            6  +
            7  +typedef struct Datatest1 Datatest1;
            8  +typedef struct Datatest2 Datatest2;
            9  +
           10  +/*
           11  +** An instance of the following structure contains parameters used to
           12  +** customize the test function in this file. Test procedure:
           13  +**
           14  +**   1. Create a data-source based on the "datasource definition" vars.
           15  +**
           16  +**   2. Insert nRow key value pairs into the database.
           17  +**
           18  +**   3. Delete all keys from the database. Deletes are done in the same 
           19  +**      order as the inserts.
           20  +**
           21  +** During steps 2 and 3 above, after each Datatest1.nVerify inserts or
           22  +** deletes, the following:
           23  +**
           24  +**   a. Run Datasource.nTest key lookups and check the results are as expected.
           25  +**
           26  +**   b. If Datasource.bTestScan is true, run a handful (8) of range
           27  +**      queries (scanning forwards and backwards). Check that the results
           28  +**      are as expected.
           29  +**
           30  +**   c. Close and reopen the database. Then run (a) and (b) again.
           31  +*/
           32  +struct Datatest1 {
           33  +  /* Datasource definition */
           34  +  DatasourceDefn defn;
           35  +
           36  +  /* Test procedure parameters */
           37  +  int nRow;                       /* Number of rows to insert then delete */
           38  +  int nVerify;                    /* How often to verify the db contents */
           39  +  int nTest;                      /* Number of keys to test (0==all) */
           40  +  int bTestScan;                  /* True to do scan tests */
           41  +};
           42  +
           43  +/*
           44  +** An instance of the following data structure is used to describe the
           45  +** second type of test case in this file. The chief difference between 
           46  +** these tests and those described by Datatest1 is that these tests also
           47  +** experiment with range-delete operations. Tests proceed as follows:
           48  +**
           49  +**     1. Open the datasource described by Datatest2.defn. 
           50  +**
           51  +**     2. Open a connection on an empty database.
           52  +**
           53  +**     3. Do this Datatest2.nIter times:
           54  +**
           55  +**        a) Insert Datatest2.nWrite key-value pairs from the datasource.
           56  +**
           57  +**        b) Select two pseudo-random keys and use them as the start
           58  +**           and end points of a range-delete operation.
           59  +**
           60  +**        c) Verify that the contents of the database are as expected (see
           61  +**           below for details).
           62  +**
           63  +**        d) Close and then reopen the database handle.
           64  +**
           65  +**        e) Verify that the contents of the database are still as expected.
           66  +**
           67  +** The inserts and range deletes are run twice - once on the database being
           68  +** tested and once using a control system (sqlite3, kc etc. - something that 
           69  +** works). In order to verify that the contents of the db being tested are
           70  +** correct, the test runs a bunch of scans and lookups on both the test and
           71  +** control databases. If the results are the same, the test passes.
           72  +*/
           73  +struct Datatest2 {
           74  +  DatasourceDefn defn;
           75  +  int nRange;
           76  +  int nWrite;                     /* Number of writes per iteration */
           77  +  int nIter;                      /* Total number of iterations to run */
           78  +};
           79  +
           80  +/*
           81  +** Generate a unique name for the test case pTest with database system
           82  +** zSystem.
           83  +*/
           84  +static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){
           85  +  char *zRet;
           86  +  char *zData;
           87  +  zData = testDatasourceName(&pTest->defn);
           88  +  zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", 
           89  +      zSystem, zData, bRecover, pTest->nRow, pTest->nVerify
           90  +  );
           91  +  testFree(zData);
           92  +  return zRet;
           93  +}
           94  +
           95  +int testControlDb(TestDb **ppDb){
           96  +#ifdef HAVE_KYOTOCABINET
           97  +  return tdb_open("kyotocabinet", "tmp.db", 1, ppDb);
           98  +#else
           99  +  return tdb_open("sqlite3", "", 1, ppDb);
          100  +#endif
          101  +}
          102  +
          103  +void testDatasourceFetch(
          104  +  TestDb *pDb,                    /* Database handle */
          105  +  Datasource *pData,
          106  +  int iKey,
          107  +  int *pRc                        /* IN/OUT: Error code */
          108  +){
          109  +  void *pKey; int nKey;           /* Database key to query for */
          110  +  void *pVal; int nVal;           /* Expected result of query */
          111  +
          112  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          113  +  testFetch(pDb, pKey, nKey, pVal, nVal, pRc);
          114  +}
          115  +
          116  +/*
          117  +** This function is called to test that the contents of database pDb
          118  +** are as expected. In this case, expected is defined as containing
          119  +** key-value pairs iFirst through iLast, inclusive, from data source 
          120  +** pData. In other words, a loop like the following could be used to
          121  +** construct a database with identical contents from scratch.
          122  +**
          123  +**   for(i=iFirst; i<=iLast; i++){
          124  +**     testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
          125  +**     // insert (pKey, nKey) -> (pVal, nVal) into database
          126  +**   }
          127  +**
          128  +** The key domain consists of keys 0 to (nRow-1), inclusive, from
          129  +** data source pData. For both scan and lookup tests, keys are selected
          130  +** pseudo-randomly from within this set.
          131  +**
          132  +** This function runs nLookupTest lookup tests and nScanTest scan tests.
          133  +**
          134  +** A lookup test consists of selecting a key from the domain and querying
          135  +** pDb for it. The test fails if the presence of the key and, if present,
          136  +** the associated value do not match the expectations defined above.
          137  +**
          138  +** A scan test involves selecting a key from the domain and running
          139  +** the following queries:
          140  +**
          141  +**   1. Scan all keys equal to or greater than the key, in ascending order.
          142  +**   2. Scan all keys equal to or smaller than the key, in descending order.
          143  +**
          144  +** Additionally, if nLookupTest is greater than zero, the following are
          145  +** run once:
          146  +**
          147  +**   1. Scan all keys in the db, in ascending order.
          148  +**   2. Scan all keys in the db, in descending order.
          149  +**
          150  +** As you would assume, the test fails if the returned values do not match
          151  +** expectations.
          152  +*/
          153  +void testDbContents(
          154  +  TestDb *pDb,                    /* Database handle being tested */
          155  +  Datasource *pData,              /* pDb contains data from here */
          156  +  int nRow,                       /* Size of key domain */
          157  +  int iFirst,                     /* Index of first key from pData in pDb */
          158  +  int iLast,                      /* Index of last key from pData in pDb */
          159  +  int nLookupTest,                /* Number of lookup tests to run */
          160  +  int nScanTest,                  /* Number of scan tests to run */
          161  +  int *pRc                        /* IN/OUT: Error code */
          162  +){
          163  +  int j;
          164  +  int rc = *pRc;
          165  +
          166  +  if( rc==0 && nScanTest ){
          167  +    TestDb *pDb2 = 0;
          168  +
          169  +    /* Open a control db (i.e. one that we assume works) */
          170  +    rc = testControlDb(&pDb2);
          171  +
          172  +    for(j=iFirst; rc==0 && j<=iLast; j++){
          173  +      void *pKey; int nKey;         /* Database key to insert */
          174  +      void *pVal; int nVal;         /* Database value to insert */
          175  +      testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal);
          176  +      rc = tdb_write(pDb2, pKey, nKey, pVal, nVal);
          177  +    }
          178  +
          179  +    if( rc==0 ){
          180  +      int iKey1;
          181  +      int iKey2;
          182  +      void *pKey1; int nKey1;       /* Start key */
          183  +      void *pKey2; int nKey2;       /* Final key */
          184  +
          185  +      iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow;
          186  +      iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow;
          187  +      testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
          188  +      pKey1 = testMalloc(nKey1+1);
          189  +      memcpy(pKey1, pKey2, nKey1+1);
          190  +      testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
          191  +
          192  +      testScanCompare(pDb2, pDb, 0, 0, 0,         0, 0,         &rc);
          193  +      testScanCompare(pDb2, pDb, 0, 0, 0,         pKey2, nKey2, &rc);
          194  +      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0,         &rc);
          195  +      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc);
          196  +      testScanCompare(pDb2, pDb, 1, 0, 0,         0, 0,         &rc);
          197  +      testScanCompare(pDb2, pDb, 1, 0, 0,         pKey2, nKey2, &rc);
          198  +      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0,         &rc);
          199  +      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc);
          200  +      testFree(pKey1);
          201  +    }
          202  +    tdb_close(pDb2);
          203  +  }
          204  +
          205  +  /* Test some lookups. */
          206  +  for(j=0; rc==0 && j<nLookupTest; j++){
          207  +    int iKey;                     /* Datasource key to test */
          208  +    void *pKey; int nKey;         /* Database key to query for */
          209  +    void *pVal; int nVal;         /* Expected result of query */
          210  +
          211  +    if( nLookupTest>=nRow ){
          212  +      iKey = j;
          213  +    }else{
          214  +      iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow;
          215  +    }
          216  +
          217  +    testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          218  +    if( iFirst>iKey || iKey>iLast ){
          219  +      pVal = 0;
          220  +      nVal = -1;
          221  +    }
          222  +
          223  +    testFetch(pDb, pKey, nKey, pVal, nVal, &rc);
          224  +  }
          225  +
          226  +  *pRc = rc;
          227  +}
          228  +
          229  +/*
          230  +** This function should be called during long running test cases to output
          231  +** the progress dots (...) to stdout.
          232  +*/
          233  +void testCaseProgress(int i, int n, int nDot, int *piDot){
          234  +  int iDot = *piDot;
          235  +  while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){
          236  +    printf(".");
          237  +    fflush(stdout);
          238  +    iDot++;
          239  +  }
          240  +  *piDot = iDot;
          241  +}
          242  +
          243  +int testCaseNDot(void){ return 20; }
          244  +
          245  +#if 0
          246  +static void printScanCb(
          247  +    void *pCtx, void *pKey, int nKey, void *pVal, int nVal
          248  +){
          249  +  printf("%s\n", (char *)pKey);
          250  +  fflush(stdout);
          251  +}
          252  +#endif
          253  +
          254  +void testReopenRecover(TestDb **ppDb, int *pRc){
          255  +  if( *pRc==0 ){
          256  +    const char *zLib = tdb_library_name(*ppDb);
          257  +    const char *zDflt = tdb_default_db(zLib);
          258  +    testCopyLsmdb(zDflt, "bak.db");
          259  +    testClose(ppDb);
          260  +    testCopyLsmdb("bak.db", zDflt);
          261  +    *pRc = tdb_open(zLib, 0, 0, ppDb);
          262  +  }
          263  +}
          264  +
          265  +
          266  +static void doDataTest1(
          267  +  const char *zSystem,            /* Database system to test */
          268  +  int bRecover,
          269  +  Datatest1 *p,                   /* Structure containing test parameters */
          270  +  int *pRc                        /* OUT: Error code */
          271  +){
          272  +  int i;
          273  +  int iDot;
          274  +  int rc = LSM_OK;
          275  +  Datasource *pData;
          276  +  TestDb *pDb;
          277  +
          278  +  /* Start the test case, open a database and allocate the datasource. */
          279  +  pDb = testOpen(zSystem, 1, &rc);
          280  +  pData = testDatasourceNew(&p->defn);
          281  +
          282  +  i = 0;
          283  +  iDot = 0;
          284  +  while( rc==LSM_OK && i<p->nRow ){
          285  +
          286  +    /* Insert some data */
          287  +    testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
          288  +    i += p->nVerify;
          289  +
          290  +    /* Check that the db content is correct. */
          291  +    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
          292  +
          293  +    if( bRecover ){
          294  +      testReopenRecover(&pDb, &rc);
          295  +    }else{
          296  +      testReopen(&pDb, &rc);
          297  +    }
          298  +
          299  +    /* Check that the db content is still correct. */
          300  +    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
          301  +
          302  +    /* Update the progress dots... */
          303  +    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
          304  +  }
          305  +
          306  +  i = 0;
          307  +  iDot = 0;
          308  +  while( rc==LSM_OK && i<p->nRow ){
          309  +
          310  +    /* Delete some entries */
          311  +    testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
          312  +    i += p->nVerify;
          313  +
          314  +    /* Check that the db content is correct. */
          315  +    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
          316  +
          317  +    /* Close and reopen the database. */
          318  +    if( bRecover ){
          319  +      testReopenRecover(&pDb, &rc);
          320  +    }else{
          321  +      testReopen(&pDb, &rc);
          322  +    }
          323  +
          324  +    /* Check that the db content is still correct. */
          325  +    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
          326  +
          327  +    /* Update the progress dots... */
          328  +    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
          329  +  }
          330  +
          331  +  /* Free the datasource, close the database and finish the test case. */
          332  +  testDatasourceFree(pData);
          333  +  tdb_close(pDb);
          334  +  testCaseFinish(rc);
          335  +  *pRc = rc;
          336  +}
          337  +
          338  +
          339  +void test_data_1(
          340  +  const char *zSystem,            /* Database system name */
          341  +  const char *zPattern,           /* Run test cases that match this pattern */
          342  +  int *pRc                        /* IN/OUT: Error code */
          343  +){
          344  +  Datatest1 aTest[] = {
          345  +    { {DATA_RANDOM,     500,600,   1000,2000},     1000,  100,  10,  0},
          346  +    { {DATA_RANDOM,     20,25,     100,200},       1000,  250, 1000, 1},
          347  +    { {DATA_RANDOM,     8,10,      100,200},       1000,  250, 1000, 1},
          348  +    { {DATA_RANDOM,     8,10,      10,20},         1000,  250, 1000, 1},
          349  +    { {DATA_RANDOM,     8,10,      1000,2000},     1000,  250, 1000, 1},
          350  +    { {DATA_RANDOM,     8,100,     10000,20000},    100,   25,  100, 1},
          351  +    { {DATA_RANDOM,     80,100,    10,20},         1000,  250, 1000, 1},
          352  +    { {DATA_RANDOM,     5000,6000, 10,20},          100,   25,  100, 1},
          353  +    { {DATA_SEQUENTIAL, 5,10,      10,20},         1000,  250, 1000, 1},
          354  +    { {DATA_SEQUENTIAL, 5,10,      100,200},       1000,  250, 1000, 1},
          355  +    { {DATA_SEQUENTIAL, 5,10,      1000,2000},     1000,  250, 1000, 1},
          356  +    { {DATA_SEQUENTIAL, 5,100,     10000,20000},    100,   25,  100, 1},
          357  +    { {DATA_RANDOM,     10,10,     100,100},     100000, 1000,  100, 0},
          358  +    { {DATA_SEQUENTIAL, 10,10,     100,100},     100000, 1000,  100, 0},
          359  +  };
          360  +
          361  +  int i;
          362  +  int bRecover;
          363  +
          364  +  for(bRecover=0; bRecover<2; bRecover++){
          365  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          366  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          367  +      char *zName = getName(zSystem, bRecover, &aTest[i]);
          368  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          369  +        doDataTest1(zSystem, bRecover, &aTest[i], pRc);
          370  +      }
          371  +      testFree(zName);
          372  +    }
          373  +  }
          374  +}
          375  +
          376  +void testCompareDb(
          377  +  Datasource *pData,
          378  +  int nData,
          379  +  int iSeed,
          380  +  TestDb *pControl,
          381  +  TestDb *pDb,
          382  +  int *pRc
          383  +){
          384  +  int i;
          385  +
          386  +  static int nCall = 0;
          387  +  nCall++;
          388  +
          389  +  testScanCompare(pControl, pDb, 0, 0, 0,         0, 0,         pRc);
          390  +  testScanCompare(pControl, pDb, 1, 0, 0,         0, 0,         pRc);
          391  +
          392  +  if( *pRc==0 ){
          393  +    int iKey1;
          394  +    int iKey2;
          395  +    void *pKey1; int nKey1;       /* Start key */
          396  +    void *pKey2; int nKey2;       /* Final key */
          397  +
          398  +    iKey1 = testPrngValue(iSeed) % nData;
          399  +    iKey2 = testPrngValue(iSeed+1) % nData;
          400  +    testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
          401  +    pKey1 = testMalloc(nKey1+1);
          402  +    memcpy(pKey1, pKey2, nKey1+1);
          403  +    testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
          404  +
          405  +    testScanCompare(pControl, pDb, 0, 0, 0,         pKey2, nKey2, pRc);
          406  +    testScanCompare(pControl, pDb, 0, pKey1, nKey1, 0, 0,         pRc);
          407  +    testScanCompare(pControl, pDb, 0, pKey1, nKey1, pKey2, nKey2, pRc);
          408  +    testScanCompare(pControl, pDb, 1, 0, 0,         pKey2, nKey2, pRc);
          409  +    testScanCompare(pControl, pDb, 1, pKey1, nKey1, 0, 0,         pRc);
          410  +    testScanCompare(pControl, pDb, 1, pKey1, nKey1, pKey2, nKey2, pRc);
          411  +    testFree(pKey1);
          412  +  }
          413  +
          414  +  for(i=0; i<nData && *pRc==0; i++){
          415  +    void *pKey; int nKey;
          416  +    testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
          417  +    testFetchCompare(pControl, pDb, pKey, nKey, pRc);
          418  +  }
          419  +}
          420  +
          421  +static void doDataTest2(
          422  +  const char *zSystem,            /* Database system to test */
          423  +  int bRecover,
          424  +  Datatest2 *p,                   /* Structure containing test parameters */
          425  +  int *pRc                        /* OUT: Error code */
          426  +){
          427  +  TestDb *pDb;
          428  +  TestDb *pControl;
          429  +  Datasource *pData;
          430  +  int i;
          431  +  int rc = LSM_OK;
          432  +  int iDot = 0;
          433  +
          434  +  /* Start the test case, open a database and allocate the datasource. */
          435  +  pDb = testOpen(zSystem, 1, &rc);
          436  +  pData = testDatasourceNew(&p->defn);
          437  +  rc = testControlDb(&pControl);
          438  +
          439  +  if( tdb_lsm(pDb) ){
          440  +    int nBuf = 32 * 1024 * 1024;
          441  +    lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf);
          442  +  }
          443  +
          444  +  for(i=0; rc==0 && i<p->nIter; i++){
          445  +    void *pKey1; int nKey1;
          446  +    void *pKey2; int nKey2;
          447  +    int ii;
          448  +    int nRange = MIN(p->nIter*p->nWrite, p->nRange);
          449  +
          450  +    for(ii=0; rc==0 && ii<p->nWrite; ii++){
          451  +      int iKey = (i*p->nWrite + ii) % p->nRange;
          452  +      testWriteDatasource(pControl, pData, iKey, &rc);
          453  +      testWriteDatasource(pDb, pData, iKey, &rc);
          454  +    }
          455  +
          456  +    testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0);
          457  +    pKey1 = testMallocCopy(pKey1, nKey1);
          458  +    testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0);
          459  +
          460  +    testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc);
          461  +    testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc);
          462  +    testFree(pKey1);
          463  +
          464  +    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
          465  +    if( bRecover ){
          466  +      testReopenRecover(&pDb, &rc);
          467  +    }else{
          468  +      testReopen(&pDb, &rc);
          469  +    }
          470  +    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
          471  +
          472  +    /* Update the progress dots... */
          473  +    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
          474  +  }
          475  +
          476  +  testClose(&pDb);
          477  +  testClose(&pControl);
          478  +  testDatasourceFree(pData);
          479  +  testCaseFinish(rc);
          480  +  *pRc = rc;
          481  +}
          482  +
          483  +static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){
          484  +  char *zRet;
          485  +  char *zData;
          486  +  zData = testDatasourceName(&pTest->defn);
          487  +  zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", 
          488  +      zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter
          489  +  );
          490  +  testFree(zData);
          491  +  return zRet;
          492  +}
          493  +
          494  +void test_data_2(
          495  +  const char *zSystem,            /* Database system name */
          496  +  const char *zPattern,           /* Run test cases that match this pattern */
          497  +  int *pRc                        /* IN/OUT: Error code */
          498  +){
          499  +  Datatest2 aTest[] = {
          500  +      /* defn,                                 nRange, nWrite, nIter */
          501  +    { {DATA_RANDOM,     20,25,     100,200},   10000,  10,     50   },
          502  +    { {DATA_RANDOM,     20,25,     100,200},   10000,  200,    50   },
          503  +    { {DATA_RANDOM,     20,25,     100,200},   100,    10,     1000 },
          504  +    { {DATA_RANDOM,     20,25,     100,200},   100,    200,    50   },
          505  +  };
          506  +
          507  +  int i;
          508  +  int bRecover;
          509  +
          510  +  for(bRecover=0; bRecover<2; bRecover++){
          511  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          512  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          513  +      char *zName = getName2(zSystem, bRecover, &aTest[i]);
          514  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          515  +        doDataTest2(zSystem, bRecover, &aTest[i], pRc);
          516  +      }
          517  +      testFree(zName);
          518  +    }
          519  +  }
          520  +}
          521  +
          522  +/*************************************************************************
          523  +** Test case data3.*
          524  +*/
          525  +
          526  +typedef struct Datatest3 Datatest3;
          527  +struct Datatest3 {
          528  +  int nRange;                     /* Keys are between 1 and this value, incl. */
          529  +  int nIter;                      /* Number of iterations */
          530  +  int nWrite;                     /* Number of writes per iteration */
          531  +  int nDelete;                    /* Number of deletes per iteration */
          532  +
          533  +  int nValMin;                    /* Minimum value size for writes */
          534  +  int nValMax;                    /* Maximum value size for writes */
          535  +};
          536  +
          537  +void testPutU32(u8 *aBuf, u32 iVal){
          538  +  aBuf[0] = (iVal >> 24) & 0xFF;
          539  +  aBuf[1] = (iVal >> 16) & 0xFF;
          540  +  aBuf[2] = (iVal >>  8) & 0xFF;
          541  +  aBuf[3] = (iVal >>  0) & 0xFF;
          542  +}
          543  +
          544  +void dt3PutKey(u8 *aBuf, int iKey){
          545  +  assert( iKey<100000 && iKey>=0 );
          546  +  sprintf((char *)aBuf, "%.5d", iKey);
          547  +}
          548  +
          549  +static void doDataTest3(
          550  +  const char *zSystem,            /* Database system to test */
          551  +  Datatest3 *p,                   /* Structure containing test parameters */
          552  +  int *pRc                        /* OUT: Error code */
          553  +){
          554  +  int iDot = 0;
          555  +  int rc = *pRc;
          556  +  TestDb *pDb;
          557  +  u8 *abPresent;                  /* Array of boolean */
          558  +  char *aVal;                     /* Buffer to hold values */
          559  +  int i;
          560  +  u32 iSeq = 10;                  /* prng counter */
          561  +
          562  +  abPresent = (u8 *)testMalloc(p->nRange+1);
          563  +  aVal = (char *)testMalloc(p->nValMax+1);
          564  +  pDb = testOpen(zSystem, 1, &rc);
          565  +
          566  +  for(i=0; i<p->nIter && rc==0; i++){
          567  +    int ii;
          568  +
          569  +    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
          570  +
          571  +    /* Perform nWrite inserts */
          572  +    for(ii=0; ii<p->nWrite; ii++){
          573  +      u8 aKey[6];
          574  +      u32 iKey;
          575  +      int nVal;
          576  +
          577  +      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
          578  +      nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin;
          579  +      testPrngString(testPrngValue(iSeq++), aVal, nVal);
          580  +      dt3PutKey(aKey, iKey);
          581  +
          582  +      testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc);
          583  +      abPresent[iKey] = 1;
          584  +    }
          585  +
          586  +    /* Perform nDelete deletes */
          587  +    for(ii=0; ii<p->nDelete; ii++){
          588  +      u8 aKey1[6];
          589  +      u8 aKey2[6];
          590  +      u32 iKey;
          591  +
          592  +      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
          593  +      dt3PutKey(aKey1, iKey-1);
          594  +      dt3PutKey(aKey2, iKey+1);
          595  +
          596  +      testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc);
          597  +      abPresent[iKey] = 0;
          598  +    }
          599  +
          600  +    testReopen(&pDb, &rc);
          601  +
          602  +    for(ii=1; rc==0 && ii<=p->nRange; ii++){
          603  +      int nDbVal;
          604  +      void *pDbVal;
          605  +      u8 aKey[6];
          606  +      int dbrc;
          607  +
          608  +      dt3PutKey(aKey, ii);
          609  +      dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal);
          610  +      testCompareInt(0, dbrc, &rc);
          611  +
          612  +      if( abPresent[ii] ){
          613  +        testCompareInt(1, (nDbVal>0), &rc);
          614  +      }else{
          615  +        testCompareInt(1, (nDbVal<0), &rc);
          616  +      }
          617  +    }
          618  +  }
          619  +
          620  +  testClose(&pDb);
          621  +  testCaseFinish(rc);
          622  +  *pRc = rc;
          623  +}
          624  +
          625  +static char *getName3(const char *zSystem, Datatest3 *p){
          626  +  return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)",
          627  +      zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, 
          628  +      p->nValMin, p->nValMax
          629  +  );
          630  +}
          631  +
          632  +void test_data_3(
          633  +  const char *zSystem,            /* Database system name */
          634  +  const char *zPattern,           /* Run test cases that match this pattern */
          635  +  int *pRc                        /* IN/OUT: Error code */
          636  +){
          637  +  Datatest3 aTest[] = {
          638  +    /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */
          639  +    {  100,    1000,  5,      5,       50,      100 },
          640  +    {  100,    1000,  2,      2,        5,       10 },
          641  +  };
          642  +
          643  +  int i;
          644  +
          645  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          646  +    char *zName = getName3(zSystem, &aTest[i]);
          647  +    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          648  +      doDataTest3(zSystem, &aTest[i], pRc);
          649  +    }
          650  +    testFree(zName);
          651  +  }
          652  +}
          653  +
          654  +

Added ext/lsm1/lsm-test/lsmtest2.c.

            1  +
            2  +/*
            3  +** This file contains tests related to recovery following application 
            4  +** and system crashes (power failures) while writing to the database.
            5  +*/
            6  +
            7  +#include "lsmtest.h"
            8  +
            9  +/*
           10  +** Structure used by testCksumDatabase() to accumulate checksum values in.
           11  +*/
           12  +typedef struct Cksum Cksum;
           13  +struct Cksum {
           14  +  int nRow;
           15  +  int cksum1;
           16  +  int cksum2;
           17  +};
           18  +
           19  +/*
           20  +** tdb_scan() callback used by testCksumDatabase()
           21  +*/
           22  +static void scanCksumDb(
           23  +  void *pCtx, 
           24  +  void *pKey, int nKey,
           25  +  void *pVal, int nVal
           26  +){
           27  +  Cksum *p = (Cksum *)pCtx;
           28  +  int i;
           29  +
           30  +  p->nRow++;
           31  +  for(i=0; i<nKey; i++){
           32  +    p->cksum1 += ((u8 *)pKey)[i];
           33  +    p->cksum2 += p->cksum1;
           34  +  }
           35  +  for(i=0; i<nVal; i++){
           36  +    p->cksum1 += ((u8 *)pVal)[i];
           37  +    p->cksum2 += p->cksum1;
           38  +  }
           39  +}
           40  +
           41  +/*
           42  +** tdb_scan() callback used by testCountDatabase()
           43  +*/
           44  +static void scanCountDb(
           45  +  void *pCtx, 
           46  +  void *pKey, int nKey,
           47  +  void *pVal, int nVal
           48  +){
           49  +  Cksum *p = (Cksum *)pCtx;
           50  +  p->nRow++;
           51  +
           52  +  unused_parameter(pKey);
           53  +  unused_parameter(nKey);
           54  +  unused_parameter(pVal);
           55  +  unused_parameter(nVal);
           56  +}
           57  +
           58  +
           59  +/*
           60  +** Iterate through the entire contents of database pDb. Write a checksum
           61  +** string based on the db contents into buffer zOut before returning. A
           62  +** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size:
           63  +**
           64  +**    * 32-bit integer (10 bytes)
           65  +**    * 1 space        (1 byte)
           66  +**    * 32-bit hex     (8 bytes)
           67  +**    * 1 space        (1 byte)
           68  +**    * 32-bit hex     (8 bytes)
           69  +**    * nul-terminator (1 byte)
           70  +**
           71  +** The number of entries in the database is returned.
           72  +*/
           73  +int testCksumDatabase(
           74  +  TestDb *pDb,                    /* Database handle */
           75  +  char *zOut                      /* Buffer to write checksum to */
           76  +){
           77  +  Cksum cksum;
           78  +  memset(&cksum, 0, sizeof(Cksum));
           79  +  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb);
           80  +  sprintf(zOut, "%d %x %x", 
           81  +      cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2
           82  +  );
           83  +  assert( strlen(zOut)<TEST_CKSUM_BYTES );
           84  +  return cksum.nRow;
           85  +}
           86  +
           87  +int testCountDatabase(TestDb *pDb){
           88  +  Cksum cksum;
           89  +  memset(&cksum, 0, sizeof(Cksum));
           90  +  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCountDb);
           91  +  return cksum.nRow;
           92  +}
           93  +
           94  +/*
           95  +** This function is a no-op if *pRc is not 0 when it is called.
           96  +**
           97  +** Otherwise, the two nul-terminated strings z1 and z1 are compared. If
           98  +** they are the same, the function returns without doing anything. Otherwise,
           99  +** an error message is printed, *pRc is set to 1 and the test_failed()
          100  +** function called.
          101  +*/
          102  +void testCompareStr(const char *z1, const char *z2, int *pRc){
          103  +  if( *pRc==0 ){
          104  +    if( strcmp(z1, z2) ){
          105  +      testPrintError("testCompareStr: \"%s\" != \"%s\"\n", z1, z2);
          106  +      *pRc = 1;
          107  +      test_failed();
          108  +    }
          109  +  }
          110  +}
          111  +
          112  +/*
          113  +** This function is a no-op if *pRc is not 0 when it is called.
          114  +**
          115  +** Otherwise, the two integers i1 and i2 are compared. If they are equal,
          116  +** the function returns without doing anything. Otherwise, an error message 
          117  +** is printed, *pRc is set to 1 and the test_failed() function called.
          118  +*/
          119  +void testCompareInt(int i1, int i2, int *pRc){
          120  +  if( *pRc==0 && i1!=i2 ){
          121  +    testPrintError("testCompareInt: %d != %d\n", i1, i2);
          122  +    *pRc = 1;
          123  +    test_failed();
          124  +  }
          125  +}
          126  +
          127  +void testCaseStart(int *pRc, char *zFmt, ...){
          128  +  va_list ap;
          129  +  va_start(ap, zFmt);
          130  +  vprintf(zFmt, ap);
          131  +  printf(" ...");
          132  +  va_end(ap);
          133  +  *pRc = 0;
          134  +  fflush(stdout);
          135  +}
          136  +
          137  +/*
          138  +** This function is a no-op if *pRc is non-zero when it is called. Zero
          139  +** is returned in this case.
          140  +**
          141  +** Otherwise, the zFmt (a printf style format string) and following arguments 
          142  +** are used to create a test case name. If zPattern is NULL or a glob pattern
          143  +** that matches the test case name, 1 is returned and the test case started.
          144  +** Otherwise, zero is returned and the test case does not start.
          145  +*/
          146  +int testCaseBegin(int *pRc, const char *zPattern, const char *zFmt, ...){
          147  +  int res = 0;
          148  +  if( *pRc==0 ){
          149  +    char *zTest;
          150  +    va_list ap;
          151  +
          152  +    va_start(ap, zFmt);
          153  +    zTest = testMallocVPrintf(zFmt, ap);
          154  +    va_end(ap);
          155  +    if( zPattern==0 || testGlobMatch(zPattern, zTest) ){
          156  +      printf("%-50s ...", zTest);
          157  +      res = 1;
          158  +    }
          159  +    testFree(zTest);
          160  +    fflush(stdout);
          161  +  }
          162  +
          163  +  return res;
          164  +}
          165  +
          166  +void testCaseFinish(int rc){
          167  +  if( rc==0 ){
          168  +    printf("Ok\n");
          169  +  }else{
          170  +    printf("FAILED\n");
          171  +  }
          172  +  fflush(stdout);
          173  +}
          174  +
          175  +void testCaseSkip(){
          176  +  printf("Skipped\n");
          177  +}
          178  +
          179  +void testSetupSavedLsmdb(
          180  +  const char *zCfg,
          181  +  const char *zFile,
          182  +  Datasource *pData,
          183  +  int nRow,
          184  +  int *pRc
          185  +){
          186  +  if( *pRc==0 ){
          187  +    int rc;
          188  +    TestDb *pDb;
          189  +    rc = tdb_lsm_open(zCfg, zFile, 1, &pDb);
          190  +    if( rc==0 ){
          191  +      testWriteDatasourceRange(pDb, pData, 0, nRow, &rc);
          192  +      testClose(&pDb);
          193  +      if( rc==0 ) testSaveDb(zFile, "log");
          194  +    }
          195  +    *pRc = rc;
          196  +  }
          197  +}
          198  +
          199  +/*
          200  +** This function is a no-op if *pRc is non-zero when it is called.
          201  +**
          202  +** Open the LSM database identified by zFile and compute its checksum
          203  +** (a string, as returned by testCksumDatabase()). If the checksum is
          204  +** identical to zExpect1 or, if it is not NULL, zExpect2, the test passes.
          205  +** Otherwise, print an error message and set *pRc to 1.
          206  +*/
          207  +static void testCompareCksumLsmdb(
          208  +  const char *zFile,              /* Path to LSM database */
          209  +  int bCompress,                  /* True if db is compressed */
          210  +  const char *zExpect1,           /* Expected checksum 1 */
          211  +  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
          212  +  int *pRc                        /* IN/OUT: Test case error code */
          213  +){
          214  +  if( *pRc==0 ){
          215  +    char zCksum[TEST_CKSUM_BYTES];
          216  +    TestDb *pDb;
          217  +
          218  +    *pRc = tdb_lsm_open((bCompress?"compression=1 mmap=0":""), zFile, 0, &pDb);
          219  +    testCksumDatabase(pDb, zCksum);
          220  +    testClose(&pDb);
          221  +
          222  +    if( *pRc==0 ){
          223  +      int r1 = 0;
          224  +      int r2 = -1;
          225  +
          226  +      r1 = strcmp(zCksum, zExpect1);
          227  +      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
          228  +      if( r1 && r2 ){
          229  +        if( zExpect2 ){
          230  +          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
          231  +              zCksum, zExpect1, zExpect2
          232  +          );
          233  +        }else{
          234  +          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
          235  +              zCksum, zExpect1
          236  +          );
          237  +        }
          238  +        *pRc = 1;
          239  +        test_failed();
          240  +      }
          241  +    }
          242  +  }
          243  +}
          244  +
          245  +#if 0 /* not used */
          246  +static void testCompareCksumBtdb(
          247  +  const char *zFile,              /* Path to LSM database */
          248  +  const char *zExpect1,           /* Expected checksum 1 */
          249  +  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
          250  +  int *pRc                        /* IN/OUT: Test case error code */
          251  +){
          252  +  if( *pRc==0 ){
          253  +    char zCksum[TEST_CKSUM_BYTES];
          254  +    TestDb *pDb;
          255  +
          256  +    *pRc = tdb_open("bt", zFile, 0, &pDb);
          257  +    testCksumDatabase(pDb, zCksum);
          258  +    testClose(&pDb);
          259  +
          260  +    if( *pRc==0 ){
          261  +      int r1 = 0;
          262  +      int r2 = -1;
          263  +
          264  +      r1 = strcmp(zCksum, zExpect1);
          265  +      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
          266  +      if( r1 && r2 ){
          267  +        if( zExpect2 ){
          268  +          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
          269  +              zCksum, zExpect1, zExpect2
          270  +          );
          271  +        }else{
          272  +          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
          273  +              zCksum, zExpect1
          274  +          );
          275  +        }
          276  +        *pRc = 1;
          277  +        test_failed();
          278  +      }
          279  +    }
          280  +  }
          281  +}
          282  +#endif /* not used */
          283  +
          284  +/* Above this point are reusable test routines. Not clear that they
          285  +** should really be in this file.
          286  +*************************************************************************/
          287  +
          288  +/*
          289  +** This test verifies that if a system crash occurs while doing merge work
          290  +** on the db, no data is lost.
          291  +*/
          292  +static void crash_test1(int bCompress, int *pRc){
          293  +  const char *DBNAME = "testdb.lsm";
          294  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 200, 200};
          295  +
          296  +  const int nRow = 5000;          /* Database size */
          297  +  const int nIter = 200;          /* Number of test iterations */
          298  +  const int nWork = 20;           /* Maximum lsm_work() calls per iteration */
          299  +  const int nPage = 15;           /* Pages per lsm_work call */
          300  +
          301  +  int i;
          302  +  int iDot = 0;
          303  +  Datasource *pData;
          304  +  CksumDb *pCksumDb;
          305  +  TestDb *pDb;
          306  +  char *zCfg;
          307  +
          308  +  const char *azConfig[2] = {
          309  +    "page_size=1024 block_size=65536 autoflush=16384 safety=2 mmap=0", 
          310  +    "page_size=1024 block_size=65536 autoflush=16384 safety=2 "
          311  +    " compression=1 mmap=0"
          312  +  };
          313  +  assert( bCompress==0 || bCompress==1 );
          314  +
          315  +  /* Allocate datasource. And calculate the expected checksums. */
          316  +  pData = testDatasourceNew(&defn);
          317  +  pCksumDb = testCksumArrayNew(pData, nRow, nRow, 1);
          318  +
          319  +  /* Setup and save the initial database. */
          320  +
          321  +  zCfg = testMallocPrintf("%s automerge=7", azConfig[bCompress]);
          322  +  testSetupSavedLsmdb(zCfg, DBNAME, pData, 5000, pRc);
          323  +  testFree(zCfg);
          324  +
          325  +  for(i=0; i<nIter && *pRc==0; i++){
          326  +    int iWork;
          327  +    int testrc = 0;
          328  +
          329  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          330  +
          331  +    /* Restore and open the database. */
          332  +    testRestoreDb(DBNAME, "log");
          333  +    testrc = tdb_lsm_open(azConfig[bCompress], DBNAME, 0, &pDb);
          334  +    assert( testrc==0 );
          335  +
          336  +    /* Call lsm_work() on the db */
          337  +    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nWork*2)));
          338  +    for(iWork=0; testrc==0 && iWork<nWork; iWork++){
          339  +      int nWrite = 0;
          340  +      lsm_db *db = tdb_lsm(pDb);
          341  +      testrc = lsm_work(db, 0, nPage, &nWrite);
          342  +      /* assert( testrc!=0 || nWrite>0 ); */
          343  +      if( testrc==0 ) testrc = lsm_checkpoint(db, 0);
          344  +    }
          345  +    tdb_close(pDb);
          346  +
          347  +    /* Check that the database content is still correct */
          348  +    testCompareCksumLsmdb(DBNAME, 
          349  +        bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc);
          350  +  }
          351  +
          352  +  testCksumArrayFree(pCksumDb);
          353  +  testDatasourceFree(pData);
          354  +}
          355  +
          356  +/*
          357  +** This test verifies that if a system crash occurs while committing a
          358  +** transaction to the log file, no earlier transactions are lost or damaged.
          359  +*/
          360  +static void crash_test2(int bCompress, int *pRc){
          361  +  const char *DBNAME = "testdb.lsm";
          362  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
          363  +
          364  +  const int nIter = 200;
          365  +  const int nInsert = 20;
          366  +
          367  +  int i;
          368  +  int iDot = 0;
          369  +  Datasource *pData;
          370  +  CksumDb *pCksumDb;
          371  +  TestDb *pDb;
          372  +
          373  +  /* Allocate datasource. And calculate the expected checksums. */
          374  +  pData = testDatasourceNew(&defn);
          375  +  pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1);
          376  +
          377  +  /* Setup and save the initial database. */
          378  +  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
          379  +
          380  +  for(i=0; i<nIter && *pRc==0; i++){
          381  +    int iIns;
          382  +    int testrc = 0;
          383  +
          384  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          385  +
          386  +    /* Restore and open the database. */
          387  +    testRestoreDb(DBNAME, "log");
          388  +    testrc = tdb_lsm_open("safety=2", DBNAME, 0, &pDb);
          389  +    assert( testrc==0 );
          390  +
          391  +    /* Insert nInsert records into the database. Crash midway through. */
          392  +    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nInsert+2)));
          393  +    for(iIns=0; iIns<nInsert; iIns++){
          394  +      void *pKey; int nKey;
          395  +      void *pVal; int nVal;
          396  +
          397  +      testDatasourceEntry(pData, 100+iIns, &pKey, &nKey, &pVal, &nVal);
          398  +      testrc = tdb_write(pDb, pKey, nKey, pVal, nVal);
          399  +      if( testrc ) break;
          400  +    }
          401  +    tdb_close(pDb);
          402  +
          403  +    /* Check that no data was lost when the system crashed. */
          404  +    testCompareCksumLsmdb(DBNAME, bCompress,
          405  +      testCksumArrayGet(pCksumDb, 100 + iIns),
          406  +      testCksumArrayGet(pCksumDb, 100 + iIns + 1),
          407  +      pRc
          408  +    );
          409  +  }
          410  +
          411  +  testDatasourceFree(pData);
          412  +  testCksumArrayFree(pCksumDb);
          413  +}
          414  +
          415  +
          416  +/*
          417  +** This test verifies that if a system crash occurs when checkpointing
          418  +** the database, data is not lost (assuming that any writes not synced
          419  +** to the db have been synced into the log file).
          420  +*/
          421  +static void crash_test3(int bCompress, int *pRc){
          422  +  const char *DBNAME = "testdb.lsm";
          423  +  const int nIter = 100;
          424  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
          425  +
          426  +  int i;
          427  +  int iDot = 0;
          428  +  Datasource *pData;
          429  +  CksumDb *pCksumDb;
          430  +  TestDb *pDb;
          431  +
          432  +  /* Allocate datasource. And calculate the expected checksums. */
          433  +  pData = testDatasourceNew(&defn);
          434  +  pCksumDb = testCksumArrayNew(pData, 110, 150, 10);
          435  +
          436  +  /* Setup and save the initial database. */
          437  +  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
          438  +
          439  +  for(i=0; i<nIter && *pRc==0; i++){
          440  +    int iOpen;
          441  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          442  +    testRestoreDb(DBNAME, "log");
          443  +
          444  +    for(iOpen=0; iOpen<5; iOpen++){
          445  +      /* Open the database. Insert 10 more records. */
          446  +      pDb = testOpen("lsm", 0, pRc);
          447  +      testWriteDatasourceRange(pDb, pData, 100+iOpen*10, 10, pRc);
          448  +
          449  +      /* Schedule a crash simulation then close the db. */
          450  +      tdb_lsm_prepare_sync_crash(pDb, 1 + (i%2));
          451  +      tdb_close(pDb);
          452  +
          453  +      /* Open the database and check that the crash did not cause any
          454  +      ** data loss.  */
          455  +      testCompareCksumLsmdb(DBNAME, bCompress,
          456  +        testCksumArrayGet(pCksumDb, 110 + iOpen*10), 0,
          457  +        pRc
          458  +      );
          459  +    }
          460  +  }
          461  +
          462  +  testDatasourceFree(pData);
          463  +  testCksumArrayFree(pCksumDb);
          464  +}
          465  +
          466  +void do_crash_test(const char *zPattern, int *pRc){
          467  +  struct Test {
          468  +    const char *zTest;
          469  +    void (*x)(int, int *);
          470  +    int bCompress;
          471  +  } aTest [] = {
          472  +    { "crash.lsm.1",     crash_test1, 0 },
          473  +#ifdef HAVE_ZLIB
          474  +    { "crash.lsm_zip.1", crash_test1, 1 },
          475  +#endif
          476  +    { "crash.lsm.2",     crash_test2, 0 },
          477  +    { "crash.lsm.3",     crash_test3, 0 },
          478  +  };
          479  +  int i;
          480  +
          481  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          482  +    struct Test *p = &aTest[i];
          483  +    if( testCaseBegin(pRc, zPattern, "%s", p->zTest) ){
          484  +      p->x(p->bCompress, pRc);
          485  +      testCaseFinish(*pRc);
          486  +    }
          487  +  }
          488  +}

Added ext/lsm1/lsm-test/lsmtest3.c.

            1  +
            2  +
            3  +/*
            4  +** This file contains tests related to the explicit rollback of database
            5  +** transactions and sub-transactions.
            6  +*/
            7  +
            8  +
            9  +/*
           10  +** Repeat 2000 times (until the db contains 100,000 entries):
           11  +**
           12  +**   1. Open a transaction and insert 500 rows, opening a nested 
           13  +**      sub-transaction each 100 rows.
           14  +**
           15  +**   2. Roll back to each sub-transaction savepoint. Check the database
           16  +**      checksum looks Ok.
           17  +**
           18  +**   3. Every second iteration, roll back the main transaction. Check the
           19  +**      db checksum is correct. Every other iteration, commit the main
           20  +**      transaction (increasing the size of the db by 100 rows).
           21  +*/
           22  +
           23  +
           24  +#include "lsmtest.h"
           25  +
           26  +struct CksumDb {
           27  +  int nFirst;
           28  +  int nLast;
           29  +  int nStep;
           30  +  char **azCksum;
           31  +};
           32  +
           33  +CksumDb *testCksumArrayNew(
           34  +  Datasource *pData, 
           35  +  int nFirst, 
           36  +  int nLast, 
           37  +  int nStep
           38  +){
           39  +  TestDb *pDb;
           40  +  CksumDb *pRet;
           41  +  int i;
           42  +  int nEntry;
           43  +  int rc = 0;
           44  +
           45  +  assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 );
           46  + 
           47  +  pRet = malloc(sizeof(CksumDb));
           48  +  memset(pRet, 0, sizeof(CksumDb));
           49  +  pRet->nFirst = nFirst;
           50  +  pRet->nLast = nLast;
           51  +  pRet->nStep = nStep;
           52  +  nEntry = 1 + ((nLast - nFirst) / nStep);
           53  +
           54  +  /* Allocate space so that azCksum is an array of nEntry pointers to
           55  +  ** buffers each TEST_CKSUM_BYTES in size.  */
           56  +  pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES));
           57  +  for(i=0; i<nEntry; i++){
           58  +    char *pStart = (char *)(&pRet->azCksum[nEntry]);
           59  +    pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES];
           60  +  }
           61  +
           62  +  tdb_open("lsm", "tempdb.lsm", 1, &pDb);
           63  +  testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc);
           64  +  for(i=0; i<nEntry; i++){
           65  +    testCksumDatabase(pDb, pRet->azCksum[i]);
           66  +    if( i==nEntry ) break;
           67  +    testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc);
           68  +  }
           69  +
           70  +  tdb_close(pDb);
           71  +
           72  +  return pRet;
           73  +}
           74  +
           75  +char *testCksumArrayGet(CksumDb *p, int nRow){
           76  +  int i;
           77  +  assert( nRow>=p->nFirst );
           78  +  assert( nRow<=p->nLast );
           79  +  assert( ((nRow-p->nFirst) % p->nStep)==0 );
           80  +
           81  +  i = (nRow - p->nFirst) / p->nStep;
           82  +  return p->azCksum[i];
           83  +}
           84  +
           85  +void testCksumArrayFree(CksumDb *p){
           86  +  free(p->azCksum);
           87  +  memset(p, 0x55, sizeof(*p));
           88  +  free(p);
           89  +}
           90  +
           91  +/* End of CksumDb code.
           92  +**************************************************************************/
           93  +
           94  +/*
           95  +** Test utility function. Write key-value pair $i from datasource pData 
           96  +** into database pDb.
           97  +*/
           98  +void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
           99  +  void *pKey; int nKey;
          100  +  void *pVal; int nVal;
          101  +  testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
          102  +  testWrite(pDb, pKey, nKey, pVal, nVal, pRc);
          103  +}
          104  +
          105  +/*
          106  +** Test utility function. Delete datasource pData key $i from database pDb.
          107  +*/
          108  +void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
          109  +  void *pKey; int nKey;
          110  +  testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
          111  +  testDelete(pDb, pKey, nKey, pRc);
          112  +}
          113  +
          114  +/*
          115  +** This function inserts nWrite key/value pairs into database pDb - the
          116  +** nWrite key value pairs starting at iFirst from data source pData.
          117  +*/
          118  +void testWriteDatasourceRange(
          119  +  TestDb *pDb,                    /* Database to write to */
          120  +  Datasource *pData,              /* Data source to read values from */
          121  +  int iFirst,                     /* Index of first key/value pair */
          122  +  int nWrite,                     /* Number of key/value pairs to write */
          123  +  int *pRc                        /* IN/OUT: Error code */
          124  +){
          125  +  int i;
          126  +  for(i=0; i<nWrite; i++){
          127  +    testWriteDatasource(pDb, pData, iFirst+i, pRc);
          128  +  }
          129  +}
          130  +
          131  +void testDeleteDatasourceRange(
          132  +  TestDb *pDb,                    /* Database to write to */
          133  +  Datasource *pData,              /* Data source to read keys from */
          134  +  int iFirst,                     /* Index of first key */
          135  +  int nWrite,                     /* Number of keys to delete */
          136  +  int *pRc                        /* IN/OUT: Error code */
          137  +){
          138  +  int i;
          139  +  for(i=0; i<nWrite; i++){
          140  +    testDeleteDatasource(pDb, pData, iFirst+i, pRc);
          141  +  }
          142  +}
          143  +
          144  +static char *getName(const char *zSystem){ 
          145  +  char *zRet; 
          146  +  zRet = testMallocPrintf("rollback.%s", zSystem);
          147  +  return zRet;
          148  +}
          149  +
          150  +static int rollback_test_1(
          151  +  const char *zSystem,
          152  +  Datasource *pData
          153  +){
          154  +  const int nRepeat = 100;
          155  +
          156  +  TestDb *pDb;
          157  +  int rc;
          158  +  int i;
          159  +  CksumDb *pCksum;
          160  +  char *zName;
          161  +
          162  +  zName = getName(zSystem);
          163  +  testCaseStart(&rc, zName);
          164  +  testFree(zName);
          165  +
          166  +  pCksum = testCksumArrayNew(pData, 0, nRepeat*100, 100);
          167  +  pDb = 0;
          168  +  rc = tdb_open(zSystem, 0, 1, &pDb);
          169  +  if( pDb && tdb_transaction_support(pDb)==0 ){
          170  +    testCaseSkip();
          171  +    goto skip_rollback_test;
          172  +  }
          173  +
          174  +  for(i=0; i<nRepeat && rc==0; i++){
          175  +    char zCksum[TEST_CKSUM_BYTES];
          176  +    int nCurrent = (((i+1)/2) * 100);
          177  +    int nDbRow;
          178  +    int iTrans;
          179  +
          180  +    /* Check that the database is the expected size. */
          181  +    nDbRow = testCountDatabase(pDb);
          182  +    testCompareInt(nCurrent, nDbRow, &rc);
          183  +
          184  +    for(iTrans=2; iTrans<=6 && rc==0; iTrans++){
          185  +      tdb_begin(pDb, iTrans);
          186  +      testWriteDatasourceRange(pDb, pData, nCurrent, 100, &rc);
          187  +      nCurrent += 100;
          188  +    }
          189  +
          190  +    testCksumDatabase(pDb, zCksum);
          191  +    testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          192  +
          193  +    for(iTrans=6; iTrans>2 && rc==0; iTrans--){
          194  +      tdb_rollback(pDb, iTrans);
          195  +      nCurrent -= 100;
          196  +      testCksumDatabase(pDb, zCksum);
          197  +      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          198  +    }
          199  +
          200  +    if( i%2 ){
          201  +      tdb_rollback(pDb, 0);
          202  +      nCurrent -= 100;
          203  +      testCksumDatabase(pDb, zCksum);
          204  +      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          205  +    }else{
          206  +      tdb_commit(pDb, 0);
          207  +    }
          208  +  }
          209  +  testCaseFinish(rc);
          210  +
          211  + skip_rollback_test:
          212  +  tdb_close(pDb);
          213  +  testCksumArrayFree(pCksum);
          214  +  return rc;
          215  +}
          216  +
          217  +void test_rollback(
          218  +  const char *zSystem, 
          219  +  const char *zPattern, 
          220  +  int *pRc
          221  +){
          222  +  if( *pRc==0 ){
          223  +    int bRun = 1;
          224  +
          225  +    if( zPattern ){
          226  +      char *zName = getName(zSystem);
          227  +      bRun = testGlobMatch(zPattern, zName);
          228  +      testFree(zName);
          229  +    }
          230  +
          231  +    if( bRun ){
          232  +      DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 };
          233  +      Datasource *pData = testDatasourceNew(&defn);
          234  +      *pRc = rollback_test_1(zSystem, pData);
          235  +      testDatasourceFree(pData);
          236  +    }
          237  +  }
          238  +}

Added ext/lsm1/lsm-test/lsmtest4.c.

            1  +
            2  +/*
            3  +** This file contains test cases involving multiple database clients.
            4  +*/
            5  +
            6  +#include "lsmtest.h"
            7  +
            8  +/*
            9  +** The following code implements test cases "mc1.*".
           10  +**
           11  +** This test case uses one writer and $nReader readers. All connections
           12  +** are driven by a single thread. All connections are opened at the start
           13  +** of the test and remain open until the test is finished.
           14  +**
           15  +** The test consists of $nStep steps. Each step the following is performed:
           16  +**
           17  +**   1. The writer inserts $nWriteStep records into the db.
           18  +**
           19  +**   2. The writer checks that the contents of the db are as expected.
           20  +**
           21  +**   3. Each reader that currently has an open read transaction also checks
           22  +**      that the contents of the db are as expected (according to the snapshot
           23  +**      the read transaction is reading - see below).
           24  +**
           25  +** After step 1, reader 1 opens a read transaction. After step 2, reader
           26  +** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1
           27  +** closes the current read transaction and opens a new one. And so on.
           28  +** The result is that at step N (for N > $nReader), there exists a reader
           29  +** with an open read transaction reading the snapshot committed following
           30  +** steps (N-$nReader-1) to N. 
           31  +*/
           32  +typedef struct Mctest Mctest;
           33  +struct Mctest {
           34  +  DatasourceDefn defn;            /* Datasource to use */
           35  +  int nStep;                      /* Total number of steps in test */
           36  +  int nWriteStep;                 /* Number of rows to insert each step */
           37  +  int nReader;                    /* Number of read connections */
           38  +};
           39  +static void do_mc_test(
           40  +  const char *zSystem,            /* Database system to test */
           41  +  Mctest *pTest,
           42  +  int *pRc                        /* IN/OUT: return code */
           43  +){
           44  +  const int nDomain = pTest->nStep * pTest->nWriteStep;
           45  +  Datasource *pData;              /* Source of data */
           46  +  TestDb *pDb;                    /* First database connection (writer) */
           47  +  int iReader;                    /* Used to iterate through aReader */
           48  +  int iStep;                      /* Current step in test */
           49  +  int iDot = 0;                   /* Current step in test */
           50  +
           51  +  /* Array of reader connections */
           52  +  struct Reader {
           53  +    TestDb *pDb;                  /* Connection handle */
           54  +    int iLast;                    /* Current snapshot contains keys 0..iLast */
           55  +  } *aReader;
           56  +
           57  +  /* Create a data source */
           58  +  pData = testDatasourceNew(&pTest->defn);
           59  +
           60  +  /* Open the writer connection */
           61  +  pDb = testOpen(zSystem, 1, pRc);
           62  +
           63  +  /* Allocate aReader */
           64  +  aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader);
           65  +  for(iReader=0; iReader<pTest->nReader; iReader++){
           66  +    aReader[iReader].pDb = testOpen(zSystem, 0, pRc);
           67  +  }
           68  +
           69  +  for(iStep=0; iStep<pTest->nStep; iStep++){
           70  +    int iLast;
           71  +    int iBegin;                   /* Start read trans using aReader[iBegin] */
           72  +
           73  +    /* Insert nWriteStep more records into the database */
           74  +    int iFirst = iStep*pTest->nWriteStep;
           75  +    testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc);
           76  +
           77  +    /* Check that the db is Ok according to the writer */
           78  +    iLast = (iStep+1) * pTest->nWriteStep - 1;
           79  +    testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc);
           80  +
           81  +    /* Have reader (iStep % nReader) open a read transaction here. */
           82  +    iBegin = (iStep % pTest->nReader);
           83  +    if( iBegin<iStep ) tdb_commit(aReader[iBegin].pDb, 0);
           84  +    tdb_begin(aReader[iBegin].pDb, 1);
           85  +    aReader[iBegin].iLast = iLast;
           86  +
           87  +    /* Check that the db is Ok for each open reader */
           88  +    for(iReader=0; iReader<pTest->nReader && aReader[iReader].iLast; iReader++){
           89  +      iLast = aReader[iReader].iLast;
           90  +      testDbContents(
           91  +          aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc
           92  +      );
           93  +    }
           94  +
           95  +    /* Report progress */
           96  +    testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot);
           97  +  }
           98  +
           99  +  /* Close all readers */
          100  +  for(iReader=0; iReader<pTest->nReader; iReader++){
          101  +    testClose(&aReader[iReader].pDb);
          102  +  }
          103  +  testFree(aReader);
          104  +
          105  +  /* Close the writer-connection and free the datasource */
          106  +  testClose(&pDb);
          107  +  testDatasourceFree(pData);
          108  +}
          109  +
          110  +
          111  +void test_mc(
          112  +  const char *zSystem,            /* Database system name */
          113  +  const char *zPattern,           /* Run test cases that match this pattern */
          114  +  int *pRc                        /* IN/OUT: Error code */
          115  +){
          116  +  int i;
          117  +  Mctest aTest[] = {
          118  +    { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 },
          119  +  };
          120  +
          121  +  for(i=0; i<ArraySize(aTest); i++){
          122  +    if( testCaseBegin(pRc, zPattern, "mc1.%s.%d", zSystem, i) ){
          123  +      do_mc_test(zSystem, &aTest[i], pRc);
          124  +      testCaseFinish(*pRc);
          125  +    }
          126  +  }
          127  +}

Added ext/lsm1/lsm-test/lsmtest5.c.

            1  +
            2  +/*
            3  +** This file is broken into three semi-autonomous parts:
            4  +**
            5  +**   1. The database functions.
            6  +**   2. The thread wrappers.
            7  +**   3. The implementation of the mt1.* tests.
            8  +*/
            9  +
           10  +/*************************************************************************
           11  +** DATABASE CONTENTS:
           12  +**
           13  +**   The database contains up to N key/value pairs, where N is some large 
           14  +**   number (say 10,000,000). Keys are integer values between 0 and (N-1).
           15  +**   The value associated with each key is a pseudo-random blob of data.
           16  +**
           17  +**   Key/value pair keys are encoded as the two bytes "k." followed by a 
           18  +**   10-digit decimal number. i.e. key 45 -> "k.0000000045".
           19  +**
           20  +**   As well as the key/value pairs, the database also contains checksum 
           21  +**   entries. The checksums form a hierarchy - for every F key/value
           22  +**   entries there is one level 1 checksum. And for each F level 1 checksums
           23  +**   there is one level 2 checksum. And so on.
           24  +**
           25  +**   Checksum keys are encoded as the two byte "c." followed by the 
           26  +**   checksum level, followed by a 10 digit decimal number containing
           27  +**   the value of the first key that contributes to the checksum value.
           28  +**   For example, assuming F==10, the level 1 checksum that spans keys
           29  +**   10 to 19 is "c.1.0000000010".
           30  +**
           31  +**   Clients may perform one of two operations on the database: a read
           32  +**   or a write.
           33  +** 
           34  +** READ OPERATIONS:
           35  +**
           36  +**   A read operation scans a range of F key/value pairs. It computes
           37  +**   the expected checksum and then compares the computed value to the
           38  +**   actual value stored in the level 1 checksum entry. It then scans 
           39  +**   the group of F level 1 checksums, and compares the computed checksum 
           40  +**   to the associated level 2 checksum value, and so on until the 
           41  +**   highest level checksum value has been verified.
           42  +**
           43  +**   If a checksum ever fails to match the expected value, the test 
           44  +**   has failed.
           45  +**
           46  +** WRITE OPERATIONS:
           47  +**
           48  +**   A write operation involves writing (possibly clobbering) a single
           49  +**   key/value pair. The associated level 1 checksum is then recalculated
           50  +**   updated. Then the level 2 checksum, and so on until the highest
           51  +**   level checksum has been modified.
           52  +**
           53  +**   All updates occur inside a single transaction.
           54  +**
           55  +** INTERFACE:
           56  +**
           57  +**   The interface used by test cases to read and write the db consists
           58  +**   of type DbParameters and the following functions:
           59  +**
           60  +**       dbReadOperation()
           61  +**       dbWriteOperation()
           62  +*/
           63  +
           64  +#include "lsmtest.h"
           65  +
           66  +typedef struct DbParameters DbParameters;
           67  +struct DbParameters {
           68  +  int nFanout;                    /* Checksum fanout (F) */
           69  +  int nKey;                       /* Size of key space (N) */
           70  +};
           71  +
           72  +#define DB_KEY_BYTES          (2+5+10+1)
           73  +
           74  +/*
           75  +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
           76  +** This function populates the buffer with a nul-terminated key string 
           77  +** corresponding to key iKey.
           78  +*/
           79  +static void dbFormatKey(
           80  +  DbParameters *pParam,
           81  +  int iLevel,
           82  +  int iKey,                       /* Key value */
           83  +  char *aBuf                      /* Write key string here */
           84  +){
           85  +  if( iLevel==0 ){
           86  +    snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey);
           87  +  }else{
           88  +    int f = 1;
           89  +    int i;
           90  +    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
           91  +    snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f));
           92  +  }
           93  +}
           94  +
           95  +/*
           96  +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
           97  +** This function populates the buffer with the string representation of
           98  +** checksum value iVal.
           99  +*/
          100  +static void dbFormatCksumValue(u32 iVal, char *aBuf){
          101  +  snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal);
          102  +}
          103  +
          104  +/*
          105  +** Return the highest level of checksum in the database described
          106  +** by *pParam.
          107  +*/
          108  +static int dbMaxLevel(DbParameters *pParam){
          109  +  int iMax;
          110  +  int n = 1;
          111  +  for(iMax=0; n<pParam->nKey; iMax++){
          112  +    n = n * pParam->nFanout;
          113  +  }
          114  +  return iMax;
          115  +}
          116  +
          117  +static void dbCksum(
          118  +  void *pCtx,                     /* IN/OUT: Pointer to u32 containing cksum */
          119  +  void *pKey, int nKey,           /* Database key. Unused. */
          120  +  void *pVal, int nVal            /* Database value. Checksum this. */
          121  +){
          122  +  u8 *aVal = (u8 *)pVal;
          123  +  u32 *pCksum = (u32 *)pCtx;
          124  +  u32 cksum = *pCksum;
          125  +  int i;
          126  +
          127  +  unused_parameter(pKey);
          128  +  unused_parameter(nKey);
          129  +
          130  +  for(i=0; i<nVal; i++){
          131  +    cksum += (cksum<<3) + (int)aVal[i];
          132  +  }
          133  +
          134  +  *pCksum = cksum;
          135  +}
          136  +
          137  +/*
          138  +** Compute the value of the checksum stored on level iLevel that contains
          139  +** data from key iKey by scanning the pParam->nFanout entries at level 
          140  +** iLevel-1.
          141  +*/
          142  +static u32 dbComputeCksum(
          143  +  DbParameters *pParam,           /* Database parameters */
          144  +  TestDb *pDb,                    /* Database connection handle */
          145  +  int iLevel,                     /* Level of checksum to compute */
          146  +  int iKey,                       /* Compute checksum for this key */
          147  +  int *pRc                        /* IN/OUT: Error code */
          148  +){
          149  +  u32 cksum = 0;
          150  +  if( *pRc==0 ){
          151  +    int nFirst;
          152  +    int nLast;
          153  +    int iFirst = 0;
          154  +    int iLast = 0;
          155  +    int i;
          156  +    int f = 1;
          157  +    char zFirst[DB_KEY_BYTES];
          158  +    char zLast[DB_KEY_BYTES];
          159  +
          160  +    assert( iLevel>=1 );
          161  +    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
          162  +
          163  +    iFirst = f*(iKey/f);
          164  +    iLast = iFirst + f - 1;
          165  +    dbFormatKey(pParam, iLevel-1, iFirst, zFirst);
          166  +    dbFormatKey(pParam, iLevel-1, iLast, zLast);
          167  +    nFirst = strlen(zFirst);
          168  +    nLast = strlen(zLast);
          169  +
          170  +    *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum);
          171  +  }
          172  +
          173  +  return cksum;
          174  +}
          175  +
          176  +static void dbReadOperation(
          177  +  DbParameters *pParam,           /* Database parameters */
          178  +  TestDb *pDb,                    /* Database connection handle */
          179  +  void (*xDelay)(void *),
          180  +  void *pDelayCtx,
          181  +  int iKey,                       /* Key to read */
          182  +  int *pRc                        /* IN/OUT: Error code */
          183  +){
          184  +  const int iMax = dbMaxLevel(pParam);
          185  +  int i;
          186  +
          187  +  if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc);
          188  +  for(i=1; *pRc==0 && i<=iMax; i++){
          189  +    char zCksum[DB_KEY_BYTES];
          190  +    char zKey[DB_KEY_BYTES];
          191  +    u32 iCksum = 0;
          192  +
          193  +    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
          194  +    if( iCksum ){
          195  +      if( xDelay && i==1 ) xDelay(pDelayCtx);
          196  +      dbFormatCksumValue(iCksum, zCksum);
          197  +      dbFormatKey(pParam, i, iKey, zKey);
          198  +      testFetchStr(pDb, zKey, zCksum, pRc);
          199  +    }
          200  +  }
          201  +  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
          202  +}
          203  +
          204  +static int dbWriteOperation(
          205  +  DbParameters *pParam,           /* Database parameters */
          206  +  TestDb *pDb,                    /* Database connection handle */
          207  +  int iKey,                       /* Key to write to */
          208  +  const char *zValue,             /* Nul-terminated value to write */
          209  +  int *pRc                        /* IN/OUT: Error code */
          210  +){
          211  +  const int iMax = dbMaxLevel(pParam);
          212  +  char zKey[DB_KEY_BYTES];
          213  +  int i;
          214  +  int rc;
          215  +
          216  +  assert( iKey>=0 && iKey<pParam->nKey );
          217  +  dbFormatKey(pParam, 0, iKey, zKey);
          218  +
          219  +  /* Open a write transaction. This may fail - SQLITE4_BUSY */
          220  +  if( *pRc==0 && tdb_transaction_support(pDb) ){
          221  +    rc = tdb_begin(pDb, 2);
          222  +    if( rc==5 ) return 0;
          223  +    *pRc = rc;
          224  +  }
          225  +
          226  +  testWriteStr(pDb, zKey, zValue, pRc);
          227  +  for(i=1; i<=iMax; i++){
          228  +    char zCksum[DB_KEY_BYTES];
          229  +    u32 iCksum = 0;
          230  +
          231  +    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
          232  +    dbFormatCksumValue(iCksum, zCksum);
          233  +    dbFormatKey(pParam, i, iKey, zKey);
          234  +    testWriteStr(pDb, zKey, zCksum, pRc);
          235  +  }
          236  +  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
          237  +  return 1;
          238  +}
          239  +
          240  +/*************************************************************************
          241  +** The following block contains testXXX() functions that implement a
          242  +** wrapper around the systems native multi-thread support. There are no
          243  +** synchronization primitives - just functions to launch and join 
          244  +** threads. Wrapper functions are:
          245  +**
          246  +**    testThreadSupport()
          247  +**
          248  +**    testThreadInit()
          249  +**    testThreadShutdown()
          250  +**    testThreadLaunch()
          251  +**    testThreadWait()
          252  +**
          253  +**    testThreadSetHalt()
          254  +**    testThreadGetHalt()
          255  +**    testThreadSetResult()
          256  +**    testThreadGetResult()
          257  +**
          258  +**    testThreadEnterMutex()
          259  +**    testThreadLeaveMutex()
          260  +*/
          261  +typedef struct ThreadSet ThreadSet;
          262  +#ifdef LSM_MUTEX_PTHREADS
          263  +
          264  +#include <pthread.h>
          265  +#include <unistd.h>
          266  +
          267  +typedef struct Thread Thread;
          268  +struct Thread {
          269  +  int rc;
          270  +  char *zMsg;
          271  +  pthread_t id;
          272  +  void (*xMain)(ThreadSet *, int, void *);
          273  +  void *pCtx;
          274  +  ThreadSet *pThreadSet;
          275  +};
          276  +
          277  +struct ThreadSet {
          278  +  int bHalt;                      /* Halt flag */
          279  +  int nThread;                    /* Number of threads */
          280  +  Thread *aThread;                /* Array of Thread structures */
          281  +  pthread_mutex_t mutex;          /* Mutex used for cheating */
          282  +};
          283  +
          284  +/*
          285  +** Return true if this build supports threads, or false otherwise. If
          286  +** this function returns false, no other testThreadXXX() functions should
          287  +** be called.
          288  +*/
          289  +static int testThreadSupport(){ return 1; }
          290  +
          291  +/*
          292  +** Allocate and return a thread-set handle with enough space allocated
          293  +** to handle up to nMax threads. Each call to this function should be
          294  +** matched by a call to testThreadShutdown() to delete the object.
          295  +*/
          296  +static ThreadSet *testThreadInit(int nMax){
          297  +  int nByte;                      /* Total space to allocate */
          298  +  ThreadSet *p;                   /* Return value */
          299  +
          300  +  nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax;
          301  +  p = (ThreadSet *)testMalloc(nByte);
          302  +  p->nThread = nMax;
          303  +  p->aThread = (Thread *)&p[1];
          304  +  pthread_mutex_init(&p->mutex, 0);
          305  +
          306  +  return p;
          307  +}
          308  +
          309  +/*
          310  +** Delete a thread-set object and release all resources held by it.
          311  +*/
          312  +static void testThreadShutdown(ThreadSet *p){
          313  +  int i;
          314  +  for(i=0; i<p->nThread; i++){
          315  +    testFree(p->aThread[i].zMsg);
          316  +  }
          317  +  pthread_mutex_destroy(&p->mutex);
          318  +  testFree(p);
          319  +}
          320  +
          321  +static void *ttMain(void *pArg){
          322  +  Thread *pThread = (Thread *)pArg;
          323  +  int iThread;
          324  +  iThread = (pThread - pThread->pThreadSet->aThread);
          325  +  pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx);
          326  +  return 0;
          327  +}
          328  +
          329  +/*
          330  +** Launch a new thread.
          331  +*/
          332  +static int testThreadLaunch(
          333  +  ThreadSet *p,
          334  +  int iThread,
          335  +  void (*xMain)(ThreadSet *, int, void *),
          336  +  void *pCtx
          337  +){
          338  +  int rc;
          339  +  Thread *pThread;
          340  +
          341  +  assert( iThread>=0 && iThread<p->nThread );
          342  +
          343  +  pThread = &p->aThread[iThread];
          344  +  assert( pThread->pThreadSet==0 );
          345  +  pThread->xMain = xMain;
          346  +  pThread->pCtx = pCtx;
          347  +  pThread->pThreadSet = p;
          348  +  rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread);
          349  +
          350  +  return rc;
          351  +}
          352  +
          353  +/*
          354  +** Set the thread-set "halt" flag.
          355  +*/
          356  +static void testThreadSetHalt(ThreadSet *pThreadSet){
          357  +  pThreadSet->bHalt = 1;
          358  +}
          359  +
          360  +/*
          361  +** Return the current value of the thread-set "halt" flag.
          362  +*/
          363  +static int testThreadGetHalt(ThreadSet *pThreadSet){
          364  +  return pThreadSet->bHalt;
          365  +}
          366  +
          367  +static void testThreadSleep(ThreadSet *pThreadSet, int nMs){
          368  +  int nRem = nMs;
          369  +  while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){
          370  +    usleep(50000);
          371  +    nRem -= 50;
          372  +  }
          373  +}
          374  +
          375  +/*
          376  +** Wait for all threads launched to finish before returning. If nMs
          377  +** is greater than zero, set the "halt" flag to tell all threads
          378  +** to halt after waiting nMs milliseconds.
          379  +*/
          380  +static void testThreadWait(ThreadSet *pThreadSet, int nMs){
          381  +  int i;
          382  +
          383  +  testThreadSleep(pThreadSet, nMs);
          384  +  testThreadSetHalt(pThreadSet);
          385  +  for(i=0; i<pThreadSet->nThread; i++){
          386  +    Thread *pThread = &pThreadSet->aThread[i];
          387  +    if( pThread->xMain ){
          388  +      pthread_join(pThread->id, 0);
          389  +    }
          390  +  }
          391  +}
          392  +
          393  +/*
          394  +** Set the result for thread iThread. 
          395  +*/
          396  +static void testThreadSetResult(
          397  +  ThreadSet *pThreadSet,          /* Thread-set handle */
          398  +  int iThread,                    /* Set result for this thread */
          399  +  int rc,                         /* Result error code */
          400  +  char *zFmt,                     /* Result string format */
          401  +  ...                             /* Result string formatting args... */
          402  +){
          403  +  va_list ap;
          404  +
          405  +  testFree(pThreadSet->aThread[iThread].zMsg);
          406  +  pThreadSet->aThread[iThread].rc = rc;
          407  +  pThreadSet->aThread[iThread].zMsg = 0;
          408  +  if( zFmt ){
          409  +    va_start(ap, zFmt);
          410  +    pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap);
          411  +    va_end(ap);
          412  +  }
          413  +}
          414  +
          415  +/*
          416  +** Retrieve the result for thread iThread. 
          417  +*/
          418  +static int testThreadGetResult(
          419  +  ThreadSet *pThreadSet,          /* Thread-set handle */
          420  +  int iThread,                    /* Get result for this thread */
          421  +  const char **pzRes              /* OUT: Pointer to result string */
          422  +){
          423  +  if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg;
          424  +  return pThreadSet->aThread[iThread].rc;
          425  +}
          426  +
          427  +/*
          428  +** Enter and leave the test case mutex.
          429  +*/
          430  +#if 0
          431  +static void testThreadEnterMutex(ThreadSet *p){
          432  +  pthread_mutex_lock(&p->mutex);
          433  +}
          434  +static void testThreadLeaveMutex(ThreadSet *p){
          435  +  pthread_mutex_unlock(&p->mutex);
          436  +}
          437  +#endif
          438  +#endif
          439  +
          440  +#if !defined(LSM_MUTEX_PTHREADS)
          441  +static int testThreadSupport(){ return 0; }
          442  +
          443  +#define testThreadInit(a) 0
          444  +#define testThreadShutdown(a)
          445  +#define testThreadLaunch(a,b,c,d) 0
          446  +#define testThreadWait(a,b)
          447  +#define testThreadSetHalt(a)
          448  +#define testThreadGetHalt(a) 0
          449  +#define testThreadGetResult(a,b,c) 0
          450  +#define testThreadSleep(a,b) 0
          451  +
          452  +static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){
          453  +  unused_parameter(a);
          454  +  unused_parameter(b);
          455  +  unused_parameter(c);
          456  +  unused_parameter(d);
          457  +}
          458  +#endif
          459  +/* End of threads wrapper.
          460  +*************************************************************************/
          461  +
          462  +/*************************************************************************
          463  +** Below this point is the third part of this file - the implementation
          464  +** of the mt1.* tests.
          465  +*/
          466  +typedef struct Mt1Test Mt1Test;
          467  +struct Mt1Test {
          468  +  DbParameters param;             /* Description of database to read/write */
          469  +  int nReadwrite;                 /* Number of read/write threads */
          470  +  int nFastReader;                /* Number of fast reader threads */
          471  +  int nSlowReader;                /* Number of slow reader threads */
          472  +  int nMs;                        /* How long to run for */
          473  +  const char *zSystem;            /* Database system to test */
          474  +};
          475  +
          476  +typedef struct Mt1DelayCtx Mt1DelayCtx;
          477  +struct Mt1DelayCtx {
          478  +  ThreadSet *pSet;                /* Threadset to sleep within */
          479  +  int nMs;                        /* Sleep in ms */
          480  +};
          481  +
          482  +static void xMt1Delay(void *pCtx){
          483  +  Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx;
          484  +  testThreadSleep(p->pSet, p->nMs);
          485  +}
          486  +
          487  +#define MT1_THREAD_RDWR 0
          488  +#define MT1_THREAD_SLOW 1
          489  +#define MT1_THREAD_FAST 2
          490  +
          491  +static void xMt1Work(lsm_db *pDb, void *pCtx){
          492  +#if 0
          493  +  char *z = 0;
          494  +  lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z);
          495  +  printf("%s\n", z);
          496  +  fflush(stdout);
          497  +#endif
          498  +}
          499  +
          500  +/*
          501  +** This is the main() proc for all threads in test case "mt1".
          502  +*/
          503  +static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){
          504  +  Mt1Test *p = (Mt1Test *)pCtx;   /* Test parameters */
          505  +  Mt1DelayCtx delay;
          506  +  int nRead = 0;                  /* Number of calls to dbReadOperation() */
          507  +  int nWrite = 0;                 /* Number of completed database writes */
          508  +  int rc = 0;                     /* Error code */
          509  +  int iPrng;                      /* Prng argument variable */
          510  +  TestDb *pDb;                    /* Database handle */
          511  +  int eType;
          512  +
          513  +  delay.pSet = pThreadSet;
          514  +  delay.nMs = 0;
          515  +  if( iThread<p->nReadwrite ){
          516  +    eType = MT1_THREAD_RDWR;
          517  +  }else if( iThread<(p->nReadwrite+p->nFastReader) ){
          518  +    eType = MT1_THREAD_FAST;
          519  +  }else{
          520  +    eType = MT1_THREAD_SLOW;
          521  +    delay.nMs = (p->nMs / 20);
          522  +  }
          523  +
          524  +  /* Open a new database connection. Initialize the pseudo-random number
          525  +  ** argument based on the thread number.  */
          526  +  iPrng = testPrngValue(iThread);
          527  +  pDb = testOpen(p->zSystem, 0, &rc);
          528  +
          529  +  if( rc==0 ){
          530  +    tdb_lsm_config_work_hook(pDb, xMt1Work, 0);
          531  +  }
          532  +
          533  +  /* Loop until either an error occurs or some other thread sets the
          534  +  ** halt flag.  */
          535  +  while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){
          536  +    int iKey;
          537  +
          538  +    /* Perform a read operation on an arbitrarily selected key. */
          539  +    iKey = (testPrngValue(iPrng++) % p->param.nKey);
          540  +    dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc);
          541  +    if( rc ) continue;
          542  +    nRead++;
          543  +
          544  +    /* Attempt to write an arbitrary key value pair (and update the associated
          545  +    ** checksum entries). dbWriteOperation() returns 1 if the write is
          546  +    ** successful, or 0 if it failed with an LSM_BUSY error.  */
          547  +    if( eType==MT1_THREAD_RDWR ){
          548  +      char aValue[50];
          549  +      char aRnd[25];
          550  +
          551  +      iKey = (testPrngValue(iPrng++) % p->param.nKey);
          552  +      testPrngString(iPrng, aRnd, sizeof(aRnd));
          553  +      iPrng += sizeof(aRnd);
          554  +      snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd);
          555  +      nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc);
          556  +    }
          557  +  }
          558  +  testClose(&pDb);
          559  +
          560  +  /* If an error has occured, set the thread error code and the threadset 
          561  +  ** halt flag to tell the other test threads to halt. Otherwise, set the
          562  +  ** thread error code to 0 and post a message with the number of read
          563  +  ** and write operations completed.  */
          564  +  if( rc ){
          565  +    testThreadSetResult(pThreadSet, iThread, rc, 0);
          566  +    testThreadSetHalt(pThreadSet);
          567  +  }else{
          568  +    testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite);
          569  +  }
          570  +}
          571  +
          572  +static void do_test_mt1(
          573  +  const char *zSystem,            /* Database system name */
          574  +  const char *zPattern,           /* Run test cases that match this pattern */
          575  +  int *pRc                        /* IN/OUT: Error code */
          576  +){
          577  +  Mt1Test aTest[] = {
          578  +    /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */
          579  +    { {10, 1000},     4, 0, 0,   10000,   0 },
          580  +    { {10, 1000},     4, 4, 2,   100000,  0 },
          581  +    { {10, 100000},   4, 0, 0,   10000,   0 },
          582  +    { {10, 100000},   4, 4, 2,   100000,  0 },
          583  +  };
          584  +  int i;
          585  +
          586  +  for(i=0; *pRc==0 && i<ArraySize(aTest); i++){
          587  +    Mt1Test *p = &aTest[i];
          588  +    int bRun = testCaseBegin(pRc, zPattern, 
          589  +        "mt1.%s.db=%d,%d.ms=%d.rdwr=%d.fast=%d.slow=%d", 
          590  +        zSystem, p->param.nFanout, p->param.nKey, 
          591  +        p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader
          592  +    );
          593  +    if( bRun ){
          594  +      TestDb *pDb;
          595  +      ThreadSet *pSet;
          596  +      int iThread;
          597  +      int nThread;
          598  +
          599  +      p->zSystem = zSystem;
          600  +      pDb = testOpen(zSystem, 1, pRc);
          601  +
          602  +      nThread = p->nReadwrite + p->nFastReader + p->nSlowReader;
          603  +      pSet = testThreadInit(nThread);
          604  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          605  +        testThreadLaunch(pSet, iThread, mt1Main, (void *)p);
          606  +      }
          607  +
          608  +      testThreadWait(pSet, p->nMs);
          609  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          610  +        *pRc = testThreadGetResult(pSet, iThread, 0);
          611  +      }
          612  +      testCaseFinish(*pRc);
          613  +
          614  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          615  +        const char *zMsg = 0;
          616  +        *pRc = testThreadGetResult(pSet, iThread, &zMsg);
          617  +        printf("  Info: thread %d (%d): %s\n", iThread, *pRc, zMsg);
          618  +      }
          619  +
          620  +      testThreadShutdown(pSet);
          621  +      testClose(&pDb);
          622  +    }
          623  +  }
          624  +}
          625  +
          626  +void test_mt(
          627  +  const char *zSystem,            /* Database system name */
          628  +  const char *zPattern,           /* Run test cases that match this pattern */
          629  +  int *pRc                        /* IN/OUT: Error code */
          630  +){
          631  +  if( testThreadSupport()==0 ) return;
          632  +  do_test_mt1(zSystem, zPattern, pRc);
          633  +}

Added ext/lsm1/lsm-test/lsmtest6.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +typedef struct OomTest OomTest;
            5  +struct OomTest {
            6  +  lsm_env *pEnv;
            7  +  int iNext;                      /* Next value to pass to testMallocOom() */
            8  +  int nFail;                      /* Number of OOM events injected */
            9  +  int bEnable;
           10  +  int rc;                         /* Test case error code */
           11  +};
           12  +
           13  +static void testOomStart(OomTest *p){
           14  +  memset(p, 0, sizeof(OomTest));
           15  +  p->iNext = 1;
           16  +  p->bEnable = 1;
           17  +  p->nFail = 1;
           18  +  p->pEnv = tdb_lsm_env();
           19  +}
           20  +
           21  +static void xOomHook(OomTest *p){
           22  +  p->nFail++;
           23  +}
           24  +
           25  +static int testOomContinue(OomTest *p){
           26  +  if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){
           27  +    return 0;
           28  +  }
           29  +  p->nFail = 0;
           30  +  testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p);
           31  +  return 1;
           32  +}
           33  +
           34  +static void testOomEnable(OomTest *p, int bEnable){
           35  +  p->bEnable = bEnable;
           36  +  testMallocOomEnable(p->pEnv, bEnable);
           37  +}
           38  +
           39  +static void testOomNext(OomTest *p){
           40  +  p->iNext++;
           41  +}
           42  +
           43  +static int testOomHit(OomTest *p){
           44  +  return (p->nFail>0);
           45  +}
           46  +
           47  +static int testOomFinish(OomTest *p){
           48  +  return p->rc;
           49  +}
           50  +
           51  +static void testOomAssert(OomTest *p, int bVal){
           52  +  if( bVal==0 ){
           53  +    test_failed();
           54  +    p->rc = 1;
           55  +  }
           56  +}
           57  +
           58  +/*
           59  +** Test that the error code matches the state of the OomTest object passed
           60  +** as the first argument. Specifically, check that rc is LSM_NOMEM if an 
           61  +** OOM error has already been injected, or LSM_OK if not.
           62  +*/
           63  +static void testOomAssertRc(OomTest *p, int rc){
           64  +  testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM);
           65  +  testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 );
           66  +}
           67  +
           68  +static void testOomOpen(
           69  +  OomTest *pOom,
           70  +  const char *zName,
           71  +  lsm_db **ppDb,
           72  +  int *pRc
           73  +){
           74  +  if( *pRc==LSM_OK ){
           75  +    int rc;
           76  +    rc = lsm_new(tdb_lsm_env(), ppDb);
           77  +    if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName);
           78  +    testOomAssertRc(pOom, rc);
           79  +    *pRc = rc;
           80  +  }
           81  +}
           82  +
           83  +static void testOomFetch(
           84  +  OomTest *pOom,
           85  +  lsm_db *pDb,
           86  +  void *pKey, int nKey,
           87  +  void *pVal, int nVal,
           88  +  int *pRc
           89  +){
           90  +  testOomAssertRc(pOom, *pRc);
           91  +  if( *pRc==LSM_OK ){
           92  +    lsm_cursor *pCsr;
           93  +    int rc;
           94  +
           95  +    rc = lsm_csr_open(pDb, &pCsr);
           96  +    if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0);
           97  +    testOomAssertRc(pOom, rc);
           98  +
           99  +    if( rc==LSM_OK ){
          100  +      const void *p; int n;
          101  +      testOomAssert(pOom, lsm_csr_valid(pCsr));
          102  +
          103  +      rc = lsm_csr_key(pCsr, &p, &n);
          104  +      testOomAssertRc(pOom, rc);
          105  +      testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) );
          106  +    }
          107  +
          108  +    if( rc==LSM_OK ){
          109  +      const void *p; int n;
          110  +      testOomAssert(pOom, lsm_csr_valid(pCsr));
          111  +
          112  +      rc = lsm_csr_value(pCsr, &p, &n);
          113  +      testOomAssertRc(pOom, rc);
          114  +      testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) );
          115  +    }
          116  +
          117  +    lsm_csr_close(pCsr);
          118  +    *pRc = rc;
          119  +  }
          120  +}
          121  +
          122  +static void testOomWrite(
          123  +  OomTest *pOom,
          124  +  lsm_db *pDb,
          125  +  void *pKey, int nKey,
          126  +  void *pVal, int nVal,
          127  +  int *pRc
          128  +){
          129  +  testOomAssertRc(pOom, *pRc);
          130  +  if( *pRc==LSM_OK ){
          131  +    int rc;
          132  +
          133  +    rc = lsm_insert(pDb, pKey, nKey, pVal, nVal);
          134  +    testOomAssertRc(pOom, rc);
          135  +
          136  +    *pRc = rc;
          137  +  }
          138  +}
          139  +
          140  +
          141  +static void testOomFetchStr(
          142  +  OomTest *pOom,
          143  +  lsm_db *pDb,
          144  +  const char *zKey,
          145  +  const char *zVal,
          146  +  int *pRc
          147  +){
          148  +  int nKey = strlen(zKey);
          149  +  int nVal = strlen(zVal);
          150  +  testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
          151  +}
          152  +
          153  +static void testOomFetchData(
          154  +  OomTest *pOom,
          155  +  lsm_db *pDb,
          156  +  Datasource *pData,
          157  +  int iKey,
          158  +  int *pRc
          159  +){
          160  +  void *pKey; int nKey;
          161  +  void *pVal; int nVal;
          162  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          163  +  testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
          164  +}
          165  +
          166  +static void testOomWriteStr(
          167  +  OomTest *pOom,
          168  +  lsm_db *pDb,
          169  +  const char *zKey,
          170  +  const char *zVal,
          171  +  int *pRc
          172  +){
          173  +  int nKey = strlen(zKey);
          174  +  int nVal = strlen(zVal);
          175  +  testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
          176  +}
          177  +
          178  +static void testOomWriteData(
          179  +  OomTest *pOom,
          180  +  lsm_db *pDb,
          181  +  Datasource *pData,
          182  +  int iKey,
          183  +  int *pRc
          184  +){
          185  +  void *pKey; int nKey;
          186  +  void *pVal; int nVal;
          187  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          188  +  testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
          189  +}
          190  +
          191  +static void testOomScan(
          192  +  OomTest *pOom, 
          193  +  lsm_db *pDb, 
          194  +  int bReverse,
          195  +  const void *pKey, int nKey,
          196  +  int nScan,
          197  +  int *pRc
          198  +){
          199  +  if( *pRc==0 ){
          200  +    int rc;
          201  +    int iScan = 0;
          202  +    lsm_cursor *pCsr;
          203  +    int (*xAdvance)(lsm_cursor *);
          204  +    
          205  +
          206  +    rc = lsm_csr_open(pDb, &pCsr);
          207  +    testOomAssertRc(pOom, rc);
          208  +
          209  +    if( rc==LSM_OK ){
          210  +      if( bReverse ){
          211  +        rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE);
          212  +        xAdvance = lsm_csr_prev;
          213  +      }else{
          214  +        rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE);
          215  +        xAdvance = lsm_csr_next;
          216  +      }
          217  +    }
          218  +    testOomAssertRc(pOom, rc);
          219  +
          220  +    while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan<nScan ){
          221  +      const void *p; int n;
          222  +
          223  +      rc = lsm_csr_key(pCsr, &p, &n);
          224  +      testOomAssertRc(pOom, rc);
          225  +      if( rc==LSM_OK ){
          226  +        rc = lsm_csr_value(pCsr, &p, &n);
          227  +        testOomAssertRc(pOom, rc);
          228  +      }
          229  +      if( rc==LSM_OK ){
          230  +        rc = xAdvance(pCsr);
          231  +        testOomAssertRc(pOom, rc);
          232  +      }
          233  +      iScan++;
          234  +    }
          235  +
          236  +    lsm_csr_close(pCsr);
          237  +    *pRc = rc;
          238  +  }
          239  +}
          240  +
          241  +#define LSMTEST6_TESTDB "testdb.lsm" 
          242  +
          243  +#ifndef _WIN32
          244  +# include <unistd.h>
          245  +#endif
          246  +#include <sys/types.h>
          247  +#include <sys/stat.h>
          248  +#include <fcntl.h>
          249  +
          250  +void testDeleteLsmdb(const char *zFile){
          251  +  char *zLog = testMallocPrintf("%s-log", zFile);
          252  +  char *zShm = testMallocPrintf("%s-shm", zFile);
          253  +  unlink(zFile);
          254  +  unlink(zLog);
          255  +  unlink(zShm);
          256  +  testFree(zLog);
          257  +  testFree(zShm);
          258  +}
          259  +
          260  +static void copy_file(const char *zFrom, const char *zTo, int isDatabase){
          261  +
          262  +  if( access(zFrom, F_OK) ){
          263  +    unlink(zTo);
          264  +  }else{
          265  +    int fd1;
          266  +    int fd2;
          267  +    off_t sz;
          268  +    off_t i;
          269  +    struct stat buf;
          270  +    u8 *aBuf;
          271  +
          272  +    fd1 = open(zFrom, O_RDONLY | _O_BINARY, 0644);
          273  +    fd2 = open(zTo, O_RDWR | O_CREAT | _O_BINARY, 0644);
          274  +
          275  +    fstat(fd1, &buf);
          276  +    sz = buf.st_size;
          277  +    ftruncate(fd2, sz);
          278  +
          279  +    aBuf = testMalloc(4096);
          280  +    for(i=0; i<sz; i+=4096){
          281  +      int bLockPage = isDatabase && i == 0;
          282  +      int nByte = MIN((bLockPage ? 4066 : 4096), sz - i);
          283  +      memset(aBuf, 0, 4096);
          284  +      read(fd1, aBuf, nByte);
          285  +      write(fd2, aBuf, nByte);
          286  +      if( bLockPage ){
          287  +        lseek(fd1, 4096, SEEK_SET);
          288  +        lseek(fd2, 4096, SEEK_SET);
          289  +      }
          290  +    }
          291  +    testFree(aBuf);
          292  +
          293  +    close(fd1);
          294  +    close(fd2);
          295  +  }
          296  +}
          297  +
          298  +void testCopyLsmdb(const char *zFrom, const char *zTo){
          299  +  char *zLog1 = testMallocPrintf("%s-log", zFrom);
          300  +  char *zLog2 = testMallocPrintf("%s-log", zTo);
          301  +  char *zShm1 = testMallocPrintf("%s-shm", zFrom);
          302  +  char *zShm2 = testMallocPrintf("%s-shm", zTo);
          303  +
          304  +  unlink(zShm2);
          305  +  unlink(zLog2);
          306  +  unlink(zTo);
          307  +  copy_file(zFrom, zTo, 1);
          308  +  copy_file(zLog1, zLog2, 0);
          309  +  copy_file(zShm1, zShm2, 0);
          310  +
          311  +  testFree(zLog1); testFree(zLog2); testFree(zShm1); testFree(zShm2);
          312  +}
          313  +
          314  +/*
          315  +** File zFile is the path to a database. This function makes backups
          316  +** of the database file and its log as follows:
          317  +**
          318  +**     cp $(zFile)         $(zFile)-save
          319  +**     cp $(zFile)-$(zAux) $(zFile)-save-$(zAux)
          320  +**
          321  +** Function testRestoreDb() can be used to copy the files back in the
          322  +** other direction.
          323  +*/
          324  +void testSaveDb(const char *zFile, const char *zAux){
          325  +  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
          326  +  char *zFileSave = testMallocPrintf("%s-save", zFile);
          327  +  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
          328  +
          329  +  unlink(zFileSave);
          330  +  unlink(zLogSave);
          331  +  copy_file(zFile, zFileSave, 1);
          332  +  copy_file(zLog, zLogSave, 0);
          333  +
          334  +  testFree(zLog); testFree(zFileSave); testFree(zLogSave);
          335  +}
          336  +
          337  +/*
          338  +** File zFile is the path to a database. This function restores
          339  +** a backup of the database made by a previous call to testSaveDb().
          340  +** Specifically, it does the equivalent of:
          341  +**
          342  +**     cp $(zFile)-save         $(zFile)
          343  +**     cp $(zFile)-save-$(zAux) $(zFile)-$(zAux)
          344  +*/
          345  +void testRestoreDb(const char *zFile, const char *zAux){
          346  +  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
          347  +  char *zFileSave = testMallocPrintf("%s-save", zFile);
          348  +  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
          349  +
          350  +  copy_file(zFileSave, zFile, 1);
          351  +  copy_file(zLogSave, zLog, 0);
          352  +
          353  +  testFree(zLog); testFree(zFileSave); testFree(zLogSave);
          354  +}
          355  +
          356  +
          357  +static int lsmWriteStr(lsm_db *pDb, const char *zKey, const char *zVal){
          358  +  int nKey = strlen(zKey);
          359  +  int nVal = strlen(zVal);
          360  +  return lsm_insert(pDb, (void *)zKey, nKey, (void *)zVal, nVal);
          361  +}
          362  +
          363  +static void setup_delete_db(){
          364  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          365  +}
          366  +
          367  +/*
          368  +** Create a small database. With the following content:
          369  +**
          370  +**    "one"   -> "one"
          371  +**    "two"   -> "four"
          372  +**    "three" -> "nine"
          373  +**    "four"  -> "sixteen"
          374  +**    "five"  -> "twentyfive"
          375  +**    "six"   -> "thirtysix"
          376  +**    "seven" -> "fourtynine"
          377  +**    "eight" -> "sixtyfour"
          378  +*/
          379  +static void setup_populate_db(){
          380  +  const char *azStr[] = {
          381  +    "one",   "one",
          382  +    "two",   "four",
          383  +    "three", "nine",
          384  +    "four",  "sixteen",
          385  +    "five",  "twentyfive",
          386  +    "six",   "thirtysix",
          387  +    "seven", "fourtynine",
          388  +    "eight", "sixtyfour",
          389  +  };
          390  +  int rc;
          391  +  int ii;
          392  +  lsm_db *pDb;
          393  +
          394  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          395  +
          396  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          397  +  if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB);
          398  +
          399  +  for(ii=0; rc==LSM_OK && ii<ArraySize(azStr); ii+=2){
          400  +    rc = lsmWriteStr(pDb, azStr[ii], azStr[ii+1]);
          401  +  }
          402  +  lsm_close(pDb);
          403  +
          404  +  testSaveDb(LSMTEST6_TESTDB, "log");
          405  +  assert( rc==LSM_OK );
          406  +}
          407  +
          408  +static Datasource *getDatasource(void){
          409  +  const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 };
          410  +  return testDatasourceNew(&defn);
          411  +}
          412  +
          413  +/*
          414  +** Set up a database file with the following properties:
          415  +**
          416  +**   * Page size is 1024 bytes.
          417  +**   * Block size is 64 KB.
          418  +**   * Contains 5000 key-value pairs starting at 0 from the
          419  +**     datasource returned getDatasource().
          420  +*/
          421  +static void setup_populate_db2(){
          422  +  Datasource *pData;
          423  +  int ii;
          424  +  int rc;
          425  +  int nBlocksize = 64*1024;
          426  +  int nPagesize = 1024;
          427  +  int nWritebuffer = 4*1024;
          428  +  lsm_db *pDb;
          429  +
          430  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          431  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          432  +  if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB);
          433  +
          434  +  lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &nBlocksize); 
          435  +  lsm_config(pDb, LSM_CONFIG_PAGE_SIZE, &nPagesize); 
          436  +  lsm_config(pDb, LSM_CONFIG_AUTOFLUSH, &nWritebuffer); 
          437  +
          438  +  pData = getDatasource();
          439  +  for(ii=0; rc==LSM_OK && ii<5000; ii++){
          440  +    void *pKey; int nKey;
          441  +    void *pVal; int nVal;
          442  +    testDatasourceEntry(pData, ii, &pKey, &nKey, &pVal, &nVal);
          443  +    lsm_insert(pDb, pKey, nKey, pVal, nVal);
          444  +  }
          445  +  testDatasourceFree(pData);
          446  +  lsm_close(pDb);
          447  +
          448  +  testSaveDb(LSMTEST6_TESTDB, "log");
          449  +  assert( rc==LSM_OK );
          450  +}
          451  +
          452  +/*
          453  +** Test the results of OOM conditions in lsm_new().
          454  +*/
          455  +static void simple_oom_1(OomTest *pOom){
          456  +  int rc;
          457  +  lsm_db *pDb;
          458  +
          459  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          460  +  testOomAssertRc(pOom, rc);
          461  +
          462  +  lsm_close(pDb);
          463  +}
          464  +
          465  +/*
          466  +** Test the results of OOM conditions in lsm_open().
          467  +*/
          468  +static void simple_oom_2(OomTest *pOom){
          469  +  int rc;
          470  +  lsm_db *pDb;
          471  +
          472  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          473  +  if( rc==LSM_OK ){
          474  +    rc = lsm_open(pDb, "testdb.lsm");
          475  +  }
          476  +  testOomAssertRc(pOom, rc);
          477  +
          478  +  lsm_close(pDb);
          479  +}
          480  +
          481  +/*
          482  +** Test the results of OOM conditions in simple fetch operations.
          483  +*/
          484  +static void simple_oom_3(OomTest *pOom){
          485  +  int rc = LSM_OK;
          486  +  lsm_db *pDb;
          487  +
          488  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          489  +
          490  +  testOomFetchStr(pOom, pDb, "four",  "sixteen",    &rc);
          491  +  testOomFetchStr(pOom, pDb, "seven", "fourtynine", &rc);
          492  +  testOomFetchStr(pOom, pDb, "one",   "one",        &rc);
          493  +  testOomFetchStr(pOom, pDb, "eight", "sixtyfour",  &rc);
          494  +
          495  +  lsm_close(pDb);
          496  +}
          497  +
          498  +/*
          499  +** Test the results of OOM conditions in simple write operations.
          500  +*/
          501  +static void simple_oom_4(OomTest *pOom){
          502  +  int rc = LSM_OK;
          503  +  lsm_db *pDb;
          504  +
          505  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          506  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          507  +
          508  +  testOomWriteStr(pOom, pDb, "123", "onetwothree", &rc);
          509  +  testOomWriteStr(pOom, pDb, "456", "fourfivesix", &rc);
          510  +  testOomWriteStr(pOom, pDb, "789", "seveneightnine", &rc);
          511  +  testOomWriteStr(pOom, pDb, "123", "teneleventwelve", &rc);
          512  +  testOomWriteStr(pOom, pDb, "456", "fourteenfifteensixteen", &rc);
          513  +
          514  +  lsm_close(pDb);
          515  +}
          516  +
          517  +static void simple_oom_5(OomTest *pOom){
          518  +  Datasource *pData = getDatasource();
          519  +  int rc = LSM_OK;
          520  +  lsm_db *pDb;
          521  +
          522  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          523  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          524  +
          525  +  testOomFetchData(pOom, pDb, pData, 3333, &rc);
          526  +  testOomFetchData(pOom, pDb, pData, 0, &rc);
          527  +  testOomFetchData(pOom, pDb, pData, 4999, &rc);
          528  +
          529  +  lsm_close(pDb);
          530  +  testDatasourceFree(pData);
          531  +}
          532  +
          533  +static void simple_oom_6(OomTest *pOom){
          534  +  Datasource *pData = getDatasource();
          535  +  int rc = LSM_OK;
          536  +  lsm_db *pDb;
          537  +
          538  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          539  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          540  +
          541  +  testOomWriteData(pOom, pDb, pData, 5000, &rc);
          542  +  testOomWriteData(pOom, pDb, pData, 5001, &rc);
          543  +  testOomWriteData(pOom, pDb, pData, 5002, &rc);
          544  +  testOomFetchData(pOom, pDb, pData, 5001, &rc);
          545  +  testOomFetchData(pOom, pDb, pData, 1234, &rc);
          546  +
          547  +  lsm_close(pDb);
          548  +  testDatasourceFree(pData);
          549  +}
          550  +
          551  +static void simple_oom_7(OomTest *pOom){
          552  +  Datasource *pData = getDatasource();
          553  +  int rc = LSM_OK;
          554  +  lsm_db *pDb;
          555  +
          556  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          557  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          558  +  testOomScan(pOom, pDb, 0, "abc", 3, 20, &rc);
          559  +  lsm_close(pDb);
          560  +  testDatasourceFree(pData);
          561  +}
          562  +
          563  +static void simple_oom_8(OomTest *pOom){
          564  +  Datasource *pData = getDatasource();
          565  +  int rc = LSM_OK;
          566  +  lsm_db *pDb;
          567  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          568  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          569  +  testOomScan(pOom, pDb, 1, "xyz", 3, 20, &rc);
          570  +  lsm_close(pDb);
          571  +  testDatasourceFree(pData);
          572  +}
          573  +
          574  +/*
          575  +** This test case has two clients connected to a database. The first client
          576  +** hits an OOM while writing to the database. Check that the second 
          577  +** connection is still able to query the db following the OOM.
          578  +*/
          579  +static void simple_oom2_1(OomTest *pOom){
          580  +  const int nRecord = 100;        /* Number of records initially in db */
          581  +  const int nIns = 10;            /* Number of records inserted with OOM */
          582  +
          583  +  Datasource *pData = getDatasource();
          584  +  int rc = LSM_OK;
          585  +  lsm_db *pDb1;
          586  +  lsm_db *pDb2;
          587  +  int i;
          588  +
          589  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          590  +
          591  +  /* Open the two connections. Initialize the in-memory tree so that it
          592  +  ** contains 100 records. Do all this with OOM injection disabled. */
          593  +  testOomEnable(pOom, 0);
          594  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb1, &rc);
          595  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb2, &rc);
          596  +  for(i=0; i<nRecord; i++){
          597  +    testOomWriteData(pOom, pDb1, pData, i, &rc);
          598  +  }
          599  +  testOomEnable(pOom, 1);
          600  +  assert( rc==0 );
          601  +
          602  +  /* Insert 10 more records using pDb1. Stop when an OOM is encountered. */
          603  +  for(i=nRecord; i<nRecord+nIns; i++){
          604  +    testOomWriteData(pOom, pDb1, pData, i, &rc);
          605  +    if( rc ) break;
          606  +  }
          607  +  testOomAssertRc(pOom, rc);
          608  +
          609  +  /* Switch off OOM injection. Write a few rows using pDb2. Then check
          610  +  ** that the database may be successfully queried.  */
          611  +  testOomEnable(pOom, 0);
          612  +  rc = 0;
          613  +  for(; i<nRecord+nIns && rc==0; i++){
          614  +    testOomWriteData(pOom, pDb2, pData, i, &rc);
          615  +  }
          616  +  for(i=0; i<nRecord+nIns; i++) testOomFetchData(pOom, pDb2, pData, i, &rc);
          617  +  testOomEnable(pOom, 1);
          618  +
          619  +  lsm_close(pDb1);
          620  +  lsm_close(pDb2);
          621  +  testDatasourceFree(pData);
          622  +}
          623  +
          624  +
          625  +static void do_test_oom1(const char *zPattern, int *pRc){
          626  +  struct SimpleOom {
          627  +    const char *zName;
          628  +    void (*xSetup)(void);
          629  +    void (*xFunc)(OomTest *);
          630  +  } aSimple[] = {
          631  +    { "oom1.lsm.1", setup_delete_db,    simple_oom_1 },
          632  +    { "oom1.lsm.2", setup_delete_db,    simple_oom_2 },
          633  +    { "oom1.lsm.3", setup_populate_db,  simple_oom_3 },
          634  +    { "oom1.lsm.4", setup_delete_db,    simple_oom_4 },
          635  +    { "oom1.lsm.5", setup_populate_db2, simple_oom_5 },
          636  +    { "oom1.lsm.6", setup_populate_db2, simple_oom_6 },
          637  +    { "oom1.lsm.7", setup_populate_db2, simple_oom_7 },
          638  +    { "oom1.lsm.8", setup_populate_db2, simple_oom_8 },
          639  +
          640  +    { "oom2.lsm.1", setup_delete_db,    simple_oom2_1 },
          641  +  };
          642  +  int i;
          643  +
          644  +  for(i=0; i<ArraySize(aSimple); i++){
          645  +    if( *pRc==0 && testCaseBegin(pRc, zPattern, "%s", aSimple[i].zName) ){
          646  +      OomTest t;
          647  +
          648  +      if( aSimple[i].xSetup ){
          649  +        aSimple[i].xSetup();
          650  +      }
          651  +
          652  +      for(testOomStart(&t); testOomContinue(&t); testOomNext(&t)){
          653  +        aSimple[i].xFunc(&t);
          654  +      }
          655  +
          656  +      printf("(%d injections).", t.iNext-2);
          657  +      testCaseFinish( (*pRc = testOomFinish(&t)) );
          658  +      testMallocOom(tdb_lsm_env(), 0, 0, 0, 0);
          659  +    }
          660  +  }
          661  +}
          662  +
          663  +void test_oom(
          664  +  const char *zPattern,           /* Run test cases that match this pattern */
          665  +  int *pRc                        /* IN/OUT: Error code */
          666  +){
          667  +  do_test_oom1(zPattern, pRc);
          668  +}

Added ext/lsm1/lsm-test/lsmtest7.c.

            1  +
            2  +
            3  +#include "lsmtest.h"
            4  +
            5  +
            6  +/*
            7  +** Test that the rules for when lsm_csr_next() and lsm_csr_prev() are
            8  +** enforced. Specifically:
            9  +**
           10  +**   * Both functions always return LSM_MISUSE if the cursor is at EOF
           11  +**     when they are called.
           12  +**
           13  +**   * lsm_csr_next() may only be used after lsm_csr_seek(LSM_SEEK_GE) or 
           14  +**     lsm_csr_first(). 
           15  +**
           16  +**   * lsm_csr_prev() may only be used after lsm_csr_seek(LSM_SEEK_LE) or 
           17  +**     lsm_csr_last().
           18  +*/
           19  +static void do_test_api1_lsm(lsm_db *pDb, int *pRc){
           20  +  int ret;
           21  +  lsm_cursor *pCsr;
           22  +  lsm_cursor *pCsr2;
           23  +  int nKey;
           24  +  const void *pKey;
           25  +
           26  +  ret = lsm_csr_open(pDb, &pCsr);
           27  +  testCompareInt(LSM_OK, ret, pRc);
           28  +
           29  +  ret = lsm_csr_next(pCsr);
           30  +  testCompareInt(LSM_MISUSE, ret, pRc);
           31  +  ret = lsm_csr_prev(pCsr);
           32  +  testCompareInt(LSM_MISUSE, ret, pRc);
           33  +
           34  +  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_GE);
           35  +  testCompareInt(LSM_OK, ret, pRc);
           36  +  ret = lsm_csr_next(pCsr);
           37  +  testCompareInt(LSM_OK, ret, pRc);
           38  +  ret = lsm_csr_prev(pCsr);
           39  +  testCompareInt(LSM_MISUSE, ret, pRc);
           40  +
           41  +  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LE);
           42  +  testCompareInt(LSM_OK, ret, pRc);
           43  +  ret = lsm_csr_next(pCsr);
           44  +  testCompareInt(LSM_MISUSE, ret, pRc);
           45  +  ret = lsm_csr_prev(pCsr);
           46  +  testCompareInt(LSM_OK, ret, pRc);
           47  +
           48  +  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LEFAST);
           49  +  testCompareInt(LSM_OK, ret, pRc);
           50  +  ret = lsm_csr_next(pCsr);
           51  +  testCompareInt(LSM_MISUSE, ret, pRc);
           52  +  ret = lsm_csr_prev(pCsr);
           53  +  testCompareInt(LSM_MISUSE, ret, pRc);
           54  +
           55  +  ret = lsm_csr_key(pCsr, &pKey, &nKey);
           56  +  testCompareInt(LSM_OK, ret, pRc);
           57  +
           58  +  ret = lsm_csr_open(pDb, &pCsr2);
           59  +  testCompareInt(LSM_OK, ret, pRc);
           60  +
           61  +  ret = lsm_csr_seek(pCsr2, pKey, nKey, LSM_SEEK_EQ);
           62  +  testCompareInt(LSM_OK, ret, pRc);
           63  +  testCompareInt(1, lsm_csr_valid(pCsr2), pRc);
           64  +  ret = lsm_csr_next(pCsr2);
           65  +  testCompareInt(LSM_MISUSE, ret, pRc);
           66  +  ret = lsm_csr_prev(pCsr2);
           67  +  testCompareInt(LSM_MISUSE, ret, pRc);
           68  +
           69  +  lsm_csr_close(pCsr2);
           70  +
           71  +  ret = lsm_csr_first(pCsr);
           72  +  testCompareInt(LSM_OK, ret, pRc);
           73  +  ret = lsm_csr_next(pCsr);
           74  +  testCompareInt(LSM_OK, ret, pRc);
           75  +  ret = lsm_csr_prev(pCsr);
           76  +  testCompareInt(LSM_MISUSE, ret, pRc);
           77  +
           78  +  ret = lsm_csr_last(pCsr);
           79  +  testCompareInt(LSM_OK, ret, pRc);
           80  +  ret = lsm_csr_prev(pCsr);
           81  +  testCompareInt(LSM_OK, ret, pRc);
           82  +  ret = lsm_csr_next(pCsr);
           83  +  testCompareInt(LSM_MISUSE, ret, pRc);
           84  +
           85  +  ret = lsm_csr_first(pCsr);
           86  +  while( lsm_csr_valid(pCsr) ){
           87  +    ret = lsm_csr_next(pCsr);
           88  +    testCompareInt(LSM_OK, ret, pRc);
           89  +  }
           90  +  ret = lsm_csr_next(pCsr);
           91  +  testCompareInt(LSM_OK, ret, pRc);
           92  +  ret = lsm_csr_prev(pCsr);
           93  +  testCompareInt(LSM_MISUSE, ret, pRc);
           94  +
           95  +  ret = lsm_csr_last(pCsr);
           96  +  while( lsm_csr_valid(pCsr) ){
           97  +    ret = lsm_csr_prev(pCsr);
           98  +    testCompareInt(LSM_OK, ret, pRc);
           99  +  }
          100  +  ret = lsm_csr_prev(pCsr);
          101  +  testCompareInt(LSM_OK, ret, pRc);
          102  +  ret = lsm_csr_next(pCsr);
          103  +  testCompareInt(LSM_MISUSE, ret, pRc);
          104  +
          105  +  lsm_csr_close(pCsr);
          106  +}
          107  +
          108  +static void do_test_api1(const char *zPattern, int *pRc){
          109  +  if( testCaseBegin(pRc, zPattern, "api1.lsm") ){
          110  +    const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 };
          111  +    Datasource *pData;
          112  +    TestDb *pDb;
          113  +    int rc = 0;
          114  +
          115  +    pDb = testOpen("lsm_lomem", 1, &rc);
          116  +    pData = testDatasourceNew(&defn);
          117  +    testWriteDatasourceRange(pDb, pData, 0, 1000, pRc);
          118  +
          119  +    do_test_api1_lsm(tdb_lsm(pDb), pRc);
          120  +
          121  +    testDatasourceFree(pData);
          122  +    testClose(&pDb);
          123  +
          124  +    testCaseFinish(*pRc);
          125  +  }
          126  +}
          127  +
          128  +static lsm_db *newLsmConnection(
          129  +  const char *zDb, 
          130  +  int nPgsz, 
          131  +  int nBlksz,
          132  +  int *pRc
          133  +){
          134  +  lsm_db *db = 0;
          135  +  if( *pRc==0 ){
          136  +    int n1 = nPgsz;
          137  +    int n2 = nBlksz;
          138  +    *pRc = lsm_new(tdb_lsm_env(), &db);
          139  +    if( *pRc==0 ){
          140  +      if( n1 ) lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1);
          141  +      if( n2 ) lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2);
          142  +      *pRc = lsm_open(db, "testdb.lsm");
          143  +    }
          144  +  }
          145  +  return db;
          146  +}
          147  +
          148  +static void testPagesize(lsm_db *db, int nPgsz, int nBlksz, int *pRc){
          149  +  if( *pRc==0 ){
          150  +    int n1 = 0;
          151  +    int n2 = 0;
          152  +
          153  +    lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1);
          154  +    lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2);
          155  +
          156  +    testCompareInt(n1, nPgsz, pRc);
          157  +    testCompareInt(n2, nBlksz, pRc);
          158  +  }
          159  +}
          160  +
          161  +/*
          162  +** Test case "api2" tests that the default page and block sizes of a 
          163  +** database may only be modified before lsm_open() is called. And that
          164  +** after lsm_open() is called lsm_config() may be used to read the 
          165  +** actual page and block size of the db.
          166  +*/
          167  +static void do_test_api2(const char *zPattern, int *pRc){
          168  +  if( *pRc==0 && testCaseBegin(pRc, zPattern, "api2.lsm") ){
          169  +    lsm_db *db1 = 0;
          170  +    lsm_db *db2 = 0;
          171  +
          172  +    testDeleteLsmdb("testdb.lsm");
          173  +    db1 = newLsmConnection("testdb.lsm", 0, 0, pRc);
          174  +    testPagesize(db1, 4096, 1024, pRc);
          175  +    db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
          176  +    testPagesize(db2, 4096, 1024, pRc);
          177  +    lsm_close(db1);
          178  +    lsm_close(db2);
          179  +
          180  +    testDeleteLsmdb("testdb.lsm");
          181  +    db1 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
          182  +    testPagesize(db1, 1024, 64*1024, pRc);
          183  +    db2 = newLsmConnection("testdb.lsm", 0, 0, pRc);
          184  +    testPagesize(db2, 1024, 64*1024, pRc);
          185  +    lsm_close(db1);
          186  +    lsm_close(db2);
          187  +
          188  +    testDeleteLsmdb("testdb.lsm");
          189  +    db1 = newLsmConnection("testdb.lsm", 8192, 2*1024, pRc);
          190  +    testPagesize(db1, 8192, 2*1024, pRc);
          191  +    db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
          192  +    testPagesize(db2, 8192, 2*1024, pRc);
          193  +    lsm_close(db1);
          194  +    lsm_close(db2);
          195  +
          196  +    testCaseFinish(*pRc);
          197  +  }
          198  +}
          199  +
          200  +void test_api(
          201  +  const char *zPattern,           /* Run test cases that match this pattern */
          202  +  int *pRc                        /* IN/OUT: Error code */
          203  +){
          204  +  do_test_api1(zPattern, pRc);
          205  +  do_test_api2(zPattern, pRc);
          206  +}

Added ext/lsm1/lsm-test/lsmtest8.c.

            1  +
            2  +/*
            3  +** This file contains test cases to verify that "live-recovery" following
            4  +** a mid-transaction failure of a writer process.
            5  +*/
            6  +
            7  +
            8  +/* 
            9  +** This test file includes lsmInt.h to get access to the definition of the
           10  +** ShmHeader structure. This is required to cause strategic damage to the
           11  +** shared memory header as part of recovery testing.
           12  +*/
           13  +#include "lsmInt.h"
           14  +
           15  +#include "lsmtest.h"
           16  +
           17  +typedef struct SetupStep SetupStep;
           18  +struct SetupStep {
           19  +  int bFlush;                     /* Flush to disk and checkpoint */
           20  +  int iInsStart;                  /* First key-value from ds to insert */
           21  +  int nIns;                       /* Number of rows to insert */
           22  +  int iDelStart;                  /* First key from ds to delete */
           23  +  int nDel;                       /* Number of rows to delete */
           24  +};
           25  +
           26  +static void doSetupStep(
           27  +  TestDb *pDb, 
           28  +  Datasource *pData, 
           29  +  const SetupStep *pStep, 
           30  +  int *pRc
           31  +){
           32  +  testWriteDatasourceRange(pDb, pData, pStep->iInsStart, pStep->nIns, pRc);
           33  +  testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc);
           34  +  if( *pRc==0 ){
           35  +    int nSave = -1;
           36  +    int nBuf = 64;
           37  +    lsm_db *db = tdb_lsm(pDb);
           38  +
           39  +    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave);
           40  +    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf);
           41  +    lsm_begin(db, 1);
           42  +    lsm_commit(db, 0);
           43  +    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave);
           44  +
           45  +    *pRc = lsm_work(db, 0, 0, 0);
           46  +    if( *pRc==0 ){
           47  +      *pRc = lsm_checkpoint(db, 0);
           48  +    }
           49  +  }
           50  +}
           51  +
           52  +static void doSetupStepArray(
           53  +  TestDb *pDb, 
           54  +  Datasource *pData, 
           55  +  const SetupStep *aStep, 
           56  +  int nStep
           57  +){
           58  +  int i;
           59  +  for(i=0; i<nStep; i++){
           60  +    int rc = 0;
           61  +    doSetupStep(pDb, pData, &aStep[i], &rc);
           62  +    assert( rc==0 );
           63  +  }
           64  +}
           65  +
           66  +static void setupDatabase1(TestDb *pDb, Datasource **ppData){
           67  +  const SetupStep aStep[] = {
           68  +    { 0,                                  1,     2000, 0, 0 },
           69  +    { 1,                                  0,     0, 0, 0 },
           70  +    { 0,                                  10001, 1000, 0, 0 },
           71  +  };
           72  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 100, 500};
           73  +  Datasource *pData;
           74  +
           75  +  pData = testDatasourceNew(&defn);
           76  +  doSetupStepArray(pDb, pData, aStep, ArraySize(aStep));
           77  +  if( ppData ){
           78  +    *ppData = pData;
           79  +  }else{
           80  +    testDatasourceFree(pData);
           81  +  }
           82  +}
           83  +
           84  +#include <stdio.h>
           85  +void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){
           86  +  if( *pRc==0 ){
           87  +    FILE *fd;
           88  +    fd = fopen(zFile, "r");
           89  +    if( fd==0 ){
           90  +      *pRc = 1;
           91  +    }else{
           92  +      if( 0!=fseek(fd, iOff, SEEK_SET) ){
           93  +        *pRc = 1;
           94  +      }else{
           95  +        if( nByte!=fread(pOut, 1, nByte, fd) ){
           96  +          *pRc = 1;
           97  +        }
           98  +      }
           99  +      fclose(fd);
          100  +    }
          101  +  }
          102  +}
          103  +
          104  +void testWriteFile(
          105  +  const char *zFile, 
          106  +  int iOff, 
          107  +  void *pOut, 
          108  +  int nByte, 
          109  +  int *pRc
          110  +){
          111  +  if( *pRc==0 ){
          112  +    FILE *fd;
          113  +    fd = fopen(zFile, "r+");
          114  +    if( fd==0 ){
          115  +      *pRc = 1;
          116  +    }else{
          117  +      if( 0!=fseek(fd, iOff, SEEK_SET) ){
          118  +        *pRc = 1;
          119  +      }else{
          120  +        if( nByte!=fwrite(pOut, 1, nByte, fd) ){
          121  +          *pRc = 1;
          122  +        }
          123  +      }
          124  +      fclose(fd);
          125  +    }
          126  +  }
          127  +}
          128  +
          129  +static ShmHeader *getShmHeader(const char *zDb){
          130  +  int rc = 0;
          131  +  char *zShm = testMallocPrintf("%s-shm", zDb);
          132  +  ShmHeader *pHdr;
          133  +
          134  +  pHdr = testMalloc(sizeof(ShmHeader));
          135  +  testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc);
          136  +  assert( rc==0 );
          137  +
          138  +  return pHdr;
          139  +}
          140  +
          141  +/*
          142  +** This function makes a copy of the three files associated with LSM 
          143  +** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db",
          144  +** "test.db-log" and "test.db-shm").
          145  +**
          146  +** It then opens a new database connection to the copy with the xLock() call
          147  +** instrumented so that it appears that some other process already connected
          148  +** to the db (holding a shared lock on DMS2). This prevents recovery from
          149  +** running. Then:
          150  +**
          151  +**    1) Check that the checksum of the database is zCksum. 
          152  +**    2) Write a few keys to the database. Then delete the same keys. 
          153  +**    3) Check that the checksum is zCksum.
          154  +**    4) Flush the db to disk and run a checkpoint. 
          155  +**    5) Check once more that the checksum is still zCksum.
          156  +*/
          157  +static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){
          158  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500};
          159  +  Datasource *pData;
          160  +  const char *zCopy = "testcopy.lsm";
          161  +  char zCksum2[TEST_CKSUM_BYTES];
          162  +  TestDb *pDb = 0;
          163  +  int rc;
          164  +
          165  +  pData = testDatasourceNew(&defn);
          166  +
          167  +  testCopyLsmdb(zDb, zCopy);
          168  +  rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb);
          169  +  if( rc==0 ){
          170  +    ShmHeader *pHdr;
          171  +    lsm_db *db;
          172  +    testCksumDatabase(pDb, zCksum2);
          173  +    testCompareStr(zCksum, zCksum2, &rc);
          174  +
          175  +    testWriteDatasourceRange(pDb, pData, 1, 10, &rc);
          176  +    testDeleteDatasourceRange(pDb, pData, 1, 10, &rc);
          177  +
          178  +    /* Test that the two tree-headers are now consistent. */
          179  +    pHdr = getShmHeader(zCopy);
          180  +    if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){
          181  +      rc = 1;
          182  +    }
          183  +    testFree(pHdr);
          184  +
          185  +    if( rc==0 ){
          186  +      int nBuf = 64;
          187  +      db = tdb_lsm(pDb);
          188  +      lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf);
          189  +      lsm_begin(db, 1);
          190  +      lsm_commit(db, 0);
          191  +      rc = lsm_work(db, 0, 0, 0);
          192  +    }
          193  +
          194  +    testCksumDatabase(pDb, zCksum2);
          195  +    testCompareStr(zCksum, zCksum2, &rc);
          196  +  }
          197  +
          198  +  testDatasourceFree(pData);
          199  +  testClose(&pDb);
          200  +  testDeleteLsmdb(zCopy);
          201  +  *pRc = rc;
          202  +}
          203  +
          204  +static void doWriterCrash1(int *pRc){
          205  +  const int nWrite = 2000;
          206  +  const int nStep = 10;
          207  +  const int iWriteStart = 20000;
          208  +  int rc = 0;
          209  +  TestDb *pDb = 0;
          210  +  Datasource *pData = 0;
          211  +
          212  +  rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb);
          213  +  if( rc==0 ){
          214  +    int iDot = 0;
          215  +    char zCksum[TEST_CKSUM_BYTES];
          216  +    int i;
          217  +    setupDatabase1(pDb, &pData);
          218  +    testCksumDatabase(pDb, zCksum);
          219  +    testBegin(pDb, 2, &rc);
          220  +    for(i=0; rc==0 && i<nWrite; i+=nStep){
          221  +      testCaseProgress(i, nWrite, testCaseNDot(), &iDot);
          222  +      testWriteDatasourceRange(pDb, pData, iWriteStart+i, nStep, &rc);
          223  +      doLiveRecovery("testdb.lsm", zCksum, &rc);
          224  +    }
          225  +  }
          226  +  testCommit(pDb, 0, &rc);
          227  +  testClose(&pDb);
          228  +  testDatasourceFree(pData);
          229  +  *pRc = rc;
          230  +}
          231  +
          232  +/*
          233  +** This test case verifies that inconsistent tree-headers in shared-memory
          234  +** are resolved correctly. 
          235  +*/
          236  +static void doWriterCrash2(int *pRc){
          237  +  int rc = 0;
          238  +  TestDb *pDb = 0;
          239  +  Datasource *pData = 0;
          240  +
          241  +  rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb);
          242  +  if( rc==0 ){
          243  +    ShmHeader *pHdr1;
          244  +    ShmHeader *pHdr2;
          245  +    char zCksum1[TEST_CKSUM_BYTES];
          246  +    char zCksum2[TEST_CKSUM_BYTES];
          247  +
          248  +    pHdr1 = testMalloc(sizeof(ShmHeader));
          249  +    pHdr2 = testMalloc(sizeof(ShmHeader));
          250  +    setupDatabase1(pDb, &pData);
          251  +
          252  +    /* Grab a copy of the shared-memory header. And the db checksum */
          253  +    testReadFile("testdb.lsm-shm", 0, (void *)pHdr1, sizeof(ShmHeader), &rc);
          254  +    testCksumDatabase(pDb, zCksum1);
          255  +
          256  +    /* Modify the database */
          257  +    testBegin(pDb, 2, &rc);
          258  +    testWriteDatasourceRange(pDb, pData, 30000, 200, &rc);
          259  +    testCommit(pDb, 0, &rc);
          260  +
          261  +    /* Grab a second copy of the shared-memory header. And the db checksum */
          262  +    testReadFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          263  +    testCksumDatabase(pDb, zCksum2);
          264  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          265  +
          266  +    /* If both tree-headers are valid, tree-header-1 is used. */
          267  +    memcpy(&pHdr2->hdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1));
          268  +    pHdr2->bWriter = 1;
          269  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          270  +    doLiveRecovery("testdb.lsm", zCksum1, &rc);
          271  +
          272  +    /* If both tree-headers are valid, tree-header-1 is used. */
          273  +    memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1));
          274  +    memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1));
          275  +    pHdr2->bWriter = 1;
          276  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          277  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          278  +
          279  +    /* If tree-header 1 is invalid, tree-header-2 is used */
          280  +    memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1));
          281  +    pHdr2->hdr1.aCksum[0] = 5;
          282  +    pHdr2->hdr1.aCksum[0] = 6;
          283  +    pHdr2->bWriter = 1;
          284  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          285  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          286  +
          287  +    /* If tree-header 2 is invalid, tree-header-1 is used */
          288  +    memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1));
          289  +    pHdr2->hdr2.aCksum[0] = 5;
          290  +    pHdr2->hdr2.aCksum[0] = 6;
          291  +    pHdr2->bWriter = 1;
          292  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          293  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          294  +
          295  +    testFree(pHdr1);
          296  +    testFree(pHdr2);
          297  +    testClose(&pDb);
          298  +  }
          299  +
          300  +  *pRc = rc;
          301  +}
          302  +
          303  +void do_writer_crash_test(const char *zPattern, int *pRc){
          304  +  struct Test {
          305  +    const char *zName;
          306  +    void (*xFunc)(int *);
          307  +  } aTest[] = {
          308  +    { "writercrash1.lsm", doWriterCrash1 },
          309  +    { "writercrash2.lsm", doWriterCrash2 },
          310  +  };
          311  +  int i;
          312  +  for(i=0; i<ArraySize(aTest); i++){
          313  +    struct Test *p = &aTest[i];
          314  +    if( testCaseBegin(pRc, zPattern, p->zName) ){
          315  +      p->xFunc(pRc);
          316  +      testCaseFinish(*pRc);
          317  +    }
          318  +  }
          319  +
          320  +}
          321  +
          322  +

Added ext/lsm1/lsm-test/lsmtest9.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE
            5  +#define DATA_RANDOM     TEST_DATASOURCE_RANDOM
            6  +
            7  +typedef struct Datatest4 Datatest4;
            8  +
            9  +/*
           10  +** Test overview:
           11  +**
           12  +**   1. Insert (Datatest4.nRec) records into a database.
           13  +**
           14  +**   2. Repeat (Datatest4.nRepeat) times:
           15  +**
           16  +**      2a. Delete 2/3 of the records in the database.
           17  +**
           18  +**      2b. Run lsm_work(nMerge=1).
           19  +**
           20  +**      2c. Insert as many records as were deleted in 2a.
           21  +**
           22  +**      2d. Check database content is as expected.
           23  +**
           24  +**      2e. If (Datatest4.bReopen) is true, close and reopen the database.
           25  +*/
           26  +struct Datatest4 {
           27  +  /* Datasource definition */
           28  +  DatasourceDefn defn;
           29  +
           30  +  int nRec;
           31  +  int nRepeat;
           32  +  int bReopen;
           33  +};
           34  +
           35  +static void doDataTest4(
           36  +  const char *zSystem,            /* Database system to test */
           37  +  Datatest4 *p,                   /* Structure containing test parameters */
           38  +  int *pRc                        /* OUT: Error code */
           39  +){
           40  +  lsm_db *db = 0;
           41  +  TestDb *pDb;
           42  +  TestDb *pControl;
           43  +  Datasource *pData;
           44  +  int i;
           45  +  int rc = 0;
           46  +  int iDot = 0;
           47  +  int bMultiThreaded = 0;         /* True for MT LSM database */
           48  +
           49  +  int nRecOn3 = (p->nRec / 3);
           50  +  int iData = 0;
           51  +
           52  +  /* Start the test case, open a database and allocate the datasource. */
           53  +  rc = testControlDb(&pControl);
           54  +  pDb = testOpen(zSystem, 1, &rc);
           55  +  pData = testDatasourceNew(&p->defn);
           56  +  if( rc==0 ){
           57  +    db = tdb_lsm(pDb);
           58  +    bMultiThreaded = tdb_lsm_multithread(pDb);
           59  +  }
           60  +
           61  +  testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc);
           62  +  testWriteDatasourceRange(pDb,      pData, iData, nRecOn3*3, &rc);
           63  +
           64  +  for(i=0; rc==0 && i<p->nRepeat; i++){
           65  +
           66  +    testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc);
           67  +    testDeleteDatasourceRange(pDb,      pData, iData, nRecOn3*2, &rc);
           68  +
           69  +    if( db ){
           70  +      int nDone;
           71  +#if 0
           72  +      fprintf(stderr, "lsm_work() start...\n"); fflush(stderr);
           73  +#endif
           74  +      do {
           75  +        nDone = 0;
           76  +        rc = lsm_work(db, 1, (1<<30), &nDone);
           77  +      }while( rc==0 && nDone>0 );
           78  +      if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK;
           79  +#if 0 
           80  +      fprintf(stderr, "lsm_work() done...\n"); fflush(stderr);
           81  +#endif
           82  +    }
           83  +
           84  +if( i+1<p->nRepeat ){
           85  +    iData += (nRecOn3*2);
           86  +    testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc);
           87  +    testWriteDatasourceRange(pDb,      pData, iData+nRecOn3, nRecOn3*2, &rc);
           88  +
           89  +    testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc);
           90  +
           91  +    /* If Datatest4.bReopen is true, close and reopen the database */
           92  +    if( p->bReopen ){
           93  +      testReopen(&pDb, &rc);
           94  +      if( rc==0 ) db = tdb_lsm(pDb);
           95  +    }
           96  +}
           97  +
           98  +    /* Update the progress dots... */
           99  +    testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot);
          100  +  }
          101  +
          102  +  testClose(&pDb);
          103  +  testClose(&pControl);
          104  +  testDatasourceFree(pData);
          105  +  testCaseFinish(rc);
          106  +  *pRc = rc;
          107  +}
          108  +
          109  +static char *getName4(const char *zSystem, Datatest4 *pTest){
          110  +  char *zRet;
          111  +  char *zData;
          112  +  zData = testDatasourceName(&pTest->defn);
          113  +  zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", 
          114  +      zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen
          115  +  );
          116  +  testFree(zData);
          117  +  return zRet;
          118  +}
          119  +
          120  +void test_data_4(
          121  +  const char *zSystem,            /* Database system name */
          122  +  const char *zPattern,           /* Run test cases that match this pattern */
          123  +  int *pRc                        /* IN/OUT: Error code */
          124  +){
          125  +  Datatest4 aTest[] = {
          126  +      /* defn,                                 nRec, nRepeat, bReopen */
          127  +    { {DATA_RANDOM,     20,25,     500,600}, 10000,      10,       0   },
          128  +    { {DATA_RANDOM,     20,25,     500,600}, 10000,      10,       1   },
          129  +  };
          130  +
          131  +  int i;
          132  +
          133  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          134  +    char *zName = getName4(zSystem, &aTest[i]);
          135  +    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          136  +      doDataTest4(zSystem, &aTest[i], pRc);
          137  +    }
          138  +    testFree(zName);
          139  +  }
          140  +}
          141  +
          142  +
          143  +

Added ext/lsm1/lsm-test/lsmtest_bt.c.

            1  +
            2  +#include "lsmtest.h"
            3  +#include "bt.h"
            4  +
            5  +int do_bt(int nArg, char **azArg){
            6  +  struct Option {
            7  +    const char *zName;
            8  +    int bPgno;
            9  +    int eOpt;
           10  +  } aOpt [] = { 
           11  +    { "dbhdr",          0, BT_INFO_HDRDUMP },
           12  +    { "filename",       0, BT_INFO_FILENAME },
           13  +    { "block_freelist", 0, BT_INFO_BLOCK_FREELIST },
           14  +    { "page_freelist",  0, BT_INFO_PAGE_FREELIST },
           15  +    { "filename",       0, BT_INFO_FILENAME },
           16  +    { "page",           1, BT_INFO_PAGEDUMP },
           17  +    { "page_ascii",     1, BT_INFO_PAGEDUMP_ASCII },
           18  +    { "leaks",          0, BT_INFO_PAGE_LEAKS },
           19  +    { 0, 0 } 
           20  +  };
           21  +  int iOpt;
           22  +  int rc;
           23  +  bt_info buf;
           24  +  char *zOpt;
           25  +  char *zFile;
           26  +
           27  +  bt_db *db = 0;
           28  +
           29  +  if( nArg<2 ){
           30  +    testPrintUsage("FILENAME OPTION ...");
           31  +    return -1;
           32  +  }
           33  +  zFile = azArg[0];
           34  +  zOpt = azArg[1];
           35  +
           36  +  rc = testArgSelect(aOpt, "option", zOpt, &iOpt);
           37  +  if( rc!=0 ) return rc;
           38  +  if( nArg!=2+aOpt[iOpt].bPgno ){
           39  +    testPrintFUsage("FILENAME %s %s", zOpt, aOpt[iOpt].bPgno ? "PGNO" : "");
           40  +    return -4;
           41  +  }
           42  +
           43  +  rc = sqlite4BtNew(sqlite4_env_default(), 0, &db);
           44  +  if( rc!=SQLITE4_OK ){
           45  +    testPrintError("sqlite4BtNew() failed: %d", rc);
           46  +    return -2;
           47  +  }
           48  +  rc = sqlite4BtOpen(db, zFile);
           49  +  if( rc!=SQLITE4_OK ){
           50  +    testPrintError("sqlite4BtOpen() failed: %d", rc);
           51  +    return -3;
           52  +  }
           53  +
           54  +  buf.eType = aOpt[iOpt].eOpt;
           55  +  buf.pgno = 0;
           56  +  sqlite4_buffer_init(&buf.output, 0);
           57  +
           58  +  if( aOpt[iOpt].bPgno ){
           59  +    buf.pgno = (u32)atoi(azArg[2]);
           60  +  }
           61  +
           62  +  rc = sqlite4BtControl(db, BT_CONTROL_INFO, &buf);
           63  +  if( rc!=SQLITE4_OK ){
           64  +    testPrintError("sqlite4BtControl() failed: %d\n", rc);
           65  +    return -4;
           66  +  }
           67  +
           68  +  printf("%s\n", (char*)buf.output.p);
           69  +  sqlite4_buffer_clear(&buf.output);
           70  +  return 0;
           71  +}
           72  +
           73  +
           74  +
           75  +

Added ext/lsm1/lsm-test/lsmtest_datasource.c.

            1  +
            2  +
            3  +#include "lsmtest.h"
            4  +
            5  +struct Datasource {
            6  +  int eType;
            7  +
            8  +  int nMinKey;
            9  +  int nMaxKey;
           10  +  int nMinVal;
           11  +  int nMaxVal;
           12  +
           13  +  char *aKey;
           14  +  char *aVal;
           15  +};
           16  +
           17  +void testDatasourceEntry(
           18  +  Datasource *p, 
           19  +  int iData, 
           20  +  void **ppKey, int *pnKey,
           21  +  void **ppVal, int *pnVal
           22  +){
           23  +  assert( (ppKey==0)==(pnKey==0) );
           24  +  assert( (ppVal==0)==(pnVal==0) );
           25  +
           26  +  if( ppKey ){
           27  +    int nKey = 0;
           28  +    switch( p->eType ){
           29  +      case TEST_DATASOURCE_RANDOM: {
           30  +        int nRange = (1 + p->nMaxKey - p->nMinKey);
           31  +        nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; 
           32  +        testPrngString((u32)iData, p->aKey, nKey);
           33  +        break;
           34  +      }
           35  +      case TEST_DATASOURCE_SEQUENCE:
           36  +        nKey = sprintf(p->aKey, "%012d", iData);
           37  +        break;
           38  +    }
           39  +    *ppKey = p->aKey;
           40  +    *pnKey = nKey;
           41  +  }
           42  +  if( ppVal ){
           43  +    u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal;
           44  +    testPrngString((u32)~iData, p->aVal, (int)nVal);
           45  +    *ppVal = p->aVal;
           46  +    *pnVal = (int)nVal;
           47  +  }
           48  +}
           49  +
           50  +void testDatasourceFree(Datasource *p){
           51  +  testFree(p);
           52  +}
           53  +
           54  +/*
           55  +** Return a pointer to a nul-terminated string that corresponds to the
           56  +** contents of the datasource-definition passed as the first argument.
           57  +** The caller should eventually free the returned pointer using testFree().
           58  +*/
           59  +char *testDatasourceName(const DatasourceDefn *p){
           60  +  char *zRet;
           61  +  zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)",
           62  +      (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"),
           63  +      p->nMinKey, p->nMaxKey,
           64  +      p->nMinVal, p->nMaxVal
           65  +  );
           66  +  return zRet;
           67  +}
           68  +
           69  +Datasource *testDatasourceNew(const DatasourceDefn *pDefn){
           70  +  Datasource *p;
           71  +  int nMinKey; 
           72  +  int nMaxKey;
           73  +  int nMinVal;
           74  +  int nMaxVal; 
           75  +
           76  +  if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){
           77  +    nMinKey = 128;
           78  +    nMaxKey = 128;
           79  +  }else{
           80  +    nMinKey = MAX(0, pDefn->nMinKey);
           81  +    nMaxKey = MAX(nMinKey, pDefn->nMaxKey);
           82  +  }
           83  +  nMinVal = MAX(0, pDefn->nMinVal);
           84  +  nMaxVal = MAX(nMinVal, pDefn->nMaxVal);
           85  +
           86  +  p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1);
           87  +  p->eType = pDefn->eType;
           88  +  p->nMinKey = nMinKey;
           89  +  p->nMinVal = nMinVal;
           90  +  p->nMaxKey = nMaxKey;
           91  +  p->nMaxVal = nMaxVal;
           92  +  
           93  +  p->aKey = (char *)&p[1];
           94  +  p->aVal = &p->aKey[nMaxKey];
           95  +  return p;
           96  +};

Added ext/lsm1/lsm-test/lsmtest_func.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +
            5  +int do_work(int nArg, char **azArg){
            6  +  struct Option {
            7  +    const char *zName;
            8  +  } aOpt [] = {
            9  +    { "-nmerge" },
           10  +    { "-nkb" },
           11  +    { 0 }
           12  +  };
           13  +
           14  +  lsm_db *pDb;
           15  +  int rc;
           16  +  int i;
           17  +  const char *zDb;
           18  +  int nMerge = 1;
           19  +  int nKB = (1<<30);
           20  +
           21  +  if( nArg==0 ) goto usage;
           22  +  zDb = azArg[nArg-1];
           23  +  for(i=0; i<(nArg-1); i++){
           24  +    int iSel;
           25  +    rc = testArgSelect(aOpt, "option", azArg[i], &iSel);
           26  +    if( rc ) return rc;
           27  +    switch( iSel ){
           28  +      case 0:
           29  +        i++;
           30  +        if( i==(nArg-1) ) goto usage;
           31  +        nMerge = atoi(azArg[i]);
           32  +        break;
           33  +      case 1:
           34  +        i++;
           35  +        if( i==(nArg-1) ) goto usage;
           36  +        nKB = atoi(azArg[i]);
           37  +        break;
           38  +    }
           39  +  }
           40  +
           41  +  rc = lsm_new(0, &pDb);
           42  +  if( rc!=LSM_OK ){
           43  +    testPrintError("lsm_open(): rc=%d\n", rc);
           44  +  }else{
           45  +    rc = lsm_open(pDb, zDb);
           46  +    if( rc!=LSM_OK ){
           47  +      testPrintError("lsm_open(): rc=%d\n", rc);
           48  +    }else{
           49  +      int n = -1;
           50  +      lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n);
           51  +      n = n*2;
           52  +      lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n);
           53  +
           54  +      rc = lsm_work(pDb, nMerge, nKB, 0);
           55  +      if( rc!=LSM_OK ){
           56  +        testPrintError("lsm_work(): rc=%d\n", rc);
           57  +      }
           58  +    }
           59  +  }
           60  +  if( rc==LSM_OK ){
           61  +    rc = lsm_checkpoint(pDb, 0);
           62  +  }
           63  +
           64  +  lsm_close(pDb);
           65  +  return rc;
           66  +
           67  + usage:
           68  +  testPrintUsage("?-optimize? ?-n N? DATABASE");
           69  +  return -1;
           70  +}
           71  +
           72  +
           73  +/*
           74  +**   lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO??
           75  +*/
           76  +int do_show(int nArg, char **azArg){
           77  +  lsm_db *pDb;
           78  +  int rc;
           79  +  const char *zDb;
           80  +
           81  +  int eOpt = LSM_INFO_DB_STRUCTURE;
           82  +  unsigned int iPg = 0;
           83  +  int bConfig = 0;
           84  +  const char *zConfig = "";
           85  +
           86  +  struct Option {
           87  +    const char *zName;
           88  +    int bConfig;
           89  +    int eOpt;
           90  +  } aOpt [] = { 
           91  +    { "array",       0, LSM_INFO_ARRAY_STRUCTURE },
           92  +    { "array-pages", 0, LSM_INFO_ARRAY_PAGES },
           93  +    { "blocksize",   1, LSM_CONFIG_BLOCK_SIZE },
           94  +    { "pagesize",    1, LSM_CONFIG_PAGE_SIZE },
           95  +    { "freelist",    0, LSM_INFO_FREELIST },
           96  +    { "page-ascii",  0, LSM_INFO_PAGE_ASCII_DUMP },
           97  +    { "page-hex",    0, LSM_INFO_PAGE_HEX_DUMP },
           98  +    { 0, 0 } 
           99  +  };
          100  +
          101  +  char *z = 0; 
          102  +  int iDb = 0;                    /* Index of DATABASE in azArg[] */
          103  +
          104  +  /* Check if there is a "-config" option: */
          105  +  if( nArg>2 && strlen(azArg[0])>1 
          106  +   && memcmp(azArg[0], "-config", strlen(azArg[0]))==0
          107  +  ){
          108  +    zConfig = azArg[1];
          109  +    iDb = 2;
          110  +  }
          111  +  if( nArg<(iDb+1) ) goto usage;
          112  +
          113  +  if( nArg>(iDb+1) ){
          114  +    rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt);
          115  +    if( rc!=0 ) return rc;
          116  +    bConfig = aOpt[eOpt].bConfig;
          117  +    eOpt = aOpt[eOpt].eOpt;
          118  +    if( (bConfig==0 && eOpt==LSM_INFO_FREELIST)
          119  +     || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE)
          120  +     || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE)
          121  +    ){
          122  +      if( nArg!=(iDb+2) ) goto usage;
          123  +    }else{
          124  +      if( nArg!=(iDb+3) ) goto usage;
          125  +      iPg = atoi(azArg[iDb+2]);
          126  +    }
          127  +  }
          128  +  zDb = azArg[iDb];
          129  +
          130  +  rc = lsm_new(0, &pDb);
          131  +  tdb_lsm_configure(pDb, zConfig);
          132  +  if( rc!=LSM_OK ){
          133  +    testPrintError("lsm_new(): rc=%d\n", rc);
          134  +  }else{
          135  +    rc = lsm_open(pDb, zDb);
          136  +    if( rc!=LSM_OK ){
          137  +      testPrintError("lsm_open(): rc=%d\n", rc);
          138  +    }
          139  +  }
          140  +
          141  +  if( rc==LSM_OK ){
          142  +    if( bConfig==0 ){
          143  +      switch( eOpt ){
          144  +        case LSM_INFO_DB_STRUCTURE:
          145  +        case LSM_INFO_FREELIST:
          146  +          rc = lsm_info(pDb, eOpt, &z);
          147  +          break;
          148  +        case LSM_INFO_ARRAY_STRUCTURE:
          149  +        case LSM_INFO_ARRAY_PAGES:
          150  +        case LSM_INFO_PAGE_ASCII_DUMP:
          151  +        case LSM_INFO_PAGE_HEX_DUMP:
          152  +          rc = lsm_info(pDb, eOpt, iPg, &z);
          153  +          break;
          154  +        default:
          155  +          assert( !"no chance" );
          156  +      }
          157  +
          158  +      if( rc==LSM_OK ){
          159  +        printf("%s\n", z ? z : "");
          160  +        fflush(stdout);
          161  +      }
          162  +      lsm_free(lsm_get_env(pDb), z);
          163  +    }else{
          164  +      int iRes = -1;
          165  +      lsm_config(pDb, eOpt, &iRes);
          166  +      printf("%d\n", iRes);
          167  +      fflush(stdout);
          168  +    }
          169  +  }
          170  +
          171  +  lsm_close(pDb);
          172  +  return rc;
          173  +
          174  + usage:
          175  +  testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?");
          176  +  return -1;
          177  +}

Added ext/lsm1/lsm-test/lsmtest_io.c.

            1  +
            2  +/*
            3  +** SUMMARY
            4  +**
            5  +**   This file implements the 'io' subcommand of the test program. It is used
            6  +**   for testing the performance of various combinations of write() and fsync()
            7  +**   system calls. All operations occur on a single file, which may or may not
            8  +**   exist when a test is started.
            9  +**
           10  +**   A test consists of a series of commands. Each command is either a write
           11  +**   or an fsync. A write is specified as "<amount>@<offset>", where <amount>
           12  +**   is the amount of data written, and <offset> is the offset of the file
           13  +**   to write to. An <amount> or an <offset> is specified as an integer number
           14  +**   of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of
           15  +**   KB, MB or GB, respectively. An fsync is simply "S". All commands are
           16  +**   case-insensitive.
           17  +**
           18  +**   Example test program:
           19  +**
           20  +**        2M@6M 1492K@4M S 4096@4K S
           21  +**
           22  +**   This program writes 2 MB of data starting at the offset 6MB offset of
           23  +**   the file, followed by 1492 KB of data written at the 4MB offset of the
           24  +**   file, followed by a call to fsync(), a write of 4KB of data at byte
           25  +**   offset 4096, and finally another call to fsync().
           26  +**
           27  +**   Commands may either be specified on the command line (one command per
           28  +**   command line argument) or read from stdin. Commands read from stdin
           29  +**   must be separated by white-space.
           30  +**
           31  +** COMMAND LINE INVOCATION
           32  +**
           33  +**   The sub-command implemented in this file must be invoked with at least
           34  +**   two arguments - the path to the file to write to and the page-size to
           35  +**   use for writing. If there are more than two arguments, then each
           36  +**   subsequent argument is assumed to be a test command. If there are exactly
           37  +**   two arguments, the test commands are read from stdin.
           38  +**
           39  +**   A write command does not result in a single call to system call write().
           40  +**   Instead, the specified region is written sequentially using one or
           41  +**   more calls to write(), each of which writes not more than one page of
           42  +**   data. For example, if the page-size is 4KB, the command "2M@6M" results
           43  +**   in 512 calls to write(), each of which writes 4KB of data.
           44  +**
           45  +** EXAMPLES
           46  +**
           47  +**   Two equivalent examples:
           48  +**
           49  +**     $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S
           50  +**     3544K written in 129 ms
           51  +**     $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 
           52  +**     3544K written in 127 ms
           53  +**
           54  +*/
           55  +
           56  +#include "lsmtest.h"
           57  +
           58  +#include <sys/types.h>
           59  +#include <sys/stat.h>
           60  +#include <fcntl.h>
           61  +#ifndef _WIN32
           62  +# include <unistd.h>
           63  +#endif
           64  +#include <ctype.h>
           65  +
           66  +typedef struct IoContext IoContext;
           67  +
           68  +struct IoContext {
           69  +  int fd;
           70  +  int nWrite;
           71  +};
           72  +
           73  +/*
           74  +** As isspace(3)
           75  +*/
           76  +static int safe_isspace(char c){
           77  +  if( c&0x80) return 0;
           78  +  return isspace(c);
           79  +}
           80  +
           81  +/*
           82  +** As isdigit(3)
           83  +*/
           84  +static int safe_isdigit(char c){
           85  +  if( c&0x80) return 0;
           86  +  return isdigit(c);
           87  +}
           88  +
           89  +static i64 getNextSize(char *zIn, char **pzOut, int *pRc){
           90  +  i64 iRet = 0;
           91  +  if( *pRc==0 ){
           92  +    char *z = zIn;
           93  +
           94  +    if( !safe_isdigit(*z) ){
           95  +      *pRc = 1;
           96  +      return 0;
           97  +    }
           98  +
           99  +    /* Process digits */
          100  +    while( safe_isdigit(*z) ){
          101  +      iRet = iRet*10 + (*z - '0');
          102  +      z++;
          103  +    }
          104  +
          105  +    /* Process suffix */
          106  +    switch( *z ){
          107  +      case 'k': case 'K':
          108  +        iRet = iRet * 1024;
          109  +        z++;
          110  +        break;
          111  +
          112  +      case 'm': case 'M':
          113  +        iRet = iRet * 1024 * 1024;
          114  +        z++;
          115  +        break;
          116  +
          117  +      case 'g': case 'G':
          118  +        iRet = iRet * 1024 * 1024 * 1024;
          119  +        z++;
          120  +        break;
          121  +    }
          122  +
          123  +    if( pzOut ) *pzOut = z;
          124  +  }
          125  +  return iRet;
          126  +}
          127  +
          128  +static int doOneCmd(
          129  +  IoContext *pCtx,
          130  +  u8 *aData,
          131  +  int pgsz,
          132  +  char *zCmd,
          133  +  char **pzOut
          134  +){
          135  +  char c;
          136  +  char *z = zCmd;
          137  +
          138  +  while( safe_isspace(*z) ) z++;
          139  +  c = *z;
          140  +
          141  +  if( c==0 ){
          142  +    if( pzOut ) *pzOut = z;
          143  +    return 0;
          144  +  }
          145  +
          146  +  if( c=='s' || c=='S' ){
          147  +    if( pzOut ) *pzOut = &z[1];
          148  +    return fdatasync(pCtx->fd);
          149  +  }
          150  +
          151  +  if( safe_isdigit(c) ){
          152  +    i64 iOff = 0;
          153  +    int nByte = 0;
          154  +    int rc = 0;
          155  +    int nPg;
          156  +    int iPg;
          157  +
          158  +    nByte = getNextSize(z, &z, &rc);
          159  +    if( rc || *z!='@' ) goto bad_command;
          160  +    z++;
          161  +    iOff = getNextSize(z, &z, &rc);
          162  +    if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command;
          163  +    if( pzOut ) *pzOut = z;
          164  +
          165  +    nPg = (nByte+pgsz-1) / pgsz;
          166  +    lseek(pCtx->fd, iOff, SEEK_SET);
          167  +    for(iPg=0; iPg<nPg; iPg++){
          168  +      write(pCtx->fd, aData, pgsz);
          169  +    }
          170  +    pCtx->nWrite += nByte/1024;
          171  +
          172  +    return 0;
          173  +  }
          174  +
          175  + bad_command:
          176  +  testPrintError("unrecognized command: %s", zCmd);
          177  +  return 1;
          178  +}
          179  +
          180  +static int readStdin(char **pzOut){
          181  +  int nAlloc = 128;
          182  +  char *zOut = 0;
          183  +  int nOut = 0;
          184  +
          185  +  while( !feof(stdin) ){
          186  +    int nRead;
          187  +
          188  +    nAlloc = nAlloc*2;
          189  +    zOut = realloc(zOut, nAlloc);
          190  +    nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin);
          191  +
          192  +    if( nRead==0 ) break;
          193  +    nOut += nRead;
          194  +    zOut[nOut] = '\0';
          195  +  }
          196  +
          197  +  *pzOut = zOut;
          198  +  return 0;
          199  +}
          200  +
          201  +int do_io(int nArg, char **azArg){
          202  +  IoContext ctx;
          203  +  int pgsz;
          204  +  char *zFile;
          205  +  char *zPgsz;
          206  +  int i;
          207  +  int rc = 0;
          208  +
          209  +  char *zStdin = 0;
          210  +  char *z;
          211  +
          212  +  u8 *aData;
          213  +
          214  +  memset(&ctx, 0, sizeof(IoContext));
          215  +  if( nArg<2 ){
          216  +    testPrintUsage("FILE PGSZ ?CMD-1 ...?");
          217  +    return -1;
          218  +  }
          219  +  zFile = azArg[0];
          220  +  zPgsz = azArg[1];
          221  +
          222  +  pgsz = getNextSize(zPgsz, 0, &rc);
          223  +  if( pgsz<=0 ){
          224  +    testPrintError("Ridiculous page size: %d", pgsz);
          225  +    return -1;
          226  +  }
          227  +  aData = malloc(pgsz);
          228  +  memset(aData, 0x77, pgsz);
          229  +
          230  +  ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644);
          231  +  if( ctx.fd<0 ){
          232  +    perror("open: ");
          233  +    return -1;
          234  +  }
          235  +
          236  +  if( nArg==2 ){
          237  +    readStdin(&zStdin);
          238  +    testTimeInit();
          239  +    z = zStdin;
          240  +    while( *z && rc==0 ){
          241  +      rc = doOneCmd(&ctx, aData, pgsz, z, &z);
          242  +    }
          243  +  }else{
          244  +    testTimeInit();
          245  +    for(i=2; i<nArg; i++){
          246  +      rc = doOneCmd(&ctx, aData, pgsz, azArg[i], 0);
          247  +    }
          248  +  }
          249  +
          250  +  printf("%dK written in %d ms\n", ctx.nWrite, testTimeGet());
          251  +
          252  +  free(zStdin);
          253  +  close(ctx.fd);
          254  +
          255  +  return 0;
          256  +}

Added ext/lsm1/lsm-test/lsmtest_main.c.

            1  +
            2  +#include "stdarg.h"
            3  +#include "lsmtest.h"
            4  +#include "stdio.h"
            5  +#include "assert.h"
            6  +#include "string.h"
            7  +#include "stdlib.h"
            8  +
            9  +#include <sqlite3.h>
           10  +
           11  +#ifndef _WIN32
           12  +# include <unistd.h>
           13  +#endif
           14  +#include <stdlib.h>
           15  +#include <sys/types.h>
           16  +#include <sys/stat.h>
           17  +#include <fcntl.h>
           18  +#include <errno.h>
           19  +
           20  +
           21  +void test_failed(){ 
           22  +  assert( 0 );
           23  +  return; 
           24  +}
           25  +
           26  +#define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__)
           27  +static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){
           28  +  if( rc ){
           29  +    *pRc = rc;
           30  +    fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
           31  +    test_failed();
           32  +  }
           33  +}
           34  +
           35  +static int lsm_memcmp(u8 *a, u8 *b, int c){
           36  +  int i;
           37  +  for(i=0; i<c; i++){
           38  +    if( a[i]!=b[i] ) return a[i] - b[i];
           39  +  }
           40  +  return 0;
           41  +}
           42  +
           43  +/*
           44  +** A test utility function.
           45  +*/
           46  +void testFetch(
           47  +  TestDb *pDb,                    /* Database handle */
           48  +  void *pKey, int nKey,           /* Key to query database for */
           49  +  void *pVal, int nVal,           /* Expected value */
           50  +  int *pRc                        /* IN/OUT: Error code */
           51  +){
           52  +  if( *pRc==0 ){
           53  +    void *pDbVal;
           54  +    int nDbVal;
           55  +    int rc;
           56  +
           57  +    static int nCall = 0; nCall++;
           58  +
           59  +    rc = tdb_fetch(pDb, pKey, nKey, &pDbVal, &nDbVal);
           60  +    testSetError(rc);
           61  +    if( rc==0 && (nVal!=nDbVal || (nVal>0 && lsm_memcmp(pVal, pDbVal, nVal))) ){
           62  +      testSetError(1);
           63  +    }
           64  +  }
           65  +}
           66  +
           67  +void testWrite(
           68  +  TestDb *pDb,                    /* Database handle */
           69  +  void *pKey, int nKey,           /* Key to query database for */
           70  +  void *pVal, int nVal,           /* Value to write */
           71  +  int *pRc                        /* IN/OUT: Error code */
           72  +){
           73  +  if( *pRc==0 ){
           74  +    int rc;
           75  +static int nCall = 0;
           76  +nCall++;
           77  +    rc = tdb_write(pDb, pKey, nKey, pVal, nVal);
           78  +    testSetError(rc);
           79  +  }
           80  +}
           81  +void testDelete(
           82  +  TestDb *pDb,                    /* Database handle */
           83  +  void *pKey, int nKey,           /* Key to query database for */
           84  +  int *pRc                        /* IN/OUT: Error code */
           85  +){
           86  +  if( *pRc==0 ){
           87  +    int rc;
           88  +    *pRc = rc = tdb_delete(pDb, pKey, nKey);
           89  +    testSetError(rc);
           90  +  }
           91  +}
           92  +void testDeleteRange(
           93  +  TestDb *pDb,                    /* Database handle */
           94  +  void *pKey1, int nKey1,
           95  +  void *pKey2, int nKey2,
           96  +  int *pRc                        /* IN/OUT: Error code */
           97  +){
           98  +  if( *pRc==0 ){
           99  +    int rc;
          100  +    *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2);
          101  +    testSetError(rc);
          102  +  }
          103  +}
          104  +
          105  +void testBegin(TestDb *pDb, int iTrans, int *pRc){
          106  +  if( *pRc==0 ){
          107  +    int rc;
          108  +    rc = tdb_begin(pDb, iTrans);
          109  +    testSetError(rc);
          110  +  }
          111  +}
          112  +void testCommit(TestDb *pDb, int iTrans, int *pRc){
          113  +  if( *pRc==0 ){
          114  +    int rc;
          115  +    rc = tdb_commit(pDb, iTrans);
          116  +    testSetError(rc);
          117  +  }
          118  +}
          119  +#if 0 /* unused */
          120  +static void testRollback(TestDb *pDb, int iTrans, int *pRc){
          121  +  if( *pRc==0 ){
          122  +    int rc;
          123  +    rc = tdb_rollback(pDb, iTrans);
          124  +    testSetError(rc);
          125  +  }
          126  +}
          127  +#endif
          128  +
          129  +void testWriteStr(
          130  +  TestDb *pDb,                    /* Database handle */
          131  +  const char *zKey,               /* Key to query database for */
          132  +  const char *zVal,               /* Value to write */
          133  +  int *pRc                        /* IN/OUT: Error code */
          134  +){
          135  +  int nVal = (zVal ? strlen(zVal) : 0);
          136  +  testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
          137  +}
          138  +
          139  +#if 0 /* unused */
          140  +static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){
          141  +  testDelete(pDb, (void *)zKey, strlen(zKey), pRc);
          142  +}
          143  +#endif
          144  +void testFetchStr(
          145  +  TestDb *pDb,                    /* Database handle */
          146  +  const char *zKey,               /* Key to query database for */
          147  +  const char *zVal,               /* Value to write */
          148  +  int *pRc                        /* IN/OUT: Error code */
          149  +){
          150  +  int nVal = (zVal ? strlen(zVal) : 0);
          151  +  testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
          152  +}
          153  +
          154  +void testFetchCompare(
          155  +  TestDb *pControl, 
          156  +  TestDb *pDb, 
          157  +  void *pKey, int nKey, 
          158  +  int *pRc
          159  +){
          160  +  int rc;
          161  +  void *pDbVal1;
          162  +  void *pDbVal2;
          163  +  int nDbVal1;
          164  +  int nDbVal2;
          165  +
          166  +  static int nCall = 0;
          167  +  nCall++;
          168  +
          169  +  rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1);
          170  +  testSetError(rc);
          171  +
          172  +  rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2);
          173  +  testSetError(rc);
          174  +
          175  +  if( *pRc==0 
          176  +   && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1)))
          177  +  ){
          178  +    testSetError(1);
          179  +  }
          180  +}
          181  +
          182  +typedef struct ScanResult ScanResult;
          183  +struct ScanResult {
          184  +  TestDb *pDb;
          185  +
          186  +  int nRow;
          187  +  u32 cksum1;
          188  +  u32 cksum2;
          189  +  void *pKey1; int nKey1;
          190  +  void *pKey2; int nKey2;
          191  +
          192  +  int bReverse;
          193  +  int nPrevKey;
          194  +  u8 aPrevKey[256];
          195  +};
          196  +
          197  +static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){
          198  +  int res;
          199  +  res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2));
          200  +  if( res==0 ){
          201  +    res = nKey1 - nKey2;
          202  +  }
          203  +  return res;
          204  +}
          205  +
          206  +int test_scan_debug = 0;
          207  +
          208  +static void scanCompareCb(
          209  +  void *pCtx, 
          210  +  void *pKey, int nKey,
          211  +  void *pVal, int nVal
          212  +){
          213  +  ScanResult *p = (ScanResult *)pCtx;
          214  +  u8 *aKey = (u8 *)pKey;
          215  +  u8 *aVal = (u8 *)pVal;
          216  +  int i;
          217  +
          218  +  if( test_scan_debug ){
          219  +    printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey);
          220  +    fflush(stdout);
          221  +  }
          222  +#if 0
          223  +  if( test_scan_debug ) printf("%.20s\n", (char *)pVal);
          224  +#endif
          225  +
          226  +#if 0
          227  +  /* Check tdb_fetch() matches */
          228  +  int rc = 0;
          229  +  testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc);
          230  +  assert( rc==0 );
          231  +#endif
          232  +
          233  +  /* Update the checksum data */
          234  +  p->nRow++;
          235  +  for(i=0; i<nKey; i++){
          236  +    p->cksum1 += ((int)aKey[i] << (i&0x0F));
          237  +    p->cksum2 += p->cksum1;
          238  +  }
          239  +  for(i=0; i<nVal; i++){
          240  +    p->cksum1 += ((int)aVal[i] << (i&0x0F));
          241  +    p->cksum2 += p->cksum1;
          242  +  }
          243  +
          244  +  /* Check that the delivered row is not out of order. */
          245  +  if( nKey<(int)sizeof(p->aPrevKey) ){
          246  +    if( p->nPrevKey ){
          247  +      int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey);
          248  +      if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){
          249  +        testPrintError("Returned key out of order at %s:%d\n", 
          250  +            __FILE__, __LINE__
          251  +        );
          252  +      }
          253  +    }
          254  +
          255  +    p->nPrevKey = nKey;
          256  +    memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey));
          257  +  }
          258  +
          259  +  /* Check that the delivered row is within range. */
          260  +  if( p->pKey1 && (
          261  +      (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0)
          262  +   || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey)
          263  +  )){
          264  +    testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__);
          265  +  }
          266  +  if( p->pKey2 && (
          267  +      (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0)
          268  +   || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2<nKey)
          269  +  )){
          270  +    testPrintError("Returned key too large at %s:%d\n", __FILE__, __LINE__);
          271  +  }
          272  +
          273  +}
          274  +
          275  +/*
          276  +** Scan the contents of the two databases. Check that they match.
          277  +*/
          278  +void testScanCompare(
          279  +  TestDb *pDb1,                   /* Control (trusted) database */
          280  +  TestDb *pDb2,                   /* Database being tested */
          281  +  int bReverse,
          282  +  void *pKey1, int nKey1, 
          283  +  void *pKey2, int nKey2, 
          284  +  int *pRc
          285  +){
          286  +  static int nCall = 0; nCall++;
          287  +  if( *pRc==0 ){
          288  +    ScanResult res1;
          289  +    ScanResult res2;
          290  +    void *pRes1 = (void *)&res1;
          291  +    void *pRes2 = (void *)&res2;
          292  +
          293  +    memset(&res1, 0, sizeof(ScanResult));
          294  +    memset(&res2, 0, sizeof(ScanResult));
          295  +
          296  +    res1.pDb = pDb1;
          297  +    res1.nKey1 = nKey1; res1.pKey1 = pKey1;
          298  +    res1.nKey2 = nKey2; res1.pKey2 = pKey2;
          299  +    res1.bReverse = bReverse;
          300  +    res2.pDb = pDb2;
          301  +    res2.nKey1 = nKey1; res2.pKey1 = pKey1;
          302  +    res2.nKey2 = nKey2; res2.pKey2 = pKey2;
          303  +    res2.bReverse = bReverse;
          304  +
          305  +    tdb_scan(pDb1, pRes1, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
          306  +if( test_scan_debug ) printf("\n\n\n");
          307  +    tdb_scan(pDb2, pRes2, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
          308  +if( test_scan_debug ) printf("\n\n\n");
          309  +
          310  +    if( res1.nRow!=res2.nRow 
          311  +     || res1.cksum1!=res2.cksum1 
          312  +     || res1.cksum2!=res2.cksum2
          313  +    ){
          314  +      printf("expected: %d %X %X\n", res1.nRow, res1.cksum1, res1.cksum2);
          315  +      printf("got:      %d %X %X\n", res2.nRow, res2.cksum1, res2.cksum2);
          316  +      testSetError(1);
          317  +      *pRc = 1;
          318  +    }
          319  +  }
          320  +}
          321  +
          322  +void testClose(TestDb **ppDb){
          323  +  tdb_close(*ppDb);
          324  +  *ppDb = 0;
          325  +}
          326  +
          327  +TestDb *testOpen(const char *zSystem, int bClear, int *pRc){
          328  +  TestDb *pDb = 0;
          329  +  if( *pRc==0 ){
          330  +    int rc;
          331  +    rc = tdb_open(zSystem, 0, bClear, &pDb);
          332  +    if( rc!=0 ){
          333  +      testSetError(rc);
          334  +      *pRc = rc;
          335  +    }
          336  +  }
          337  +  return pDb;
          338  +}
          339  +
          340  +void testReopen(TestDb **ppDb, int *pRc){
          341  +  if( *pRc==0 ){
          342  +    const char *zLib;
          343  +    zLib = tdb_library_name(*ppDb);
          344  +    testClose(ppDb);
          345  +    *pRc = tdb_open(zLib, 0, 0, ppDb);
          346  +  }
          347  +}
          348  +
          349  +
          350  +#if 0 /* unused */
          351  +static void testSystemSelect(const char *zSys, int *piSel, int *pRc){
          352  +  if( *pRc==0 ){
          353  +    struct SysName { const char *zName; } *aName;
          354  +    int nSys;
          355  +    int i;
          356  +
          357  +    for(nSys=0; tdb_system_name(nSys); nSys++);
          358  +    aName = malloc(sizeof(struct SysName) * (nSys+1));
          359  +    for(i=0; i<=nSys; i++){
          360  +      aName[i].zName = tdb_system_name(i);
          361  +    }
          362  +
          363  +    *pRc = testArgSelect(aName, "db", zSys, piSel);
          364  +    free(aName);
          365  +  }
          366  +}
          367  +#endif
          368  +
          369  +char *testMallocVPrintf(const char *zFormat, va_list ap){
          370  +  int nByte;
          371  +  va_list copy;
          372  +  char *zRet;
          373  +
          374  +  __va_copy(copy, ap);
          375  +  nByte = vsnprintf(0, 0, zFormat, copy);
          376  +  va_end(copy);
          377  +
          378  +  assert( nByte>=0 );
          379  +  zRet = (char *)testMalloc(nByte+1);
          380  +  vsnprintf(zRet, nByte+1, zFormat, ap);
          381  +  return zRet;
          382  +}
          383  +
          384  +char *testMallocPrintf(const char *zFormat, ...){
          385  +  va_list ap;
          386  +  char *zRet;
          387  +
          388  +  va_start(ap, zFormat);
          389  +  zRet = testMallocVPrintf(zFormat, ap);
          390  +  va_end(ap);
          391  +
          392  +  return zRet;
          393  +}
          394  +
          395  +
          396  +/*
          397  +** A wrapper around malloc(3).
          398  +**
          399  +** This function should be used for all allocations made by test procedures.
          400  +** It has the following properties:
          401  +**
          402  +**   * Test code may assume that allocations may not fail.
          403  +**   * Returned memory is always zeroed.
          404  +**
          405  +** Allocations made using testMalloc() should be freed using testFree().
          406  +*/
          407  +void *testMalloc(int n){
          408  +  u8 *p = (u8*)malloc(n + 8);
          409  +  memset(p, 0, n+8);
          410  +  *(int*)p = n;
          411  +  return (void*)&p[8];
          412  +}
          413  +
          414  +void *testMallocCopy(void *pCopy, int nByte){
          415  +  void *pRet = testMalloc(nByte);
          416  +  memcpy(pRet, pCopy, nByte);
          417  +  return pRet;
          418  +}
          419  +
          420  +void *testRealloc(void *ptr, int n){
          421  +  if( ptr ){
          422  +    u8 *p = (u8*)ptr - 8;
          423  +    int nOrig =  *(int*)p;
          424  +    p = (u8*)realloc(p, n+8);
          425  +    if( nOrig<n ){
          426  +      memset(&p[8+nOrig], 0, n-nOrig);
          427  +    }
          428  +    *(int*)p = n;
          429  +    return (void*)&p[8];
          430  +  }
          431  +  return testMalloc(n);
          432  +}
          433  +
          434  +/*
          435  +** Free an allocation made by an earlier call to testMalloc().
          436  +*/
          437  +void testFree(void *ptr){
          438  +  if( ptr ){
          439  +    u8 *p = (u8*)ptr - 8;
          440  +    memset(p, 0x55, *(int*)p + 8);
          441  +    free(p);
          442  +  }
          443  +}
          444  +
          445  +/*
          446  +** String zPattern contains a glob pattern. Return true if zStr matches 
          447  +** the pattern, or false if it does not.
          448  +*/
          449  +int testGlobMatch(const char *zPattern, const char *zStr){
          450  +  int i = 0;
          451  +  int j = 0;
          452  +
          453  +  while( zPattern[i] ){
          454  +    char p = zPattern[i];
          455  +
          456  +    if( p=='*' || p=='%' ){
          457  +      do {
          458  +        if( testGlobMatch(&zPattern[i+1], &zStr[j]) ) return 1;
          459  +      }while( zStr[j++] );
          460  +      return 0;
          461  +    }
          462  +
          463  +    if( zStr[j]==0 || (p!='?' && p!=zStr[j]) ){
          464  +      /* Match failed. */
          465  +      return 0;
          466  +    }
          467  +
          468  +    j++;
          469  +    i++;
          470  +  }
          471  +
          472  +  return (zPattern[i]==0 && zStr[j]==0);
          473  +}
          474  +
          475  +/* 
          476  +** End of test utilities 
          477  +**************************************************************************/
          478  +
          479  +int do_test(int nArg, char **azArg){
          480  +  int j;
          481  +  int rc;
          482  +  int nFail = 0;
          483  +  const char *zPattern = 0;
          484  +
          485  +  if( nArg>1 ){
          486  +    testPrintError("Usage: test ?PATTERN?\n");
          487  +    return 1;
          488  +  }
          489  +  if( nArg==1 ){
          490  +    zPattern = azArg[0];
          491  +  }
          492  +
          493  +  for(j=0; tdb_system_name(j); j++){
          494  +    rc = 0;
          495  +
          496  +    test_data_1(tdb_system_name(j), zPattern, &rc);
          497  +    test_data_2(tdb_system_name(j), zPattern, &rc);
          498  +    test_data_3(tdb_system_name(j), zPattern, &rc);
          499  +    test_data_4(tdb_system_name(j), zPattern, &rc);
          500  +    test_rollback(tdb_system_name(j), zPattern, &rc);
          501  +    test_mc(tdb_system_name(j), zPattern, &rc);
          502  +    test_mt(tdb_system_name(j), zPattern, &rc);
          503  +
          504  +    if( rc ) nFail++;
          505  +  }
          506  +
          507  +  rc = 0;
          508  +  test_oom(zPattern, &rc);
          509  +  if( rc ) nFail++;
          510  +
          511  +  rc = 0;
          512  +  test_api(zPattern, &rc);
          513  +  if( rc ) nFail++;
          514  +
          515  +  rc = 0;
          516  +  do_crash_test(zPattern, &rc);
          517  +  if( rc ) nFail++;
          518  +
          519  +  rc = 0;
          520  +  do_writer_crash_test(zPattern, &rc);
          521  +  if( rc ) nFail++;
          522  +
          523  +  return (nFail!=0);
          524  +}
          525  +
          526  +static lsm_db *configure_lsm_db(TestDb *pDb){
          527  +  lsm_db *pLsm;
          528  +  pLsm = tdb_lsm(pDb);
          529  +  if( pLsm ){
          530  +    tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4");
          531  +  }
          532  +  return pLsm;
          533  +}
          534  +
          535  +typedef struct WriteHookEvent WriteHookEvent;
          536  +struct WriteHookEvent {
          537  +  i64 iOff;
          538  +  int nData;
          539  +  int nUs;
          540  +};
          541  +WriteHookEvent prev = {0, 0, 0};
          542  +
          543  +static void flushPrev(FILE *pOut){
          544  +  if( prev.nData ){
          545  +    fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs);
          546  +    prev.nData = 0;
          547  +  }
          548  +}
          549  +
          550  +#if 0 /* unused */
          551  +static void do_speed_write_hook2(
          552  +  void *pCtx,
          553  +  int bLog,
          554  +  i64 iOff,
          555  +  int nData,
          556  +  int nUs
          557  +){
          558  +  FILE *pOut = (FILE *)pCtx;
          559  +  if( bLog ) return;
          560  +
          561  +  if( prev.nData && nData && iOff==prev.iOff+prev.nData ){
          562  +    prev.nData += nData;
          563  +    prev.nUs += nUs;
          564  +  }else{
          565  +    flushPrev(pOut);
          566  +    if( nData==0 ){
          567  +      fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs);
          568  +    }else{
          569  +      prev.iOff = iOff;
          570  +      prev.nData = nData;
          571  +      prev.nUs = nUs;
          572  +    }
          573  +  }
          574  +}
          575  +#endif
          576  +
          577  +#define ST_REPEAT  0
          578  +#define ST_WRITE   1
          579  +#define ST_PAUSE   2
          580  +#define ST_FETCH   3
          581  +#define ST_SCAN    4
          582  +#define ST_NSCAN   5
          583  +#define ST_KEYSIZE 6
          584  +#define ST_VALSIZE 7
          585  +#define ST_TRANS   8
          586  +
          587  +
          588  +static void print_speed_test_help(){
          589  +  printf(
          590  +"\n"
          591  +"Repeat the following $repeat times:\n"
          592  +"  1. Insert $write key-value pairs. One transaction for each write op.\n"
          593  +"  2. Pause for $pause ms.\n"
          594  +"  3. Perform $fetch queries on the database.\n"
          595  +"\n"
          596  +"  Keys are $keysize bytes in size. Values are $valsize bytes in size\n"
          597  +"  Both keys and values are pseudo-randomly generated\n"
          598  +"\n"
          599  +"Options are:\n"
          600  +"  -repeat  $repeat                 (default value 10)\n"
          601  +"  -write   $write                  (default value 10000)\n"
          602  +"  -pause   $pause                  (default value 0)\n"
          603  +"  -fetch   $fetch                  (default value 0)\n"
          604  +"  -keysize $keysize                (default value 12)\n"
          605  +"  -valsize $valsize                (default value 100)\n"
          606  +"  -system  $system                 (default value \"lsm\")\n"
          607  +"  -trans   $trans                  (default value 0)\n"
          608  +"\n"
          609  +);
          610  +}
          611  +
          612  +int do_speed_test2(int nArg, char **azArg){
          613  +  struct Option {
          614  +    const char *zOpt;
          615  +    int eVal;
          616  +    int iDefault;
          617  +  } aOpt[] = {
          618  +    { "-repeat",  ST_REPEAT,    10},
          619  +    { "-write",   ST_WRITE,  10000},
          620  +    { "-pause",   ST_PAUSE,      0},
          621  +    { "-fetch",   ST_FETCH,      0},
          622  +    { "-scan",    ST_SCAN,       0},
          623  +    { "-nscan",   ST_NSCAN,      0},
          624  +    { "-keysize", ST_KEYSIZE,   12},
          625  +    { "-valsize", ST_VALSIZE,  100},
          626  +    { "-trans",   ST_TRANS,      0},
          627  +    { "-system",  -1,            0},
          628  +    { "help",     -2,            0},
          629  +    {0, 0, 0}
          630  +  };
          631  +  int i;
          632  +  int aParam[9];
          633  +  int rc = 0;
          634  +  int bReadonly = 0;
          635  +  int nContent = 0;
          636  +
          637  +  TestDb *pDb;
          638  +  Datasource *pData;
          639  +  DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 };
          640  +  char *zSystem = "";
          641  +  int bLsm = 1;
          642  +  FILE *pLog = 0;
          643  +
          644  +#ifdef NDEBUG
          645  +  /* If NDEBUG is defined, disable the dynamic memory related checks in
          646  +  ** lsmtest_mem.c. They slow things down.  */
          647  +  testMallocUninstall(tdb_lsm_env());
          648  +#endif
          649  +
          650  +  /* Initialize aParam[] with default values. */
          651  +  for(i=0; i<ArraySize(aOpt); i++){
          652  +    if( aOpt[i].zOpt ) aParam[aOpt[i].eVal] = aOpt[i].iDefault;
          653  +  }
          654  +
          655  +  /* Process the command line switches. */
          656  +  for(i=0; i<nArg; i+=2){
          657  +    int iSel;
          658  +    rc = testArgSelect(aOpt, "switch", azArg[i], &iSel);
          659  +    if( rc ){
          660  +      return rc;
          661  +    }
          662  +    if( aOpt[iSel].eVal==-2 ){
          663  +      print_speed_test_help();
          664  +      return 0;
          665  +    }
          666  +    if( i+1==nArg ){
          667  +      testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
          668  +      return 1;
          669  +    }
          670  +    if( aOpt[iSel].eVal>=0 ){
          671  +      aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]);
          672  +    }else{
          673  +      zSystem = azArg[i+1];
          674  +      bLsm = 0;
          675  +#if 0
          676  +      for(j=0; zSystem[j]; j++){
          677  +        if( zSystem[j]=='=' ) bLsm = 1;
          678  +      }
          679  +#endif
          680  +    }
          681  +  }
          682  +  
          683  +  printf("#");
          684  +  for(i=0; i<ArraySize(aOpt); i++){
          685  +    if( aOpt[i].zOpt ){
          686  +      if( aOpt[i].eVal>=0 ){
          687  +        printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]);
          688  +      }else if( aOpt[i].eVal==-1 ){
          689  +        printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem);
          690  +      }
          691  +    }
          692  +  }
          693  +  printf("\n");
          694  +
          695  +  defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE];
          696  +  defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE];
          697  +  pData = testDatasourceNew(&defn);
          698  +
          699  +  if( aParam[ST_WRITE]==0 ){
          700  +    bReadonly = 1;
          701  +  }
          702  +
          703  +  if( bLsm ){
          704  +    rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb);
          705  +  }else{
          706  +    pDb = testOpen(zSystem, !bReadonly, &rc);
          707  +  }
          708  +  if( rc!=0 ) return rc;
          709  +  if( bReadonly ){
          710  +    nContent = testCountDatabase(pDb);
          711  +  }
          712  +
          713  +#if 0
          714  +  pLog = fopen("/tmp/speed.log", "w");
          715  +  tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
          716  +#endif
          717  +
          718  +  for(i=0; i<aParam[ST_REPEAT] && rc==0; i++){
          719  +    int msWrite, msFetch;
          720  +    int iFetch;
          721  +    int nWrite = aParam[ST_WRITE];
          722  +
          723  +    if( bReadonly ){
          724  +      msWrite = 0;
          725  +    }else{
          726  +      testTimeInit();
          727  +
          728  +      if( aParam[ST_TRANS] ) testBegin(pDb, 2, &rc);
          729  +      testWriteDatasourceRange(pDb, pData, i*nWrite, nWrite, &rc);
          730  +      if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
          731  +
          732  +      msWrite = testTimeGet();
          733  +      nContent += nWrite;
          734  +    }
          735  +
          736  +    if( aParam[ST_PAUSE] ){
          737  +      if( aParam[ST_PAUSE]/1000 ) sleep(aParam[ST_PAUSE]/1000);
          738  +      if( aParam[ST_PAUSE]%1000 ) usleep(1000 * (aParam[ST_PAUSE]%1000));
          739  +    }
          740  +
          741  +    if( aParam[ST_FETCH] ){
          742  +      testTimeInit();
          743  +      if( aParam[ST_TRANS] ) testBegin(pDb, 1, &rc);
          744  +      for(iFetch=0; iFetch<aParam[ST_FETCH]; iFetch++){
          745  +        int iKey = testPrngValue(i*nWrite+iFetch) % nContent;
          746  +#ifndef NDEBUG
          747  +        testDatasourceFetch(pDb, pData, iKey, &rc);
          748  +#else
          749  +        void *pKey; int nKey;           /* Database key to query for */
          750  +        void *pVal; int nVal;           /* Result of query */
          751  +
          752  +        testDatasourceEntry(pData, iKey, &pKey, &nKey, 0, 0);
          753  +        rc = tdb_fetch(pDb, pKey, nKey, &pVal, &nVal);
          754  +        if( rc==0 && nVal<0 ) rc = 1;
          755  +        if( rc ) break;
          756  +#endif
          757  +      }
          758  +      if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
          759  +      msFetch = testTimeGet();
          760  +    }else{
          761  +      msFetch = 0;
          762  +    }
          763  +
          764  +    if( i==(aParam[ST_REPEAT]-1) ){
          765  +      testTimeInit();
          766  +      testClose(&pDb);
          767  +      msWrite += testTimeGet();
          768  +    }
          769  +
          770  +    printf("%d %d %d\n", i, msWrite, msFetch);
          771  +    fflush(stdout);
          772  +  }
          773  +
          774  +  testClose(&pDb);
          775  +  testDatasourceFree(pData);
          776  +
          777  +  if( pLog ){
          778  +    flushPrev(pLog);
          779  +    fclose(pLog);
          780  +  }
          781  +  return rc;
          782  +}
          783  +
          784  +int do_speed_tests(int nArg, char **azArg){
          785  +
          786  +  struct DbSystem {
          787  +    const char *zLibrary;
          788  +    const char *zColor;
          789  +  } aSys[] = {
          790  +    { "sqlite3",      "black" },
          791  +    { "leveldb",      "blue" },
          792  +    { "lsm",          "red" },
          793  +    { "lsm_mt2",      "orange" },
          794  +    { "lsm_mt3",      "purple" },
          795  +    { "kyotocabinet", "green" },
          796  +    {0, 0}
          797  +  };
          798  +
          799  +  int i;
          800  +  int j;
          801  +  int rc;
          802  +  int nSleep = 0;                 /* ms of rest allowed between INSERT tests */
          803  +  int nRow = 0;                   /* Number of rows to insert into database */
          804  +  int nStep;                      /* Measure INSERT time after this many rows */
          805  +  int nSelStep;                   /* Measure SELECT time after this many rows */
          806  +  int nSelTest;                   /* Number of SELECTs to run for timing */
          807  +  int doReadTest = 1;
          808  +  int doWriteTest = 1;
          809  +
          810  +  int *aTime;                     /* INSERT timing data */
          811  +  int *aWrite;                    /* Writes per nStep inserts */
          812  +  int *aSelTime;                  /* SELECT timing data */
          813  +  int isFirst = 1;
          814  +  int bSleep = 0;
          815  +
          816  +  /* File to write gnuplot script to. */
          817  +  const char *zOut = "lsmtest_speed.gnuplot";
          818  +
          819  +  u32 sys_mask = 0;
          820  +
          821  +  testMallocUninstall(tdb_lsm_env());
          822  +
          823  +  for(i=0; i<nArg; i++){
          824  +    struct Opt { 
          825  +      const char *zOpt; 
          826  +      int isSwitch;
          827  +    } aOpt[] = {
          828  +      { "sqlite3" , 0},
          829  +      { "leveldb" , 0},
          830  +      { "lsm" , 0},
          831  +      { "lsm_mt2" , 0},
          832  +      { "lsm_mt3" , 0},
          833  +      { "kyotocabinet" , 0},
          834  +      { "-rows"     , 1},
          835  +      { "-sleep"    , 2},
          836  +      { "-testmode" , 3},
          837  +      { "-out"      , 4},
          838  +      { 0, 0}
          839  +    };
          840  +    int iSel;
          841  +
          842  +    rc = testArgSelect(aOpt, "argument", azArg[i], &iSel);
          843  +    if( rc ) return rc;
          844  +
          845  +    if( aOpt[iSel].isSwitch ){
          846  +      i++;
          847  +
          848  +      if( i>=nArg ){
          849  +        testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
          850  +        return 1;
          851  +      }
          852  +      if( aOpt[iSel].isSwitch==1 ){
          853  +        nRow = atoi(azArg[i]);
          854  +      }
          855  +      if( aOpt[iSel].isSwitch==2 ){
          856  +        nSleep = atoi(azArg[i]);
          857  +      }
          858  +      if( aOpt[iSel].isSwitch==3 ){
          859  +        struct Mode {
          860  +          const char *zMode;
          861  +          int doReadTest;
          862  +          int doWriteTest;
          863  +        } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}};
          864  +        int iMode;
          865  +        rc = testArgSelect(aMode, "option", azArg[i], &iMode);
          866  +        if( rc ) return rc;
          867  +        doReadTest = aMode[iMode].doReadTest;
          868  +        doWriteTest = aMode[iMode].doWriteTest;
          869  +      }
          870  +      if( aOpt[iSel].isSwitch==4 ){
          871  +        /* The "-out FILE" switch. This option is used to specify a file to
          872  +        ** write the gnuplot script to. */
          873  +        zOut = azArg[i];
          874  +      }
          875  +    }else{
          876  +      /* A db name */
          877  +      rc = testArgSelect(aOpt, "system", azArg[i], &iSel);
          878  +      if( rc ) return rc;
          879  +      sys_mask |= (1<<iSel);
          880  +    }
          881  +  }
          882  +
          883  +  if( sys_mask==0 ) sys_mask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
          884  +  nRow = MAX(nRow, 100000);
          885  +  nStep = nRow/100;
          886  +  nSelStep = nRow/10;
          887  +  nSelTest = (nSelStep > 100000) ? 100000 : nSelStep;
          888  +
          889  +  aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep);
          890  +  aWrite = malloc(sizeof(int) * nRow/nStep);
          891  +  aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep);
          892  +
          893  +  /* This loop collects the INSERT speed data. */
          894  +  if( doWriteTest ){
          895  +    printf("Writing output to file \"%s\".\n",  zOut);
          896  +
          897  +    for(j=0; aSys[j].zLibrary; j++){
          898  +      FILE *pLog = 0;
          899  +      TestDb *pDb;                  /* Database being tested */
          900  +      lsm_db *pLsm;
          901  +      int iDot = 0;
          902  +  
          903  +      if( ((1<<j)&sys_mask)==0 ) continue;
          904  +      if( bSleep && nSleep ) sqlite3_sleep(nSleep);
          905  +      bSleep = 1;
          906  +
          907  +      testCaseBegin(&rc, 0, "speed.insert.%s", aSys[j].zLibrary);
          908  +
          909  +      rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
          910  +      if( rc ) return rc;
          911  +
          912  +      pLsm = configure_lsm_db(pDb);
          913  +#if 0
          914  +      pLog = fopen("/tmp/speed.log", "w");
          915  +      tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
          916  +#endif
          917  +  
          918  +      testTimeInit();
          919  +      for(i=0; i<nRow; i+=nStep){
          920  +        int iStep;
          921  +        int nWrite1, nWrite2;
          922  +        testCaseProgress(i, nRow, testCaseNDot(), &iDot);
          923  +        if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite1);
          924  +        for(iStep=0; iStep<nStep; iStep++){
          925  +          u32 aKey[4];                  /* 16-byte key */
          926  +          u32 aVal[25];                 /* 100 byte value */
          927  +          testPrngArray(i+iStep, aKey, ArraySize(aKey));
          928  +          testPrngArray(i+iStep, aVal, ArraySize(aVal));
          929  +          rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
          930  +        }
          931  +        aTime[(j*nRow+i)/nStep] = testTimeGet();
          932  +        if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite2);
          933  +        aWrite[i/nStep] = nWrite2 - nWrite1;
          934  +      }
          935  +
          936  +      tdb_close(pDb);
          937  +      if( pLog ) fclose(pLog);
          938  +      testCaseFinish(rc);
          939  +    }
          940  +  }
          941  +
          942  +  /* This loop collects the SELECT speed data. */
          943  +  if( doReadTest ){
          944  +    for(j=0; aSys[j].zLibrary; j++){
          945  +      int iDot = 0;
          946  +      TestDb *pDb;                  /* Database being tested */
          947  +
          948  +      if( ((1<<j)&sys_mask)==0 ) continue;
          949  +      if( bSleep && nSleep ) sqlite3_sleep(nSleep);
          950  +      bSleep = 1;
          951  +
          952  +      testCaseBegin(&rc, 0, "speed.select.%s", aSys[j].zLibrary);
          953  +
          954  +      if( doWriteTest ){
          955  +        rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
          956  +        if( rc ) return rc;
          957  +        configure_lsm_db(pDb);
          958  +
          959  +        for(i=0; i<nRow; i+=nSelStep){
          960  +          int iStep;
          961  +          int iSel;
          962  +          testCaseProgress(i, nRow, testCaseNDot(), &iDot);
          963  +          for(iStep=0; iStep<nSelStep; iStep++){
          964  +            u32 aKey[4];                  /* 16-byte key */
          965  +            u32 aVal[25];                 /* 100 byte value */
          966  +            testPrngArray(i+iStep, aKey, ArraySize(aKey));
          967  +            testPrngArray(i+iStep, aVal, ArraySize(aVal));
          968  +            rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
          969  +          }
          970  +    
          971  +          testTimeInit();
          972  +          for(iSel=0; iSel<nSelTest; iSel++){
          973  +            void *pDummy;
          974  +            int nDummy;
          975  +            u32 iKey;
          976  +            u32 aKey[4];                  /* 16-byte key */
          977  +    
          978  +            iKey = testPrngValue(iSel) % (i+nSelStep);
          979  +            testPrngArray(iKey, aKey, ArraySize(aKey));
          980  +            rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
          981  +          }
          982  +          aSelTime[(j*nRow+i)/nSelStep] = testTimeGet();
          983  +          tdb_fetch(pDb, 0, 0, 0, 0);
          984  +        }
          985  +      }else{
          986  +        int t;
          987  +        int iSel;
          988  +
          989  +        rc = tdb_open(aSys[j].zLibrary, 0, 0, &pDb);
          990  +        configure_lsm_db(pDb);
          991  +
          992  +        testTimeInit();
          993  +        for(iSel=0; rc==LSM_OK && iSel<nSelTest; iSel++){
          994  +          void *pDummy;
          995  +          int nDummy;
          996  +          u32 iKey;
          997  +          u32 aKey[4];                  /* 16-byte key */
          998  +#ifndef NDEBUG
          999  +          u32 aVal[25];                 /* 100 byte value */
         1000  +#endif
         1001  +
         1002  +          testCaseProgress(iSel, nSelTest, testCaseNDot(), &iDot);
         1003  +    
         1004  +          iKey = testPrngValue(iSel) % nRow;
         1005  +          testPrngArray(iKey, aKey, ArraySize(aKey));
         1006  +          rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
         1007  +
         1008  +#ifndef NDEBUG
         1009  +          testPrngArray(iKey, aVal, ArraySize(aVal));
         1010  +          assert( nDummy==100 && memcmp(aVal, pDummy, 100)==0 );
         1011  +#endif
         1012  +        }
         1013  +        if( rc!=LSM_OK ) return rc;
         1014  +
         1015  +        t = testTimeGet();
         1016  +        tdb_fetch(pDb, 0, 0, 0, 0);
         1017  +
         1018  +        printf("%s: %d selects/second\n", 
         1019  +            aSys[j].zLibrary, (int)((double)nSelTest*1000.0/t)
         1020  +        );
         1021  +      }
         1022  +
         1023  +      tdb_close(pDb);
         1024  +      testCaseFinish(rc);
         1025  +    }
         1026  +  }
         1027  +
         1028  +
         1029  +  if( doWriteTest ){
         1030  +    FILE *pOut = fopen(zOut, "w");
         1031  +    if( !pOut ){
         1032  +      printf("fopen(\"%s\", \"w\"): %s\n", zOut, strerror(errno));
         1033  +      return 1;
         1034  +    }
         1035  +
         1036  +    fprintf(pOut, "set xlabel \"Rows Inserted\"\n");
         1037  +    fprintf(pOut, "set ylabel \"Inserts per second\"\n");
         1038  +    if( doReadTest ){
         1039  +      fprintf(pOut, "set y2label \"Selects per second\"\n");
         1040  +    }else if( sys_mask==(1<<2) ){
         1041  +      fprintf(pOut, "set y2label \"Page writes per insert\"\n");
         1042  +    }
         1043  +    fprintf(pOut, "set yrange [0:*]\n");
         1044  +    fprintf(pOut, "set y2range [0:*]\n");
         1045  +    fprintf(pOut, "set xrange [%d:*]\n", MAX(nStep, nRow/20) );
         1046  +    fprintf(pOut, "set ytics nomirror\n");
         1047  +    fprintf(pOut, "set y2tics nomirror\n");
         1048  +    fprintf(pOut, "set key box lw 0.01\n");
         1049  +    fprintf(pOut, "plot ");
         1050  +  
         1051  +    for(j=0; aSys[j].zLibrary; j++){
         1052  +      if( (1<<j)&sys_mask ){
         1053  +        const char *zLib = aSys[j].zLibrary;
         1054  +        fprintf(pOut, "%s\"-\" ti \"%s INSERT\" with lines lc rgb \"%s\" ", 
         1055  +            (isFirst?"":", "), zLib, aSys[j].zColor
         1056  +        );
         1057  +        if( doReadTest ){
         1058  +          fprintf(pOut, ", \"-\" ti \"%s SELECT\" "
         1059  +                 "axis x1y2 with points lw 3 lc rgb \"%s\""
         1060  +              , zLib, aSys[j].zColor
         1061  +          );
         1062  +        }
         1063  +        isFirst = 0;
         1064  +      }
         1065  +    }
         1066  +
         1067  +    assert( strcmp(aSys[2].zLibrary, "lsm")==0 );
         1068  +    if( sys_mask==(1<<2) && !doReadTest ){
         1069  +      fprintf(pOut, ", \"-\" ti \"lsm pages written\" "
         1070  +        "axis x1y2 with boxes lw 1 lc rgb \"grey\""
         1071  +      );
         1072  +    }
         1073  +  
         1074  +    fprintf(pOut, "\n");
         1075  +  
         1076  +    for(j=0; aSys[j].zLibrary; j++){
         1077  +      if( ((1<<j)&sys_mask)==0 ) continue;
         1078  +      fprintf(pOut, "# Rows    Inserts per second\n");
         1079  +      for(i=0; i<nRow; i+=nStep){
         1080  +        int iTime = aTime[(j*nRow+i)/nStep];
         1081  +        int ips = (int)((i+nStep)*1000.0 / (double)iTime);
         1082  +        fprintf(pOut, "%d %d\n", i+nStep, ips);
         1083  +      }
         1084  +      fprintf(pOut, "end\n");
         1085  +  
         1086  +      if( doReadTest ){
         1087  +        fprintf(pOut, "# Rows    Selects per second\n");
         1088  +        for(i=0; i<nRow; i+=nSelStep){
         1089  +          int sps = (int)(nSelTest*1000.0/(double)aSelTime[(j*nRow+i)/nSelStep]);
         1090  +          fprintf(pOut, "%d %d\n", i+nSelStep, sps);
         1091  +        }
         1092  +        fprintf(pOut, "end\n");
         1093  +      }else if( sys_mask==(1<<2) ){
         1094  +        for(i=0; i<(nRow/nStep); i++){
         1095  +          fprintf(pOut, "%d %f\n", i*nStep, (double)aWrite[i] / (double)nStep);
         1096  +        }
         1097  +        fprintf(pOut, "end\n");
         1098  +      }
         1099  +    }
         1100  +  
         1101  +    fprintf(pOut, "pause -1\n");
         1102  +    fclose(pOut);
         1103  +  }
         1104  +
         1105  +  free(aTime);
         1106  +  free(aSelTime);
         1107  +  free(aWrite);
         1108  +  testMallocInstall(tdb_lsm_env());
         1109  +  return 0;
         1110  +}
         1111  +
         1112  +/*
         1113  +** Usage: lsmtest random ?N?
         1114  +**
         1115  +** This command prints a sequence of zero or more numbers from the PRNG
         1116  +** system to stdout. If the "N" argument is missing, values the first 10
         1117  +** values (i=0, i=1, ... i=9) are printed. Otherwise, the first N.
         1118  +**
         1119  +** This was added to verify that the PRNG values do not change between
         1120  +** runs of the lsmtest program.
         1121  +*/
         1122  +int do_random_tests(int nArg, char **azArg){
         1123  +  int i;
         1124  +  int nRand;
         1125  +  if( nArg==0 ){
         1126  +    nRand = 10;
         1127  +  }else if( nArg==1 ){
         1128  +    nRand = atoi(azArg[0]);
         1129  +  }else{
         1130  +    testPrintError("Usage: random ?N?\n");
         1131  +    return -1;
         1132  +  }
         1133  +  for(i=0; i<nRand; i++){
         1134  +    printf("0x%x\n", testPrngValue(i));
         1135  +  }
         1136  +  return 0;
         1137  +}
         1138  +
         1139  +static int testFormatSize(char *aBuf, int nBuf, i64 nByte){
         1140  +  int res;
         1141  +  if( nByte<(1<<10) ){
         1142  +    res = snprintf(aBuf, nBuf, "%d byte", (int)nByte);
         1143  +  }else if( nByte<(1<<20) ){
         1144  +    res = snprintf(aBuf, nBuf, "%dK", (int)(nByte/(1<<10)));
         1145  +  }else{
         1146  +    res = snprintf(aBuf, nBuf, "%dM", (int)(nByte/(1<<20)));
         1147  +  }
         1148  +  return res;
         1149  +}
         1150  +
         1151  +static i64 testReadSize(char *z){
         1152  +  int n = strlen(z);
         1153  +  char c = z[n-1];
         1154  +  i64 nMul = 1;
         1155  +
         1156  +  switch( c ){
         1157  +    case 'g': case 'G':
         1158  +      nMul = (1<<30);
         1159  +      break;
         1160  +
         1161  +    case 'm': case 'M':
         1162  +      nMul = (1<<20);
         1163  +      break;
         1164  +
         1165  +    case 'k': case 'K':
         1166  +      nMul = (1<<10);
         1167  +      break;
         1168  +
         1169  +    default:
         1170  +      nMul = 1;
         1171  +  }
         1172  +
         1173  +  return nMul * (i64)atoi(z);
         1174  +} 
         1175  +
         1176  +/*
         1177  +** Usage: lsmtest writespeed FILESIZE BLOCKSIZE SYNCSIZE
         1178  +*/
         1179  +static int do_writer_test(int nArg, char **azArg){
         1180  +  int nBlock;
         1181  +  int nSize;
         1182  +  int i;
         1183  +  int fd;
         1184  +  int ms;
         1185  +  char aFilesize[32];
         1186  +  char aBlockSize[32];
         1187  +
         1188  +  char *aPage;
         1189  +  int *aOrder;
         1190  +  int nSync;
         1191  +
         1192  +  i64 filesize;
         1193  +  i64 blocksize;
         1194  +  i64 syncsize;
         1195  +  int nPage = 4096;
         1196  +
         1197  +  /* How long to sleep before running a trial (in ms). */
         1198  +#if 0
         1199  +  const int nSleep = 10000;
         1200  +#endif
         1201  +  const int nSleep = 0;
         1202  +
         1203  +  if( nArg!=3 ){
         1204  +    testPrintUsage("FILESIZE BLOCKSIZE SYNCSIZE");
         1205  +    return -1;
         1206  +  }
         1207  +
         1208  +  filesize = testReadSize(azArg[0]);
         1209  +  blocksize = testReadSize(azArg[1]);
         1210  +  syncsize = testReadSize(azArg[2]);
         1211  +
         1212  +  nBlock = (int)(filesize / blocksize);
         1213  +  nSize = (int)blocksize;
         1214  +  nSync = (int)(syncsize / blocksize);
         1215  +
         1216  +  aPage = (char *)malloc(4096);
         1217  +  aOrder = (int *)malloc(nBlock * sizeof(int));
         1218  +  for(i=0; i<nBlock; i++) aOrder[i] = i;
         1219  +  for(i=0; i<(nBlock*25); i++){
         1220  +    int tmp;
         1221  +    u32 a = testPrngValue(i);
         1222  +    u32 b = testPrngValue(a);
         1223  +    a = a % nBlock;
         1224  +    b = b % nBlock;
         1225  +    tmp = aOrder[a];
         1226  +    aOrder[a] = aOrder[b];
         1227  +    aOrder[b] = tmp;
         1228  +  }
         1229  +
         1230  +  testFormatSize(aFilesize, sizeof(aFilesize), (i64)nBlock * (i64)nSize);
         1231  +  testFormatSize(aBlockSize, sizeof(aFilesize), nSize);
         1232  +
         1233  +  printf("Testing writing a %s file using %s blocks. ", aFilesize, aBlockSize);
         1234  +  if( nSync==1 ){
         1235  +    printf("Sync after each block.\n");
         1236  +  }else{
         1237  +    printf("Sync after each %d blocks.\n", nSync);
         1238  +  }
         1239  +
         1240  +  printf("Preparing file... ");
         1241  +  fflush(stdout);
         1242  +  unlink("writer.out");
         1243  +  fd = open("writer.out", O_RDWR|O_CREAT|_O_BINARY, 0664);
         1244  +  if( fd<0 ){
         1245  +    testPrintError("open(): %d - %s\n", errno, strerror(errno));
         1246  +    return -1;
         1247  +  }
         1248  +  testTimeInit();
         1249  +  for(i=0; i<nBlock; i++){
         1250  +    int iPg;
         1251  +    memset(aPage, i&0xFF, nPage);
         1252  +    for(iPg=0; iPg<(nSize/nPage); iPg++){
         1253  +      write(fd, aPage, nPage);
         1254  +    }
         1255  +  }
         1256  +  fsync(fd);
         1257  +  printf("ok (%d ms)\n", testTimeGet());
         1258  +
         1259  +  for(i=0; i<5; i++){
         1260  +    int j;
         1261  +
         1262  +    sqlite3_sleep(nSleep);
         1263  +    printf("Now writing sequentially...  ");
         1264  +    fflush(stdout);
         1265  +
         1266  +    lseek(fd, 0, SEEK_SET);
         1267  +    testTimeInit();
         1268  +    for(j=0; j<nBlock; j++){
         1269  +      int iPg;
         1270  +      if( ((j+1)%nSync)==0 ) fdatasync(fd);
         1271  +      memset(aPage, j&0xFF, nPage);
         1272  +      for(iPg=0; iPg<(nSize/nPage); iPg++){
         1273  +        write(fd, aPage, nPage);
         1274  +      }
         1275  +    }
         1276  +    fdatasync(fd);
         1277  +    ms = testTimeGet();
         1278  +    printf("%d ms\n", ms);
         1279  +    sqlite3_sleep(nSleep);
         1280  +    printf("Now in an arbitrary order... ");
         1281  +
         1282  +    fflush(stdout);
         1283  +    testTimeInit();
         1284  +    for(j=0; j<nBlock; j++){
         1285  +      int iPg;
         1286  +      if( ((j+1)%nSync)==0 ) fdatasync(fd);
         1287  +      lseek(fd, aOrder[j]*nSize, SEEK_SET);
         1288  +      memset(aPage, j&0xFF, nPage);
         1289  +      for(iPg=0; iPg<(nSize/nPage); iPg++){
         1290  +        write(fd, aPage, nPage);
         1291  +      }
         1292  +    }
         1293  +    fdatasync(fd);
         1294  +    ms = testTimeGet();
         1295  +    printf("%d ms\n", ms);
         1296  +  }
         1297  +
         1298  +  close(fd);
         1299  +  free(aPage);
         1300  +  free(aOrder);
         1301  +
         1302  +  return 0;
         1303  +}
         1304  +
         1305  +static void do_insert_work_hook(lsm_db *db, void *p){
         1306  +  char *z = 0;
         1307  +  lsm_info(db, LSM_INFO_DB_STRUCTURE, &z);
         1308  +  if( z ){
         1309  +    printf("%s\n", z);
         1310  +    fflush(stdout);
         1311  +    lsm_free(lsm_get_env(db), z);
         1312  +  }
         1313  +
         1314  +  unused_parameter(p);
         1315  +}
         1316  +
         1317  +typedef struct InsertWriteHook InsertWriteHook;
         1318  +struct InsertWriteHook {
         1319  +  FILE *pOut;
         1320  +  int bLog;
         1321  +  i64 iOff;
         1322  +  int nData;
         1323  +};
         1324  +
         1325  +static void flushHook(InsertWriteHook *pHook){
         1326  +  if( pHook->nData ){
         1327  +    fprintf(pHook->pOut, "write %s %d %d\n", 
         1328  +        (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData
         1329  +    );
         1330  +    pHook->nData = 0;
         1331  +    fflush(pHook->pOut);
         1332  +  }
         1333  +}
         1334  +
         1335  +static void do_insert_write_hook(
         1336  +  void *pCtx,
         1337  +  int bLog,
         1338  +  i64 iOff,
         1339  +  int nData,
         1340  +  int nUs
         1341  +){
         1342  +  InsertWriteHook *pHook = (InsertWriteHook *)pCtx;
         1343  +  if( bLog ) return;
         1344  +
         1345  +  if( nData==0 ){
         1346  +    flushHook(pHook);
         1347  +    fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db"));
         1348  +  }else if( pHook->nData 
         1349  +         && bLog==pHook->bLog 
         1350  +         && iOff==(pHook->iOff+pHook->nData) 
         1351  +  ){
         1352  +    pHook->nData += nData;
         1353  +  }else{
         1354  +    flushHook(pHook);
         1355  +    pHook->bLog = bLog;
         1356  +    pHook->iOff = iOff;
         1357  +    pHook->nData = nData;
         1358  +  }
         1359  +}
         1360  +
         1361  +static int do_replay(int nArg, char **azArg){
         1362  +  char aBuf[4096];
         1363  +  FILE *pInput;
         1364  +  FILE *pClose = 0;
         1365  +  const char *zDb;
         1366  +
         1367  +  lsm_env *pEnv;
         1368  +  lsm_file *pOut;
         1369  +  int rc;
         1370  +
         1371  +  if( nArg!=2 ){
         1372  +    testPrintError("Usage: replay WRITELOG FILE\n");
         1373  +    return 1;
         1374  +  }
         1375  +
         1376  +  if( strcmp(azArg[0], "-")==0 ){
         1377  +    pInput = stdin;
         1378  +  }else{
         1379  +    pClose = pInput = fopen(azArg[0], "r");
         1380  +  }
         1381  +  zDb = azArg[1];
         1382  +  pEnv = tdb_lsm_env();
         1383  +  rc = pEnv->xOpen(pEnv, zDb, 0, &pOut);
         1384  +  if( rc!=LSM_OK ) return rc;
         1385  +
         1386  +  while( feof(pInput)==0 ){
         1387  +    char zLine[80];
         1388  +    fgets(zLine, sizeof(zLine)-1, pInput);
         1389  +    zLine[sizeof(zLine)-1] = '\0';
         1390  +
         1391  +    if( 0==memcmp("sync db", zLine, 7) ){
         1392  +      rc = pEnv->xSync(pOut);
         1393  +      if( rc!=0 ) break;
         1394  +    }else{
         1395  +      int iOff;
         1396  +      int nData;
         1397  +      int nMatch;
         1398  +      nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData);
         1399  +      if( nMatch==2 ){
         1400  +        int i;
         1401  +        for(i=0; i<nData; i+=sizeof(aBuf)){
         1402  +          memset(aBuf, i&0xFF, sizeof(aBuf));
         1403  +          rc = pEnv->xWrite(pOut, iOff+i, aBuf, sizeof(aBuf));
         1404  +          if( rc!=0 ) break;
         1405  +        }
         1406  +      }
         1407  +    }
         1408  +  }
         1409  +  if( pClose ) fclose(pClose);
         1410  +  pEnv->xClose(pOut);
         1411  +
         1412  +  return rc;
         1413  +}
         1414  +
         1415  +static int do_insert(int nArg, char **azArg){
         1416  +  const char *zDb = "lsm";
         1417  +  TestDb *pDb = 0;
         1418  +  int i;
         1419  +  int rc;
         1420  +  const int nRow = 1 * 1000 * 1000;
         1421  +
         1422  +  DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 };
         1423  +  Datasource *pData = 0;
         1424  +
         1425  +  if( nArg>1 ){
         1426  +    testPrintError("Usage: insert ?DATABASE?\n");
         1427  +    return 1;
         1428  +  }
         1429  +  if( nArg==1 ){ zDb = azArg[0]; }
         1430  +
         1431  +  testMallocUninstall(tdb_lsm_env());
         1432  +  for(i=0; zDb[i] && zDb[i]!='='; i++);
         1433  +  if( zDb[i] ){
         1434  +    rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb);
         1435  +  }else{
         1436  +    rc = tdb_open(zDb, 0, 1, &pDb);
         1437  +  }
         1438  +
         1439  +  if( rc!=0 ){
         1440  +    testPrintError("Error opening db \"%s\": %d\n", zDb, rc);
         1441  +  }else{
         1442  +    InsertWriteHook hook;
         1443  +    memset(&hook, 0, sizeof(hook));
         1444  +    hook.pOut = fopen("writelog.txt", "w");
         1445  +
         1446  +    pData = testDatasourceNew(&defn);
         1447  +    tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0);
         1448  +    tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook);
         1449  +
         1450  +    if( rc==0 ){
         1451  +      for(i=0; i<nRow; i++){
         1452  +        void *pKey; int nKey;     /* Database key to insert */
         1453  +        void *pVal; int nVal;     /* Database value to insert */
         1454  +        testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
         1455  +        tdb_write(pDb, pKey, nKey, pVal, nVal);
         1456  +      }
         1457  +    }
         1458  +
         1459  +    testDatasourceFree(pData);
         1460  +    tdb_close(pDb);
         1461  +    flushHook(&hook);
         1462  +    fclose(hook.pOut);
         1463  +  }
         1464  +  testMallocInstall(tdb_lsm_env());
         1465  +
         1466  +  return rc;
         1467  +}
         1468  +
         1469  +static int st_do_show(int a, char **b)      { return do_show(a, b); }
         1470  +static int st_do_work(int a, char **b)      { return do_work(a, b); }
         1471  +static int st_do_io(int a, char **b)        { return do_io(a, b); }
         1472  +
         1473  +#ifdef __linux__
         1474  +#include <sys/time.h>
         1475  +#include <sys/resource.h>
         1476  +
         1477  +static void lsmtest_rusage_report(void){
         1478  +  struct rusage r;
         1479  +  memset(&r, 0, sizeof(r));
         1480  +
         1481  +  getrusage(RUSAGE_SELF, &r);
         1482  +  printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", 
         1483  +      (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock
         1484  +  );
         1485  +}
         1486  +#else
         1487  +static void lsmtest_rusage_report(void){
         1488  +  /* no-op */
         1489  +}
         1490  +#endif
         1491  +
         1492  +int main(int argc, char **argv){
         1493  +  struct TestFunc {
         1494  +    const char *zName;
         1495  +    int bRusageReport;
         1496  +    int (*xFunc)(int, char **);
         1497  +  } aTest[] = {
         1498  +    {"random",      1, do_random_tests},
         1499  +    {"writespeed",  1, do_writer_test},
         1500  +    {"io",          1, st_do_io},
         1501  +
         1502  +    {"insert",      1, do_insert},
         1503  +    {"replay",      1, do_replay},
         1504  +
         1505  +    {"speed",       1, do_speed_tests},
         1506  +    {"speed2",      1, do_speed_test2},
         1507  +    {"show",        0, st_do_show},
         1508  +    {"work",        1, st_do_work},
         1509  +    {"test",        1, do_test},
         1510  +
         1511  +    {0, 0}
         1512  +  };
         1513  +  int rc;                         /* Return Code */
         1514  +  int iFunc;                      /* Index into aTest[] */
         1515  +
         1516  +  int nLeakAlloc = 0;             /* Allocations leaked by lsm */
         1517  +  int nLeakByte = 0;              /* Bytes leaked by lsm */
         1518  +
         1519  +#ifdef LSM_DEBUG_MEM
         1520  +  FILE *pReport = 0;              /* lsm malloc() report file */
         1521  +  const char *zReport = "malloc.txt generated";
         1522  +#else
         1523  +  const char *zReport = "malloc.txt NOT generated";
         1524  +#endif
         1525  +
         1526  +  testMallocInstall(tdb_lsm_env());
         1527  +
         1528  +  if( argc<2 ){
         1529  +    testPrintError("Usage: %s sub-command ?args...?\n", argv[0]);
         1530  +    return -1;
         1531  +  }
         1532  +
         1533  +  /* Initialize error reporting */
         1534  +  testErrorInit(argc, argv);
         1535  +
         1536  +  /* Initialize PRNG system */
         1537  +  testPrngInit();
         1538  +
         1539  +  rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc);
         1540  +  if( rc==0 ){
         1541  +    rc = aTest[iFunc].xFunc(argc-2, &argv[2]);
         1542  +  }
         1543  +
         1544  +#ifdef LSM_DEBUG_MEM
         1545  +  pReport = fopen("malloc.txt", "w");
         1546  +  testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport);
         1547  +  fclose(pReport);
         1548  +#else
         1549  +  testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0);
         1550  +#endif
         1551  +
         1552  +  if( nLeakAlloc ){
         1553  +    testPrintError("Leaked %d bytes in %d allocations (%s)\n", 
         1554  +        nLeakByte, nLeakAlloc, zReport
         1555  +    );
         1556  +    if( rc==0 ) rc = -1;
         1557  +  }
         1558  +  testMallocUninstall(tdb_lsm_env());
         1559  +
         1560  +  if( aTest[iFunc].bRusageReport ){
         1561  +    lsmtest_rusage_report();
         1562  +  }
         1563  +  return rc;
         1564  +}

Added ext/lsm1/lsm-test/lsmtest_mem.c.

            1  +
            2  +#include <stdio.h>
            3  +#include <assert.h>
            4  +#include <string.h>
            5  +
            6  +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0])))
            7  +
            8  +#define MIN(x,y) ((x)<(y) ? (x) : (y))
            9  +
           10  +typedef unsigned int  u32;
           11  +typedef unsigned char u8;
           12  +typedef long long int i64;
           13  +typedef unsigned long long int u64;
           14  +
           15  +#if defined(__GLIBC__) && defined(LSM_DEBUG_MEM)
           16  +  extern int backtrace(void**,int);
           17  +  extern void backtrace_symbols_fd(void*const*,int,int);
           18  +# define TM_BACKTRACE 12
           19  +#else
           20  +# define backtrace(A,B) 1
           21  +# define backtrace_symbols_fd(A,B,C)
           22  +#endif
           23  +
           24  +
           25  +typedef struct TmBlockHdr TmBlockHdr;
           26  +typedef struct TmAgg TmAgg;
           27  +typedef struct TmGlobal TmGlobal;
           28  +
           29  +struct TmGlobal {
           30  +  /* Linked list of all currently outstanding allocations. And a table of
           31  +  ** all allocations, past and present, indexed by backtrace() info.  */
           32  +  TmBlockHdr *pFirst;
           33  +#ifdef TM_BACKTRACE
           34  +  TmAgg *aHash[10000];
           35  +#endif
           36  +
           37  +  /* Underlying malloc/realloc/free functions */
           38  +  void *(*xMalloc)(int);          /* underlying malloc(3) function */
           39  +  void *(*xRealloc)(void *, int); /* underlying realloc(3) function */
           40  +  void (*xFree)(void *);          /* underlying free(3) function */
           41  +
           42  +  /* Mutex to protect pFirst and aHash */
           43  +  void (*xEnterMutex)(TmGlobal*); /* Call this to enter the mutex */
           44  +  void (*xLeaveMutex)(TmGlobal*); /* Call this to leave mutex */
           45  +  void (*xDelMutex)(TmGlobal*);   /* Call this to delete mutex */
           46  +  void *pMutex;                   /* Mutex handle */
           47  +
           48  +  void *xSaveMalloc;
           49  +  void *xSaveRealloc;
           50  +  void *xSaveFree;
           51  +
           52  +  /* OOM injection scheduling. If nCountdown is greater than zero when a 
           53  +  ** malloc attempt is made, it is decremented. If this means nCountdown 
           54  +  ** transitions from 1 to 0, then the allocation fails. If bPersist is true 
           55  +  ** when this happens, nCountdown is then incremented back to 1 (so that the 
           56  +  ** next attempt fails too).  
           57  +  */
           58  +  int nCountdown;
           59  +  int bPersist;
           60  +  int bEnable;
           61  +  void (*xHook)(void *);
           62  +  void *pHookCtx;
           63  +};
           64  +
           65  +struct TmBlockHdr {
           66  +  TmBlockHdr *pNext;
           67  +  TmBlockHdr *pPrev;
           68  +  int nByte;
           69  +#ifdef TM_BACKTRACE
           70  +  TmAgg *pAgg;
           71  +#endif
           72  +  u32 iForeGuard;
           73  +};
           74  +
           75  +#ifdef TM_BACKTRACE
           76  +struct TmAgg {
           77  +  int nAlloc;                     /* Number of allocations at this path */
           78  +  int nByte;                      /* Total number of bytes allocated */
           79  +  int nOutAlloc;                  /* Number of outstanding allocations */
           80  +  int nOutByte;                   /* Number of outstanding bytes */
           81  +  void *aFrame[TM_BACKTRACE];     /* backtrace() output */
           82  +  TmAgg *pNext;                   /* Next object in hash-table collision */
           83  +};
           84  +#endif
           85  +
           86  +#define FOREGUARD 0x80F5E153
           87  +#define REARGUARD 0xE4676B53
           88  +static const u32 rearguard = REARGUARD;
           89  +
           90  +#define ROUND8(x) (((x)+7)&~7)
           91  +
           92  +#define BLOCK_HDR_SIZE (ROUND8( sizeof(TmBlockHdr) ))
           93  +
           94  +static void lsmtest_oom_error(void){
           95  +  static int nErr = 0;
           96  +  nErr++;
           97  +}
           98  +
           99  +static void tmEnterMutex(TmGlobal *pTm){
          100  +  pTm->xEnterMutex(pTm);
          101  +}
          102  +static void tmLeaveMutex(TmGlobal *pTm){
          103  +  pTm->xLeaveMutex(pTm);
          104  +}
          105  +
          106  +static void *tmMalloc(TmGlobal *pTm, int nByte){
          107  +  TmBlockHdr *pNew;               /* New allocation header block */
          108  +  u8 *pUser;                      /* Return value */
          109  +  int nReq;                       /* Total number of bytes requested */
          110  +
          111  +  assert( sizeof(rearguard)==4 );
          112  +  nReq = BLOCK_HDR_SIZE + nByte + 4;
          113  +  pNew = (TmBlockHdr *)pTm->xMalloc(nReq);
          114  +  memset(pNew, 0, sizeof(TmBlockHdr));
          115  +
          116  +  tmEnterMutex(pTm);
          117  +  assert( pTm->nCountdown>=0 );
          118  +  assert( pTm->bPersist==0 || pTm->bPersist==1 );
          119  +
          120  +  if( pTm->bEnable && pTm->nCountdown==1 ){
          121  +    /* Simulate an OOM error. */
          122  +    lsmtest_oom_error();
          123  +    pTm->xFree(pNew);
          124  +    pTm->nCountdown = pTm->bPersist;
          125  +    if( pTm->xHook ) pTm->xHook(pTm->pHookCtx);
          126  +    pUser = 0;
          127  +  }else{
          128  +    if( pTm->bEnable && pTm->nCountdown ) pTm->nCountdown--;
          129  +
          130  +    pNew->iForeGuard = FOREGUARD;
          131  +    pNew->nByte = nByte;
          132  +    pNew->pNext = pTm->pFirst;
          133  +
          134  +    if( pTm->pFirst ){
          135  +      pTm->pFirst->pPrev = pNew;
          136  +    }
          137  +    pTm->pFirst = pNew;
          138  +
          139  +    pUser = &((u8 *)pNew)[BLOCK_HDR_SIZE];
          140  +    memset(pUser, 0x56, nByte);
          141  +    memcpy(&pUser[nByte], &rearguard, 4);
          142  +
          143  +#ifdef TM_BACKTRACE
          144  +    {
          145  +      TmAgg *pAgg;
          146  +      int i;
          147  +      u32 iHash = 0;
          148  +      void *aFrame[TM_BACKTRACE];
          149  +      memset(aFrame, 0, sizeof(aFrame));
          150  +      backtrace(aFrame, TM_BACKTRACE);
          151  +
          152  +      for(i=0; i<ArraySize(aFrame); i++){
          153  +        iHash += (u64)(aFrame[i]) + (iHash<<3);
          154  +      }
          155  +      iHash = iHash % ArraySize(pTm->aHash);
          156  +
          157  +      for(pAgg=pTm->aHash[iHash]; pAgg; pAgg=pAgg->pNext){
          158  +        if( memcmp(pAgg->aFrame, aFrame, sizeof(aFrame))==0 ) break;
          159  +      }
          160  +      if( !pAgg ){
          161  +        pAgg = (TmAgg *)pTm->xMalloc(sizeof(TmAgg));
          162  +        memset(pAgg, 0, sizeof(TmAgg));
          163  +        memcpy(pAgg->aFrame, aFrame, sizeof(aFrame));
          164  +        pAgg->pNext = pTm->aHash[iHash];
          165  +        pTm->aHash[iHash] = pAgg;
          166  +      }
          167  +      pAgg->nAlloc++;
          168  +      pAgg->nByte += nByte;
          169  +      pAgg->nOutAlloc++;
          170  +      pAgg->nOutByte += nByte;
          171  +      pNew->pAgg = pAgg;
          172  +    }
          173  +#endif
          174  +  }
          175  +
          176  +  tmLeaveMutex(pTm);
          177  +  return pUser;
          178  +}
          179  +
          180  +static void tmFree(TmGlobal *pTm, void *p){
          181  +  if( p ){
          182  +    TmBlockHdr *pHdr;
          183  +    u8 *pUser = (u8 *)p;
          184  +
          185  +    tmEnterMutex(pTm);
          186  +    pHdr = (TmBlockHdr *)&pUser[BLOCK_HDR_SIZE * -1];
          187  +    assert( pHdr->iForeGuard==FOREGUARD );
          188  +    assert( 0==memcmp(&pUser[pHdr->nByte], &rearguard, 4) );
          189  +
          190  +    if( pHdr->pPrev ){
          191  +      assert( pHdr->pPrev->pNext==pHdr );
          192  +      pHdr->pPrev->pNext = pHdr->pNext;
          193  +    }else{
          194  +      assert( pHdr==pTm->pFirst );
          195  +      pTm->pFirst = pHdr->pNext;
          196  +    }
          197  +    if( pHdr->pNext ){
          198  +      assert( pHdr->pNext->pPrev==pHdr );
          199  +      pHdr->pNext->pPrev = pHdr->pPrev;
          200  +    }
          201  +
          202  +#ifdef TM_BACKTRACE
          203  +    pHdr->pAgg->nOutAlloc--;
          204  +    pHdr->pAgg->nOutByte -= pHdr->nByte;
          205  +#endif
          206  +
          207  +    tmLeaveMutex(pTm);
          208  +    memset(pUser, 0x58, pHdr->nByte);
          209  +    memset(pHdr, 0x57, sizeof(TmBlockHdr));
          210  +    pTm->xFree(pHdr);
          211  +  }
          212  +}
          213  +
          214  +static void *tmRealloc(TmGlobal *pTm, void *p, int nByte){
          215  +  void *pNew;
          216  +
          217  +  pNew = tmMalloc(pTm, nByte);
          218  +  if( pNew && p ){
          219  +    TmBlockHdr *pHdr;
          220  +    u8 *pUser = (u8 *)p;
          221  +    pHdr = (TmBlockHdr *)&pUser[BLOCK_HDR_SIZE * -1];
          222  +    memcpy(pNew, p, MIN(nByte, pHdr->nByte));
          223  +    tmFree(pTm, p);
          224  +  }
          225  +  return pNew;
          226  +}
          227  +
          228  +static void tmMallocOom(
          229  +  TmGlobal *pTm, 
          230  +  int nCountdown, 
          231  +  int bPersist,
          232  +  void (*xHook)(void *),
          233  +  void *pHookCtx
          234  +){
          235  +  assert( nCountdown>=0 );
          236  +  assert( bPersist==0 || bPersist==1 );
          237  +  pTm->nCountdown = nCountdown;
          238  +  pTm->bPersist = bPersist;
          239  +  pTm->xHook = xHook;
          240  +  pTm->pHookCtx = pHookCtx;
          241  +  pTm->bEnable = 1;
          242  +}
          243  +
          244  +static void tmMallocOomEnable(
          245  +  TmGlobal *pTm, 
          246  +  int bEnable
          247  +){
          248  +  pTm->bEnable = bEnable;
          249  +}
          250  +
          251  +static void tmMallocCheck(
          252  +  TmGlobal *pTm,
          253  +  int *pnLeakAlloc,
          254  +  int *pnLeakByte,
          255  +  FILE *pFile
          256  +){
          257  +  TmBlockHdr *pHdr;
          258  +  int nLeak = 0;
          259  +  int nByte = 0;
          260  +
          261  +  if( pTm==0 ) return;
          262  +
          263  +  for(pHdr=pTm->pFirst; pHdr; pHdr=pHdr->pNext){
          264  +    nLeak++; 
          265  +    nByte += pHdr->nByte;
          266  +  }
          267  +  if( pnLeakAlloc ) *pnLeakAlloc = nLeak;
          268  +  if( pnLeakByte ) *pnLeakByte = nByte;
          269  +
          270  +#ifdef TM_BACKTRACE
          271  +  if( pFile ){
          272  +    int i;
          273  +    fprintf(pFile, "LEAKS\n");
          274  +    for(i=0; i<ArraySize(pTm->aHash); i++){
          275  +      TmAgg *pAgg;
          276  +      for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){
          277  +        if( pAgg->nOutAlloc ){
          278  +          int j;
          279  +          fprintf(pFile, "%d %d ", pAgg->nOutByte, pAgg->nOutAlloc);
          280  +          for(j=0; j<TM_BACKTRACE; j++){
          281  +            fprintf(pFile, "%p ", pAgg->aFrame[j]);
          282  +          }
          283  +          fprintf(pFile, "\n");
          284  +        }
          285  +      }
          286  +    }
          287  +    fprintf(pFile, "\nALLOCATIONS\n");
          288  +    for(i=0; i<ArraySize(pTm->aHash); i++){
          289  +      TmAgg *pAgg;
          290  +      for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){
          291  +        int j;
          292  +        fprintf(pFile, "%d %d ", pAgg->nByte, pAgg->nAlloc);
          293  +        for(j=0; j<TM_BACKTRACE; j++) fprintf(pFile, "%p ", pAgg->aFrame[j]);
          294  +        fprintf(pFile, "\n");
          295  +      }
          296  +    }
          297  +  }
          298  +#else
          299  +  (void)pFile;
          300  +#endif
          301  +}
          302  +
          303  +
          304  +#include "lsm.h"
          305  +#include "stdlib.h"
          306  +
          307  +typedef struct LsmMutex LsmMutex;
          308  +struct LsmMutex {
          309  +  lsm_env *pEnv;
          310  +  lsm_mutex *pMutex;
          311  +};
          312  +
          313  +static void tmLsmMutexEnter(TmGlobal *pTm){
          314  +  LsmMutex *p = (LsmMutex *)pTm->pMutex;
          315  +  p->pEnv->xMutexEnter(p->pMutex);
          316  +}
          317  +static void tmLsmMutexLeave(TmGlobal *pTm){
          318  +  LsmMutex *p = (LsmMutex *)(pTm->pMutex);
          319  +  p->pEnv->xMutexLeave(p->pMutex);
          320  +}
          321  +static void tmLsmMutexDel(TmGlobal *pTm){
          322  +  LsmMutex *p = (LsmMutex *)pTm->pMutex;
          323  +  pTm->xFree(p);
          324  +}
          325  +static void *tmLsmMalloc(int n){ return malloc(n); }
          326  +static void tmLsmFree(void *ptr){ free(ptr); }
          327  +static void *tmLsmRealloc(void *ptr, int n){ return realloc(ptr, n); }
          328  +
          329  +static void *tmLsmEnvMalloc(lsm_env *p, size_t n){ 
          330  +  return tmMalloc((TmGlobal *)(p->pMemCtx), n); 
          331  +}
          332  +static void tmLsmEnvFree(lsm_env *p, void *ptr){ 
          333  +  tmFree((TmGlobal *)(p->pMemCtx), ptr); 
          334  +}
          335  +static void *tmLsmEnvRealloc(lsm_env *p, void *ptr, size_t n){ 
          336  +  return tmRealloc((TmGlobal *)(p->pMemCtx), ptr, n);
          337  +}
          338  +
          339  +void testMallocInstall(lsm_env *pEnv){
          340  +  TmGlobal *pGlobal;
          341  +  LsmMutex *pMutex;
          342  +  assert( pEnv->pMemCtx==0 );
          343  +
          344  +  /* Allocate and populate a TmGlobal structure. */
          345  +  pGlobal = (TmGlobal *)tmLsmMalloc(sizeof(TmGlobal));
          346  +  memset(pGlobal, 0, sizeof(TmGlobal));
          347  +  pGlobal->xMalloc = tmLsmMalloc;
          348  +  pGlobal->xRealloc = tmLsmRealloc;
          349  +  pGlobal->xFree = tmLsmFree;
          350  +  pMutex = (LsmMutex *)pGlobal->xMalloc(sizeof(LsmMutex));
          351  +  pMutex->pEnv = pEnv;
          352  +  pEnv->xMutexStatic(pEnv, LSM_MUTEX_HEAP, &pMutex->pMutex);
          353  +  pGlobal->xEnterMutex = tmLsmMutexEnter;
          354  +  pGlobal->xLeaveMutex = tmLsmMutexLeave;
          355  +  pGlobal->xDelMutex = tmLsmMutexDel;
          356  +  pGlobal->pMutex = (void *)pMutex;
          357  +
          358  +  pGlobal->xSaveMalloc = (void *)pEnv->xMalloc;
          359  +  pGlobal->xSaveRealloc = (void *)pEnv->xRealloc;
          360  +  pGlobal->xSaveFree = (void *)pEnv->xFree;
          361  +
          362  +  /* Set up pEnv to the use the new TmGlobal */
          363  +  pEnv->pMemCtx = (void *)pGlobal;
          364  +  pEnv->xMalloc = tmLsmEnvMalloc;
          365  +  pEnv->xRealloc = tmLsmEnvRealloc;
          366  +  pEnv->xFree = tmLsmEnvFree;
          367  +}
          368  +
          369  +void testMallocUninstall(lsm_env *pEnv){
          370  +  TmGlobal *p = (TmGlobal *)pEnv->pMemCtx;
          371  +  pEnv->pMemCtx = 0;
          372  +  if( p ){
          373  +    pEnv->xMalloc = p->xSaveMalloc;
          374  +    pEnv->xRealloc = p->xSaveRealloc;
          375  +    pEnv->xFree = p->xSaveFree;
          376  +    p->xDelMutex(p);
          377  +    tmLsmFree(p);
          378  +  }
          379  +}
          380  +
          381  +void testMallocCheck(
          382  +  lsm_env *pEnv,
          383  +  int *pnLeakAlloc,
          384  +  int *pnLeakByte,
          385  +  FILE *pFile
          386  +){
          387  +  if( pEnv->pMemCtx==0 ){
          388  +    *pnLeakAlloc = 0;
          389  +    *pnLeakByte = 0;
          390  +  }else{
          391  +    tmMallocCheck((TmGlobal *)(pEnv->pMemCtx), pnLeakAlloc, pnLeakByte, pFile);
          392  +  }
          393  +}
          394  +
          395  +void testMallocOom(
          396  +  lsm_env *pEnv, 
          397  +  int nCountdown, 
          398  +  int bPersist,
          399  +  void (*xHook)(void *),
          400  +  void *pHookCtx
          401  +){
          402  +  TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx);
          403  +  tmMallocOom(pTm, nCountdown, bPersist, xHook, pHookCtx);
          404  +}
          405  +
          406  +void testMallocOomEnable(lsm_env *pEnv, int bEnable){
          407  +  TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx);
          408  +  tmMallocOomEnable(pTm, bEnable);
          409  +}

Added ext/lsm1/lsm-test/lsmtest_tdb.c.

            1  +
            2  +/*
            3  +** This program attempts to test the correctness of some facets of the 
            4  +** LSM database library. Specifically, that the contents of the database
            5  +** are maintained correctly during a series of inserts and deletes.
            6  +*/
            7  +
            8  +
            9  +#include "lsmtest_tdb.h"
           10  +#include "lsm.h"
           11  +
           12  +#include "lsmtest.h"
           13  +
           14  +#include <stdlib.h>
           15  +#include <string.h>
           16  +#include <assert.h>
           17  +#ifndef _WIN32
           18  +# include <unistd.h>
           19  +#endif
           20  +#include <stdio.h>
           21  +
           22  +
           23  +typedef struct SqlDb SqlDb;
           24  +
           25  +static int error_transaction_function(TestDb *p, int iLevel){ 
           26  +  unused_parameter(p);
           27  +  unused_parameter(iLevel);
           28  +  return -1; 
           29  +}
           30  +
           31  +
           32  +/*************************************************************************
           33  +** Begin wrapper for LevelDB.
           34  +*/
           35  +#ifdef HAVE_LEVELDB
           36  +
           37  +#include <leveldb/c.h>
           38  +
           39  +typedef struct LevelDb LevelDb;
           40  +struct LevelDb {
           41  +  TestDb base;
           42  +  leveldb_t *db;
           43  +  leveldb_options_t *pOpt;
           44  +  leveldb_writeoptions_t *pWriteOpt;
           45  +  leveldb_readoptions_t *pReadOpt;
           46  +
           47  +  char *pVal;
           48  +};
           49  +
           50  +static int test_leveldb_close(TestDb *pTestDb){
           51  +  LevelDb *pDb = (LevelDb *)pTestDb;
           52  +
           53  +  leveldb_close(pDb->db);
           54  +  leveldb_writeoptions_destroy(pDb->pWriteOpt);
           55  +  leveldb_readoptions_destroy(pDb->pReadOpt);
           56  +  leveldb_options_destroy(pDb->pOpt);
           57  +  free(pDb->pVal);
           58  +  free(pDb);
           59  +
           60  +  return 0;
           61  +}
           62  +
           63  +static int test_leveldb_write(
           64  +  TestDb *pTestDb, 
           65  +  void *pKey, 
           66  +  int nKey, 
           67  +  void *pVal, 
           68  +  int nVal
           69  +){
           70  +  LevelDb *pDb = (LevelDb *)pTestDb;
           71  +  char *zErr = 0;
           72  +  leveldb_put(pDb->db, pDb->pWriteOpt, pKey, nKey, pVal, nVal, &zErr);
           73  +  return (zErr!=0);
           74  +}
           75  +
           76  +static int test_leveldb_delete(TestDb *pTestDb, void *pKey, int nKey){
           77  +  LevelDb *pDb = (LevelDb *)pTestDb;
           78  +  char *zErr = 0;
           79  +  leveldb_delete(pDb->db, pDb->pWriteOpt, pKey, nKey, &zErr);
           80  +  return (zErr!=0);
           81  +}
           82  +
           83  +static int test_leveldb_fetch(
           84  +  TestDb *pTestDb, 
           85  +  void *pKey, 
           86  +  int nKey, 
           87  +  void **ppVal, 
           88  +  int *pnVal
           89  +){
           90  +  LevelDb *pDb = (LevelDb *)pTestDb;
           91  +  char *zErr = 0;
           92  +  size_t nVal = 0;
           93  +
           94  +  if( pKey==0 ) return 0;
           95  +  free(pDb->pVal);
           96  +  pDb->pVal = leveldb_get(pDb->db, pDb->pReadOpt, pKey, nKey, &nVal, &zErr);
           97  +  *ppVal = (void *)(pDb->pVal);
           98  +  if( pDb->pVal==0 ){
           99  +    *pnVal = -1;
          100  +  }else{
          101  +    *pnVal = (int)nVal;
          102  +  }
          103  +
          104  +  return (zErr!=0);
          105  +}
          106  +
          107  +static int test_leveldb_scan(
          108  +  TestDb *pTestDb,
          109  +  void *pCtx,
          110  +  int bReverse,
          111  +  void *pKey1, int nKey1,         /* Start of search */
          112  +  void *pKey2, int nKey2,         /* End of search */
          113  +  void (*xCallback)(void *, void *, int , void *, int)
          114  +){
          115  +  LevelDb *pDb = (LevelDb *)pTestDb;
          116  +  leveldb_iterator_t *iter;
          117  +
          118  +  iter = leveldb_create_iterator(pDb->db, pDb->pReadOpt);
          119  +
          120  +  if( bReverse==0 ){
          121  +    if( pKey1 ){
          122  +      leveldb_iter_seek(iter, pKey1, nKey1);
          123  +    }else{
          124  +      leveldb_iter_seek_to_first(iter);
          125  +    }
          126  +  }else{
          127  +    if( pKey2 ){
          128  +      leveldb_iter_seek(iter, pKey2, nKey2);
          129  +
          130  +      if( leveldb_iter_valid(iter)==0 ){
          131  +        leveldb_iter_seek_to_last(iter);
          132  +      }else{
          133  +        const char *k; size_t n;
          134  +        int res;
          135  +        k = leveldb_iter_key(iter, &n);
          136  +        res = memcmp(k, pKey2, MIN(n, nKey2));
          137  +        if( res==0 ) res = n - nKey2;
          138  +        assert( res>=0 );
          139  +        if( res>0 ){
          140  +          leveldb_iter_prev(iter);
          141  +        }
          142  +      }
          143  +    }else{
          144  +      leveldb_iter_seek_to_last(iter);
          145  +    }
          146  +  }
          147  +
          148  +
          149  +  while( leveldb_iter_valid(iter) ){
          150  +    const char *k; size_t n;
          151  +    const char *v; size_t n2;
          152  +    int res;
          153  +
          154  +    k = leveldb_iter_key(iter, &n);
          155  +    if( bReverse==0 && pKey2 ){
          156  +      res = memcmp(k, pKey2, MIN(n, nKey2));
          157  +      if( res==0 ) res = n - nKey2;
          158  +      if( res>0 ) break;
          159  +    }
          160  +    if( bReverse!=0 && pKey1 ){
          161  +      res = memcmp(k, pKey1, MIN(n, nKey1));
          162  +      if( res==0 ) res = n - nKey1;
          163  +      if( res<0 ) break;
          164  +    }
          165  +
          166  +    v = leveldb_iter_value(iter, &n2);
          167  +
          168  +    xCallback(pCtx, (void *)k, n, (void *)v, n2);
          169  +
          170  +    if( bReverse==0 ){
          171  +      leveldb_iter_next(iter);
          172  +    }else{
          173  +      leveldb_iter_prev(iter);
          174  +    }
          175  +  }
          176  +
          177  +  leveldb_iter_destroy(iter);
          178  +  return 0;
          179  +}
          180  +
          181  +static int test_leveldb_open(
          182  +  const char *zSpec, 
          183  +  const char *zFilename, 
          184  +  int bClear, 
          185  +  TestDb **ppDb
          186  +){
          187  +  static const DatabaseMethods LeveldbMethods = {
          188  +    test_leveldb_close,
          189  +    test_leveldb_write,
          190  +    test_leveldb_delete,
          191  +    0,
          192  +    test_leveldb_fetch,
          193  +    test_leveldb_scan,
          194  +    error_transaction_function,
          195  +    error_transaction_function,
          196  +    error_transaction_function
          197  +  };
          198  +
          199  +  LevelDb *pLevelDb;
          200  +  char *zErr = 0;
          201  +
          202  +  if( bClear ){
          203  +    char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename);
          204  +    system(zCmd);
          205  +    sqlite3_free(zCmd);
          206  +  }
          207  +
          208  +  pLevelDb = (LevelDb *)malloc(sizeof(LevelDb));
          209  +  memset(pLevelDb, 0, sizeof(LevelDb));
          210  +
          211  +  pLevelDb->pOpt = leveldb_options_create();
          212  +  leveldb_options_set_create_if_missing(pLevelDb->pOpt, 1);
          213  +  pLevelDb->pWriteOpt = leveldb_writeoptions_create();
          214  +  pLevelDb->pReadOpt = leveldb_readoptions_create();
          215  +
          216  +  pLevelDb->db = leveldb_open(pLevelDb->pOpt, zFilename, &zErr);
          217  +
          218  +  if( zErr ){
          219  +    test_leveldb_close((TestDb *)pLevelDb);
          220  +    *ppDb = 0;
          221  +    return 1;
          222  +  }
          223  +
          224  +  *ppDb = (TestDb *)pLevelDb;
          225  +  pLevelDb->base.pMethods = &LeveldbMethods;
          226  +  return 0;
          227  +}
          228  +#endif  /* HAVE_LEVELDB */
          229  +/* 
          230  +** End wrapper for LevelDB.
          231  +*************************************************************************/
          232  +
          233  +#ifdef HAVE_KYOTOCABINET
          234  +static int kc_close(TestDb *pTestDb){
          235  +  return test_kc_close(pTestDb);
          236  +}
          237  +
          238  +static int kc_write(
          239  +  TestDb *pTestDb, 
          240  +  void *pKey, 
          241  +  int nKey, 
          242  +  void *pVal, 
          243  +  int nVal
          244  +){
          245  +  return test_kc_write(pTestDb, pKey, nKey, pVal, nVal);
          246  +}
          247  +
          248  +static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){
          249  +  return test_kc_delete(pTestDb, pKey, nKey);
          250  +}
          251  +
          252  +static int kc_delete_range(
          253  +  TestDb *pTestDb, 
          254  +  void *pKey1, int nKey1,
          255  +  void *pKey2, int nKey2
          256  +){
          257  +  return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2);
          258  +}
          259  +
          260  +static int kc_fetch(
          261  +  TestDb *pTestDb, 
          262  +  void *pKey, 
          263  +  int nKey, 
          264  +  void **ppVal, 
          265  +  int *pnVal
          266  +){
          267  +  if( pKey==0 ) return LSM_OK;
          268  +  return test_kc_fetch(pTestDb, pKey, nKey, ppVal, pnVal);
          269  +}
          270  +
          271  +static int kc_scan(
          272  +  TestDb *pTestDb,
          273  +  void *pCtx,
          274  +  int bReverse,
          275  +  void *pFirst, int nFirst,
          276  +  void *pLast, int nLast,
          277  +  void (*xCallback)(void *, void *, int , void *, int)
          278  +){
          279  +  return test_kc_scan(
          280  +      pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback
          281  +  );
          282  +}
          283  +
          284  +static int kc_open(
          285  +  const char *zSpec, 
          286  +  const char *zFilename, 
          287  +  int bClear, 
          288  +  TestDb **ppDb
          289  +){
          290  +  static const DatabaseMethods KcdbMethods = {
          291  +    kc_close,
          292  +    kc_write,
          293  +    kc_delete,
          294  +    kc_delete_range,
          295  +    kc_fetch,
          296  +    kc_scan,
          297  +    error_transaction_function,
          298  +    error_transaction_function,
          299  +    error_transaction_function
          300  +  };
          301  +
          302  +  int rc;
          303  +  TestDb *pTestDb = 0;
          304  +
          305  +  rc = test_kc_open(zFilename, bClear, &pTestDb);
          306  +  if( rc!=0 ){
          307  +    *ppDb = 0;
          308  +    return rc;
          309  +  }
          310  +  pTestDb->pMethods = &KcdbMethods;
          311  +  *ppDb = pTestDb;
          312  +  return 0;
          313  +}
          314  +#endif /* HAVE_KYOTOCABINET */
          315  +/* 
          316  +** End wrapper for Kyoto cabinet.
          317  +*************************************************************************/
          318  +
          319  +#ifdef HAVE_MDB
          320  +static int mdb_close(TestDb *pTestDb){
          321  +  return test_mdb_close(pTestDb);
          322  +}
          323  +
          324  +static int mdb_write(
          325  +  TestDb *pTestDb, 
          326  +  void *pKey, 
          327  +  int nKey, 
          328  +  void *pVal, 
          329  +  int nVal
          330  +){
          331  +  return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal);
          332  +}
          333  +
          334  +static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){
          335  +  return test_mdb_delete(pTestDb, pKey, nKey);
          336  +}
          337  +
          338  +static int mdb_fetch(
          339  +  TestDb *pTestDb, 
          340  +  void *pKey, 
          341  +  int nKey, 
          342  +  void **ppVal, 
          343  +  int *pnVal
          344  +){
          345  +  if( pKey==0 ) return LSM_OK;
          346  +  return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal);
          347  +}
          348  +
          349  +static int mdb_scan(
          350  +  TestDb *pTestDb,
          351  +  void *pCtx,
          352  +  int bReverse,
          353  +  void *pFirst, int nFirst,
          354  +  void *pLast, int nLast,
          355  +  void (*xCallback)(void *, void *, int , void *, int)
          356  +){
          357  +  return test_mdb_scan(
          358  +      pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback
          359  +  );
          360  +}
          361  +
          362  +static int mdb_open(
          363  +  const char *zSpec, 
          364  +  const char *zFilename, 
          365  +  int bClear, 
          366  +  TestDb **ppDb
          367  +){
          368  +  static const DatabaseMethods KcdbMethods = {
          369  +    mdb_close,
          370  +    mdb_write,
          371  +    mdb_delete,
          372  +    0,
          373  +    mdb_fetch,
          374  +    mdb_scan,
          375  +    error_transaction_function,
          376  +    error_transaction_function,
          377  +    error_transaction_function
          378  +  };
          379  +
          380  +  int rc;
          381  +  TestDb *pTestDb = 0;
          382  +
          383  +  rc = test_mdb_open(zSpec, zFilename, bClear, &pTestDb);
          384  +  if( rc!=0 ){
          385  +    *ppDb = 0;
          386  +    return rc;
          387  +  }
          388  +  pTestDb->pMethods = &KcdbMethods;
          389  +  *ppDb = pTestDb;
          390  +  return 0;
          391  +}
          392  +#endif /* HAVE_MDB */
          393  +
          394  +/*************************************************************************
          395  +** Begin wrapper for SQLite.
          396  +*/
          397  +
          398  +/*
          399  +** nOpenTrans:
          400  +**   The number of open nested transactions, in the same sense as used
          401  +**   by the tdb_begin/commit/rollback and SQLite 4 KV interfaces. If this
          402  +**   value is 0, there are no transactions open at all. If it is 1, then
          403  +**   there is a read transaction. If it is 2 or greater, then there are
          404  +**   (nOpenTrans-1) nested write transactions open.
          405  +*/
          406  +struct SqlDb {
          407  +  TestDb base;
          408  +  sqlite3 *db;
          409  +  sqlite3_stmt *pInsert;
          410  +  sqlite3_stmt *pDelete;
          411  +  sqlite3_stmt *pDeleteRange;
          412  +  sqlite3_stmt *pFetch;
          413  +  sqlite3_stmt *apScan[8];
          414  +
          415  +  int nOpenTrans;
          416  +
          417  +  /* Used by sql_fetch() to allocate space for results */
          418  +  int nAlloc;
          419  +  u8 *aAlloc;
          420  +};
          421  +
          422  +static int sql_close(TestDb *pTestDb){
          423  +  SqlDb *pDb = (SqlDb *)pTestDb;
          424  +  sqlite3_finalize(pDb->pInsert);
          425  +  sqlite3_finalize(pDb->pDelete);
          426  +  sqlite3_finalize(pDb->pDeleteRange);
          427  +  sqlite3_finalize(pDb->pFetch);
          428  +  sqlite3_finalize(pDb->apScan[0]);
          429  +  sqlite3_finalize(pDb->apScan[1]);
          430  +  sqlite3_finalize(pDb->apScan[2]);
          431  +  sqlite3_finalize(pDb->apScan[3]);
          432  +  sqlite3_finalize(pDb->apScan[4]);
          433  +  sqlite3_finalize(pDb->apScan[5]);
          434  +  sqlite3_finalize(pDb->apScan[6]);
          435  +  sqlite3_finalize(pDb->apScan[7]);
          436  +  sqlite3_close(pDb->db);
          437  +  free((char *)pDb->aAlloc);
          438  +  free((char *)pDb);
          439  +  return SQLITE_OK;
          440  +}
          441  +
          442  +static int sql_write(
          443  +  TestDb *pTestDb, 
          444  +  void *pKey, 
          445  +  int nKey, 
          446  +  void *pVal, 
          447  +  int nVal
          448  +){
          449  +  SqlDb *pDb = (SqlDb *)pTestDb;
          450  +  sqlite3_bind_blob(pDb->pInsert, 1, pKey, nKey, SQLITE_STATIC);
          451  +  sqlite3_bind_blob(pDb->pInsert, 2, pVal, nVal, SQLITE_STATIC);
          452  +  sqlite3_step(pDb->pInsert);
          453  +  return sqlite3_reset(pDb->pInsert);
          454  +}
          455  +
          456  +static int sql_delete(TestDb *pTestDb, void *pKey, int nKey){
          457  +  SqlDb *pDb = (SqlDb *)pTestDb;
          458  +  sqlite3_bind_blob(pDb->pDelete, 1, pKey, nKey, SQLITE_STATIC);
          459  +  sqlite3_step(pDb->pDelete);
          460  +  return sqlite3_reset(pDb->pDelete);
          461  +}
          462  +
          463  +static int sql_delete_range(
          464  +  TestDb *pTestDb, 
          465  +  void *pKey1, int nKey1,
          466  +  void *pKey2, int nKey2
          467  +){
          468  +  SqlDb *pDb = (SqlDb *)pTestDb;
          469  +  sqlite3_bind_blob(pDb->pDeleteRange, 1, pKey1, nKey1, SQLITE_STATIC);
          470  +  sqlite3_bind_blob(pDb->pDeleteRange, 2, pKey2, nKey2, SQLITE_STATIC);
          471  +  sqlite3_step(pDb->pDeleteRange);
          472  +  return sqlite3_reset(pDb->pDeleteRange);
          473  +}
          474  +
          475  +static int sql_fetch(
          476  +  TestDb *pTestDb, 
          477  +  void *pKey, 
          478  +  int nKey, 
          479  +  void **ppVal, 
          480  +  int *pnVal
          481  +){
          482  +  SqlDb *pDb = (SqlDb *)pTestDb;
          483  +  int rc;
          484  +
          485  +  sqlite3_reset(pDb->pFetch);
          486  +  if( pKey==0 ){
          487  +    assert( ppVal==0 );
          488  +    assert( pnVal==0 );
          489  +    return LSM_OK;
          490  +  }
          491  +
          492  +  sqlite3_bind_blob(pDb->pFetch, 1, pKey, nKey, SQLITE_STATIC);
          493  +  rc = sqlite3_step(pDb->pFetch);
          494  +  if( rc==SQLITE_ROW ){
          495  +    int nVal = sqlite3_column_bytes(pDb->pFetch, 0);
          496  +    u8 *aVal = (void *)sqlite3_column_blob(pDb->pFetch, 0);
          497  +
          498  +    if( nVal>pDb->nAlloc ){
          499  +      free(pDb->aAlloc);
          500  +      pDb->aAlloc = (u8 *)malloc(nVal*2);
          501  +      pDb->nAlloc = nVal*2;
          502  +    }
          503  +    memcpy(pDb->aAlloc, aVal, nVal);
          504  +    *pnVal = nVal;
          505  +    *ppVal = (void *)pDb->aAlloc;
          506  +  }else{
          507  +    *pnVal = -1;
          508  +    *ppVal = 0;
          509  +  }
          510  +
          511  +  rc = sqlite3_reset(pDb->pFetch);
          512  +  return rc;
          513  +}
          514  +
          515  +static int sql_scan(
          516  +  TestDb *pTestDb,
          517  +  void *pCtx,
          518  +  int bReverse,
          519  +  void *pFirst, int nFirst,
          520  +  void *pLast, int nLast,
          521  +  void (*xCallback)(void *, void *, int , void *, int)
          522  +){
          523  +  SqlDb *pDb = (SqlDb *)pTestDb;
          524  +  sqlite3_stmt *pScan;
          525  +
          526  +  assert( bReverse==1 || bReverse==0 );
          527  +  pScan = pDb->apScan[(pFirst==0) + (pLast==0)*2 + bReverse*4];
          528  +
          529  +  if( pFirst ) sqlite3_bind_blob(pScan, 1, pFirst, nFirst, SQLITE_STATIC);
          530  +  if( pLast ) sqlite3_bind_blob(pScan, 2, pLast, nLast, SQLITE_STATIC);
          531  +
          532  +  while( SQLITE_ROW==sqlite3_step(pScan) ){
          533  +    void *pKey; int nKey;
          534  +    void *pVal; int nVal;
          535  +
          536  +    nKey = sqlite3_column_bytes(pScan, 0);
          537  +    pKey = (void *)sqlite3_column_blob(pScan, 0);
          538  +    nVal = sqlite3_column_bytes(pScan, 1);
          539  +    pVal = (void *)sqlite3_column_blob(pScan, 1);
          540  +
          541  +    xCallback(pCtx, pKey, nKey, pVal, nVal);
          542  +  }
          543  +  return sqlite3_reset(pScan);
          544  +}
          545  +
          546  +static int sql_begin(TestDb *pTestDb, int iLevel){
          547  +  int i;
          548  +  SqlDb *pDb = (SqlDb *)pTestDb;
          549  +
          550  +  /* iLevel==0 is a no-op */
          551  +  if( iLevel==0 ) return 0;
          552  +
          553  +  /* If there are no transactions at all open, open a read transaction. */
          554  +  if( pDb->nOpenTrans==0 ){
          555  +    int rc = sqlite3_exec(pDb->db, 
          556  +        "BEGIN; SELECT * FROM sqlite_master LIMIT 1;" , 0, 0, 0
          557  +    );
          558  +    if( rc!=0 ) return rc;
          559  +    pDb->nOpenTrans = 1;
          560  +  }
          561  +
          562  +  /* Open any required write transactions */
          563  +  for(i=pDb->nOpenTrans; i<iLevel; i++){
          564  +    char *zSql = sqlite3_mprintf("SAVEPOINT x%d", i);
          565  +    int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0);
          566  +    sqlite3_free(zSql);
          567  +    if( rc!=SQLITE_OK ) return rc;
          568  +  }
          569  +
          570  +  pDb->nOpenTrans = iLevel;
          571  +  return 0;
          572  +}
          573  +
          574  +static int sql_commit(TestDb *pTestDb, int iLevel){
          575  +  SqlDb *pDb = (SqlDb *)pTestDb;
          576  +  assert( iLevel>=0 );
          577  +
          578  +  /* Close the read transaction if requested. */
          579  +  if( pDb->nOpenTrans>=1 && iLevel==0 ){
          580  +    int rc = sqlite3_exec(pDb->db, "COMMIT", 0, 0, 0);
          581  +    if( rc!=0 ) return rc;
          582  +    pDb->nOpenTrans = 0;
          583  +  }
          584  +
          585  +  /* Close write transactions as required */
          586  +  if( pDb->nOpenTrans>iLevel ){
          587  +    char *zSql = sqlite3_mprintf("RELEASE x%d", iLevel);
          588  +    int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0);
          589  +    sqlite3_free(zSql);
          590  +    if( rc!=0 ) return rc;
          591  +  }
          592  +
          593  +  pDb->nOpenTrans = iLevel;
          594  +  return 0;
          595  +}
          596  +
          597  +static int sql_rollback(TestDb *pTestDb, int iLevel){
          598  +  SqlDb *pDb = (SqlDb *)pTestDb;
          599  +  assert( iLevel>=0 );
          600  +
          601  +  if( pDb->nOpenTrans>=1 && iLevel==0 ){
          602  +    /* Close the read transaction if requested. */
          603  +    int rc = sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0);
          604  +    if( rc!=0 ) return rc;
          605  +  }else if( pDb->nOpenTrans>1 && iLevel==1 ){
          606  +    /* Or, rollback and close the top-level write transaction */
          607  +    int rc = sqlite3_exec(pDb->db, "ROLLBACK TO x1; RELEASE x1;", 0, 0, 0);
          608  +    if( rc!=0 ) return rc;
          609  +  }else{
          610  +    /* Or, just roll back some nested transactions */
          611  +    char *zSql = sqlite3_mprintf("ROLLBACK TO x%d", iLevel-1);
          612  +    int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0);
          613  +    sqlite3_free(zSql);
          614  +    if( rc!=0 ) return rc;
          615  +  }
          616  +
          617  +  pDb->nOpenTrans = iLevel;
          618  +  return 0;
          619  +}
          620  +
          621  +static int sql_open(
          622  +  const char *zSpec, 
          623  +  const char *zFilename, 
          624  +  int bClear, 
          625  +  TestDb **ppDb
          626  +){
          627  +  static const DatabaseMethods SqlMethods = {
          628  +    sql_close,
          629  +    sql_write,
          630  +    sql_delete,
          631  +    sql_delete_range,
          632  +    sql_fetch,
          633  +    sql_scan,
          634  +    sql_begin,
          635  +    sql_commit,
          636  +    sql_rollback
          637  +  };
          638  +  const char *zCreate = "CREATE TABLE IF NOT EXISTS t1(k PRIMARY KEY, v)";
          639  +  const char *zInsert = "REPLACE INTO t1 VALUES(?, ?)";
          640  +  const char *zDelete = "DELETE FROM t1 WHERE k = ?";
          641  +  const char *zRange = "DELETE FROM t1 WHERE k>? AND k<?";
          642  +  const char *zFetch  = "SELECT v FROM t1 WHERE k = ?";
          643  +
          644  +  const char *zScan0  = "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k";
          645  +  const char *zScan1  = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k";
          646  +  const char *zScan2  = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k";
          647  +  const char *zScan3  = "SELECT * FROM t1 ORDER BY k";
          648  +
          649  +  const char *zScan4  = 
          650  +    "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k DESC";
          651  +  const char *zScan5  = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k DESC";
          652  +  const char *zScan6  = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k DESC";
          653  +  const char *zScan7  = "SELECT * FROM t1 ORDER BY k DESC";
          654  +
          655  +  int rc;
          656  +  SqlDb *pDb;
          657  +  char *zPragma;
          658  +
          659  +  if( bClear && zFilename && zFilename[0] ){
          660  +    unlink(zFilename);
          661  +  }
          662  +
          663  +  pDb = (SqlDb *)malloc(sizeof(SqlDb));
          664  +  memset(pDb, 0, sizeof(SqlDb));
          665  +  pDb->base.pMethods = &SqlMethods;
          666  +
          667  +  if( 0!=(rc = sqlite3_open(zFilename, &pDb->db))
          668  +   || 0!=(rc = sqlite3_exec(pDb->db, zCreate, 0, 0, 0))
          669  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zInsert, -1, &pDb->pInsert, 0))
          670  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zDelete, -1, &pDb->pDelete, 0))
          671  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zRange, -1, &pDb->pDeleteRange, 0))
          672  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zFetch, -1, &pDb->pFetch, 0))
          673  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan0, -1, &pDb->apScan[0], 0))
          674  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan1, -1, &pDb->apScan[1], 0))
          675  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan2, -1, &pDb->apScan[2], 0))
          676  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan3, -1, &pDb->apScan[3], 0))
          677  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan4, -1, &pDb->apScan[4], 0))
          678  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan5, -1, &pDb->apScan[5], 0))
          679  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan6, -1, &pDb->apScan[6], 0))
          680  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan7, -1, &pDb->apScan[7], 0))
          681  +  ){
          682  +    *ppDb = 0;
          683  +    sql_close((TestDb *)pDb);
          684  +    return rc;
          685  +  }
          686  +
          687  +  zPragma = sqlite3_mprintf("PRAGMA page_size=%d", TESTDB_DEFAULT_PAGE_SIZE);
          688  +  sqlite3_exec(pDb->db, zPragma, 0, 0, 0);
          689  +  sqlite3_free(zPragma);
          690  +  zPragma = sqlite3_mprintf("PRAGMA cache_size=%d", TESTDB_DEFAULT_CACHE_SIZE);
          691  +  sqlite3_exec(pDb->db, zPragma, 0, 0, 0);
          692  +  sqlite3_free(zPragma);
          693  +
          694  +  /* sqlite3_exec(pDb->db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0); */
          695  +  sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0);
          696  +  sqlite3_exec(pDb->db, "PRAGMA journal_mode=WAL", 0, 0, 0);
          697  +  sqlite3_exec(pDb->db, "PRAGMA wal_autocheckpoint=4096", 0, 0, 0);
          698  +  if( zSpec ){
          699  +    rc = sqlite3_exec(pDb->db, zSpec, 0, 0, 0);
          700  +    if( rc!=SQLITE_OK ){
          701  +      sql_close((TestDb *)pDb);
          702  +      return rc;
          703  +    }
          704  +  }
          705  +
          706  +  *ppDb = (TestDb *)pDb;
          707  +  return 0;
          708  +}
          709  +/* 
          710  +** End wrapper for SQLite.
          711  +*************************************************************************/
          712  +
          713  +/*************************************************************************
          714  +** Begin exported functions.
          715  +*/
          716  +static struct Lib {
          717  +  const char *zName;
          718  +  const char *zDefaultDb;
          719  +  int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb);
          720  +} aLib[] = {
          721  +  { "sqlite3",      "testdb.sqlite",    sql_open },
          722  +  { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
          723  +  { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
          724  +#ifdef HAVE_ZLIB
          725  +  { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
          726  +#endif
          727  +  { "lsm",          "testdb.lsm",       test_lsm_open },
          728  +#ifdef LSM_MUTEX_PTHREADS
          729  +  { "lsm_mt2",      "testdb.lsm_mt2",   test_lsm_mt2 },
          730  +  { "lsm_mt3",      "testdb.lsm_mt3",   test_lsm_mt3 },
          731  +#endif
          732  +#ifdef HAVE_LEVELDB
          733  +  { "leveldb",      "testdb.leveldb",   test_leveldb_open },
          734  +#endif
          735  +#ifdef HAVE_KYOTOCABINET
          736  +  { "kyotocabinet", "testdb.kc",        kc_open },
          737  +#endif
          738  +#ifdef HAVE_MDB
          739  +  { "mdb", "./testdb.mdb",        mdb_open }
          740  +#endif
          741  +};
          742  +
          743  +const char *tdb_system_name(int i){
          744  +  if( i<0 || i>=ArraySize(aLib) ) return 0;
          745  +  return aLib[i].zName;
          746  +}
          747  +
          748  +const char *tdb_default_db(const char *zSys){
          749  +  int i;
          750  +  for(i=0; i<ArraySize(aLib); i++){
          751  +    if( strcmp(aLib[i].zName, zSys)==0 ) return aLib[i].zDefaultDb;
          752  +  }
          753  +  return 0;
          754  +}
          755  +
          756  +int tdb_open(const char *zLib, const char *zDb, int bClear, TestDb **ppDb){
          757  +  int i;
          758  +  int rc = 1;
          759  +  const char *zSpec = 0;
          760  +
          761  +  int nLib = 0;
          762  +  while( zLib[nLib] && zLib[nLib]!=' ' ){
          763  +    nLib++;
          764  +  }
          765  +  zSpec = &zLib[nLib];
          766  +  while( *zSpec==' ' ) zSpec++;
          767  +  if( *zSpec=='\0' ) zSpec = 0;
          768  +
          769  +  for(i=0; i<ArraySize(aLib); i++){
          770  +    if( strlen(aLib[i].zName)==nLib && 0==memcmp(zLib, aLib[i].zName, nLib) ){
          771  +      rc = aLib[i].xOpen(zSpec, (zDb ? zDb : aLib[i].zDefaultDb), bClear, ppDb);
          772  +      if( rc==0 ){
          773  +        (*ppDb)->zLibrary = aLib[i].zName;
          774  +      }
          775  +      break;
          776  +    }
          777  +  }
          778  +
          779  +  if( rc ){
          780  +    /* Failed to find the requested database library. Return an error. */
          781  +    *ppDb = 0;
          782  +  }
          783  +  return rc;
          784  +}
          785  +
          786  +int tdb_close(TestDb *pDb){
          787  +  if( pDb ){
          788  +    return pDb->pMethods->xClose(pDb);
          789  +  }
          790  +  return 0;
          791  +}
          792  +
          793  +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){
          794  +  return pDb->pMethods->xWrite(pDb, pKey, nKey, pVal, nVal);
          795  +}
          796  +
          797  +int tdb_delete(TestDb *pDb, void *pKey, int nKey){
          798  +  return pDb->pMethods->xDelete(pDb, pKey, nKey);
          799  +}
          800  +
          801  +int tdb_delete_range(
          802  +    TestDb *pDb, void *pKey1, int nKey1, void *pKey2, int nKey2
          803  +){
          804  +  return pDb->pMethods->xDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2);
          805  +}
          806  +
          807  +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal){
          808  +  return pDb->pMethods->xFetch(pDb, pKey, nKey, ppVal, pnVal);
          809  +}
          810  +
          811  +int tdb_scan(
          812  +  TestDb *pDb,                    /* Database handle */
          813  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          814  +  int bReverse,                   /* True to scan in reverse order */
          815  +  void *pKey1, int nKey1,         /* Start of search */
          816  +  void *pKey2, int nKey2,         /* End of search */
          817  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          818  +){
          819  +  return pDb->pMethods->xScan(
          820  +      pDb, pCtx, bReverse, pKey1, nKey1, pKey2, nKey2, xCallback
          821  +  );
          822  +}
          823  +
          824  +int tdb_begin(TestDb *pDb, int iLevel){
          825  +  return pDb->pMethods->xBegin(pDb, iLevel);
          826  +}
          827  +int tdb_commit(TestDb *pDb, int iLevel){
          828  +  return pDb->pMethods->xCommit(pDb, iLevel);
          829  +}
          830  +int tdb_rollback(TestDb *pDb, int iLevel){
          831  +  return pDb->pMethods->xRollback(pDb, iLevel);
          832  +}
          833  +
          834  +int tdb_transaction_support(TestDb *pDb){
          835  +  return (pDb->pMethods->xBegin != error_transaction_function);
          836  +}
          837  +
          838  +const char *tdb_library_name(TestDb *pDb){
          839  +  return pDb->zLibrary;
          840  +}
          841  +
          842  +/* 
          843  +** End exported functions.
          844  +*************************************************************************/

Added ext/lsm1/lsm-test/lsmtest_tdb.h.

            1  +
            2  +/*
            3  +** This file is the interface to a very simple database library used for
            4  +** testing. The interface is similar to that of the LSM. The main virtue 
            5  +** of this library is that the same API may be used to access a key-value
            6  +** store implemented by LSM, SQLite or another database system. Which 
            7  +** makes it easy to use for correctness and performance tests.
            8  +*/
            9  +
           10  +#ifndef __WRAPPER_H_
           11  +#define __WRAPPER_H_
           12  +
           13  +#ifdef __cplusplus
           14  +extern "C" {
           15  +#endif
           16  +
           17  +#include "lsm.h"
           18  +
           19  +typedef struct TestDb TestDb;
           20  +
           21  +/*
           22  +** Open a new database connection. The first argument is the name of the
           23  +** database library to use. e.g. something like:
           24  +**
           25  +**     "sqlite3"
           26  +**     "lsm"
           27  +**
           28  +** See function tdb_system_name() for a list of available database systems.
           29  +**
           30  +** The second argument is the name of the database to open (e.g. a filename).
           31  +**
           32  +** If the third parameter is non-zero, then any existing database by the
           33  +** name of zDb is removed before opening a new one. If it is zero, then an
           34  +** existing database may be opened.
           35  +*/
           36  +int tdb_open(const char *zLibrary, const char *zDb, int bClear, TestDb **ppDb);
           37  +
           38  +/*
           39  +** Close a database handle.
           40  +*/
           41  +int tdb_close(TestDb *pDb);
           42  +
           43  +/*
           44  +** Write a new key/value into the database.
           45  +*/
           46  +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal);
           47  +
           48  +/*
           49  +** Delete a key from the database.
           50  +*/
           51  +int tdb_delete(TestDb *pDb, void *pKey, int nKey);
           52  +
           53  +/*
           54  +** Delete a range of keys from the database.
           55  +*/
           56  +int tdb_delete_range(TestDb *, void *pKey1, int nKey1, void *pKey2, int nKey2);
           57  +
           58  +/*
           59  +** Query the database for key (pKey/nKey). If no entry is found, set *ppVal
           60  +** to 0 and *pnVal to -1 before returning. Otherwise, set *ppVal and *pnVal
           61  +** to a pointer to and size of the value associated with (pKey/nKey).
           62  +*/
           63  +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal);
           64  +
           65  +/*
           66  +** Open and close nested transactions. Currently, these functions only 
           67  +** work for SQLite3 and LSM systems. Use the tdb_transaction_support() 
           68  +** function to determine if a given TestDb handle supports these methods.
           69  +**
           70  +** These functions and the iLevel parameter follow the same conventions as
           71  +** the SQLite 4 transaction interface. Note that this is slightly different
           72  +** from the way LSM does things. As follows:
           73  +**
           74  +** tdb_begin():
           75  +**   A successful call to tdb_begin() with (iLevel>1) guarantees that 
           76  +**   there are at least (iLevel-1) write transactions open. If iLevel==1,
           77  +**   then it guarantees that at least a read-transaction is open. Calling
           78  +**   tdb_begin() with iLevel==0 is a no-op.
           79  +**
           80  +** tdb_commit():
           81  +**   A successful call to tdb_commit() with (iLevel>1) guarantees that 
           82  +**   there are at most (iLevel-1) write transactions open. If iLevel==1,
           83  +**   then it guarantees that there are no write transactions open (although
           84  +**   a read-transaction may remain open).  Calling tdb_commit() with 
           85  +**   iLevel==0 ensures that all transactions, read or write, have been 
           86  +**   closed and committed.
           87  +**
           88  +** tdb_rollback():
           89  +**   This call is similar to tdb_commit(), except that instead of committing
           90  +**   transactions, it reverts them. For example, calling tdb_rollback() with
           91  +**   iLevel==2 ensures that there is at most one write transaction open, and
           92  +**   restores the database to the state that it was in when that transaction
           93  +**   was opened.
           94  +**
           95  +**   In other words, tdb_commit() just closes transactions - tdb_rollback()
           96  +**   closes transactions and then restores the database to the state it
           97  +**   was in before those transactions were even opened.
           98  +*/
           99  +int tdb_begin(TestDb *pDb, int iLevel);
          100  +int tdb_commit(TestDb *pDb, int iLevel);
          101  +int tdb_rollback(TestDb *pDb, int iLevel);
          102  +
          103  +/*
          104  +** Return true if transactions are supported, or false otherwise.
          105  +*/
          106  +int tdb_transaction_support(TestDb *pDb);
          107  +
          108  +/*
          109  +** Return the name of the database library (as passed to tdb_open()) used
          110  +** by the handled passed as the first argument.
          111  +*/
          112  +const char *tdb_library_name(TestDb *pDb);
          113  +
          114  +/*
          115  +** Scan a range of database keys. Invoke the callback function for each
          116  +** key visited.
          117  +*/
          118  +int tdb_scan(
          119  +  TestDb *pDb,                    /* Database handle */
          120  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          121  +  int bReverse,                   /* True to scan in reverse order */
          122  +  void *pKey1, int nKey1,         /* Start of search */
          123  +  void *pKey2, int nKey2,         /* End of search */
          124  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          125  +);
          126  +
          127  +const char *tdb_system_name(int i);
          128  +const char *tdb_default_db(const char *zSys);
          129  +
          130  +int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb);
          131  +
          132  +/*
          133  +** If the TestDb handle passed as an argument is a wrapper around an LSM
          134  +** database, return the LSM handle. Otherwise, if the argument is some other
          135  +** database system, return NULL.
          136  +*/
          137  +lsm_db *tdb_lsm(TestDb *pDb);
          138  +
          139  +/*
          140  +** Return true if the db passed as an argument is a multi-threaded LSM
          141  +** connection.
          142  +*/
          143  +int tdb_lsm_multithread(TestDb *pDb);
          144  +
          145  +/*
          146  +** Return a pointer to the lsm_env object used by all lsm database
          147  +** connections initialized as a copy of the object returned by 
          148  +** lsm_default_env(). It may be modified (e.g. to override functions)
          149  +** if the caller can guarantee that it is not already in use.
          150  +*/
          151  +lsm_env *tdb_lsm_env(void);
          152  +
          153  +/*
          154  +** The following functions only work with LSM database handles. It is
          155  +** illegal to call them with any other type of database handle specified
          156  +** as an argument.
          157  +*/
          158  +void tdb_lsm_enable_log(TestDb *pDb, int bEnable);
          159  +void tdb_lsm_application_crash(TestDb *pDb);
          160  +void tdb_lsm_prepare_system_crash(TestDb *pDb);
          161  +void tdb_lsm_system_crash(TestDb *pDb);
          162  +void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync);
          163  +
          164  +
          165  +void tdb_lsm_safety(TestDb *pDb, int eMode);
          166  +void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *);
          167  +void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*);
          168  +int tdb_lsm_config_str(TestDb *pDb, const char *zStr);
          169  +
          170  +#ifdef __cplusplus
          171  +}  /* End of the 'extern "C"' block */
          172  +#endif
          173  +
          174  +#endif

Added ext/lsm1/lsm-test/lsmtest_tdb2.cc.

            1  +
            2  +
            3  +#include "lsmtest.h"
            4  +#include <stdlib.h>
            5  +
            6  +#ifdef HAVE_KYOTOCABINET
            7  +#include "kcpolydb.h"
            8  +extern "C" {
            9  +  struct KcDb {
           10  +    TestDb base;
           11  +    kyotocabinet::TreeDB* db;
           12  +    char *pVal;
           13  +  };
           14  +}
           15  +
           16  +int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb){
           17  +  KcDb *pKcDb;
           18  +  int ok;
           19  +  int rc = 0;
           20  +
           21  +  if( bClear ){
           22  +    char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename);
           23  +    system(zCmd);
           24  +    sqlite3_free(zCmd);
           25  +  }
           26  +
           27  +  pKcDb = (KcDb *)malloc(sizeof(KcDb));
           28  +  memset(pKcDb, 0, sizeof(KcDb));
           29  +
           30  +
           31  +  pKcDb->db = new kyotocabinet::TreeDB();
           32  +  pKcDb->db->tune_page(TESTDB_DEFAULT_PAGE_SIZE);
           33  +  pKcDb->db->tune_page_cache(
           34  +      TESTDB_DEFAULT_PAGE_SIZE * TESTDB_DEFAULT_CACHE_SIZE
           35  +  );
           36  +  ok = pKcDb->db->open(zFilename,
           37  +      kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE
           38  +  );
           39  +  if( ok==0 ){
           40  +    free(pKcDb);
           41  +    pKcDb = 0;
           42  +    rc = 1;
           43  +  }
           44  +
           45  +  *ppDb = (TestDb *)pKcDb;
           46  +  return rc;
           47  +}
           48  +
           49  +int test_kc_close(TestDb *pDb){
           50  +  KcDb *pKcDb = (KcDb *)pDb;
           51  +  if( pKcDb->pVal ){
           52  +    delete [] pKcDb->pVal;
           53  +  }
           54  +  pKcDb->db->close();
           55  +  delete pKcDb->db;
           56  +  free(pKcDb);
           57  +  return 0;
           58  +}
           59  +
           60  +int test_kc_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){
           61  +  KcDb *pKcDb = (KcDb *)pDb;
           62  +  int ok;
           63  +
           64  +  ok = pKcDb->db->set((const char *)pKey, nKey, (const char *)pVal, nVal);
           65  +  return (ok ? 0 : 1);
           66  +}
           67  +
           68  +int test_kc_delete(TestDb *pDb, void *pKey, int nKey){
           69  +  KcDb *pKcDb = (KcDb *)pDb;
           70  +  int ok;
           71  +
           72  +  ok = pKcDb->db->remove((const char *)pKey, nKey);
           73  +  return (ok ? 0 : 1);
           74  +}
           75  +
           76  +int test_kc_delete_range(
           77  +  TestDb *pDb, 
           78  +  void *pKey1, int nKey1,
           79  +  void *pKey2, int nKey2
           80  +){
           81  +  int res;
           82  +  KcDb *pKcDb = (KcDb *)pDb;
           83  +  kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor();
           84  +
           85  +  if( pKey1 ){
           86  +    res = pCur->jump((const char *)pKey1, nKey1);
           87  +  }else{
           88  +    res = pCur->jump();
           89  +  }
           90  +
           91  +  while( 1 ){
           92  +    const char *pKey; size_t nKey;
           93  +    const char *pVal; size_t nVal;
           94  +
           95  +    pKey = pCur->get(&nKey, &pVal, &nVal);
           96  +    if( pKey==0 ) break;
           97  +
           98  +#ifndef NDEBUG
           99  +    if( pKey1 ){
          100  +      res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey));
          101  +      assert( res>0 || (res==0 && nKey>nKey1) );
          102  +    }
          103  +#endif
          104  +
          105  +    if( pKey2 ){
          106  +      res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey));
          107  +      if( res>0 || (res==0 && (size_t)nKey2<nKey) ){
          108  +        delete [] pKey;
          109  +        break;
          110  +      }
          111  +    }
          112  +    pCur->remove();
          113  +    delete [] pKey;
          114  +  }
          115  +
          116  +  delete pCur;
          117  +  return 0;
          118  +}
          119  +
          120  +int test_kc_fetch(
          121  +  TestDb *pDb, 
          122  +  void *pKey, 
          123  +  int nKey, 
          124  +  void **ppVal,
          125  +  int *pnVal
          126  +){
          127  +  KcDb *pKcDb = (KcDb *)pDb;
          128  +  size_t nVal;
          129  +
          130  +  if( pKcDb->pVal ){
          131  +    delete [] pKcDb->pVal;
          132  +    pKcDb->pVal = 0;
          133  +  }
          134  +
          135  +  pKcDb->pVal = pKcDb->db->get((const char *)pKey, nKey, &nVal);
          136  +  if( pKcDb->pVal ){
          137  +    *ppVal = pKcDb->pVal;
          138  +    *pnVal = nVal;
          139  +  }else{
          140  +    *ppVal = 0;
          141  +    *pnVal = -1;
          142  +  }
          143  +
          144  +  return 0;
          145  +}
          146  +
          147  +int test_kc_scan(
          148  +  TestDb *pDb,                    /* Database handle */
          149  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          150  +  int bReverse,                   /* True for a reverse order scan */
          151  +  void *pKey1, int nKey1,         /* Start of search */
          152  +  void *pKey2, int nKey2,         /* End of search */
          153  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          154  +){
          155  +  KcDb *pKcDb = (KcDb *)pDb;
          156  +  kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor();
          157  +  int res;
          158  +
          159  +  if( bReverse==0 ){
          160  +    if( pKey1 ){
          161  +      res = pCur->jump((const char *)pKey1, nKey1);
          162  +    }else{
          163  +      res = pCur->jump();
          164  +    }
          165  +  }else{
          166  +    if( pKey2 ){
          167  +      res = pCur->jump_back((const char *)pKey2, nKey2);
          168  +    }else{
          169  +      res = pCur->jump_back();
          170  +    }
          171  +  }
          172  +
          173  +  while( res ){
          174  +    const char *pKey; size_t nKey;
          175  +    const char *pVal; size_t nVal;
          176  +    pKey = pCur->get(&nKey, &pVal, &nVal);
          177  +
          178  +    if( bReverse==0 && pKey2 ){
          179  +      res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey));
          180  +      if( res>0 || (res==0 && (size_t)nKey2<nKey) ){
          181  +        delete [] pKey;
          182  +        break;
          183  +      }
          184  +    }else if( bReverse!=0 && pKey1 ){
          185  +      res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey));
          186  +      if( res<0 || (res==0 && (size_t)nKey1>nKey) ){
          187  +        delete [] pKey;
          188  +        break;
          189  +      }
          190  +    }
          191  +
          192  +    xCallback(pCtx, (void *)pKey, (int)nKey, (void *)pVal, (int)nVal);
          193  +    delete [] pKey;
          194  +
          195  +    if( bReverse ){
          196  +      res = pCur->step_back();
          197  +    }else{
          198  +      res = pCur->step();
          199  +    }
          200  +  }
          201  +
          202  +  delete pCur;
          203  +  return 0;
          204  +}
          205  +#endif /* HAVE_KYOTOCABINET */
          206  +
          207  +#ifdef HAVE_MDB 
          208  +#include "lmdb.h"
          209  +
          210  +extern "C" {
          211  +  struct MdbDb {
          212  +    TestDb base;
          213  +    MDB_env *env;
          214  +    MDB_dbi dbi;
          215  +  };
          216  +}
          217  +
          218  +int test_mdb_open(
          219  +  const char *zSpec, 
          220  +  const char *zFilename, 
          221  +  int bClear, 
          222  +  TestDb **ppDb
          223  +){
          224  +  MDB_txn *txn;
          225  +  MdbDb *pMdb;
          226  +  int rc;
          227  +
          228  +  if( bClear ){
          229  +    char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename);
          230  +    system(zCmd);
          231  +    sqlite3_free(zCmd);
          232  +  }
          233  +
          234  +  pMdb = (MdbDb *)malloc(sizeof(MdbDb));
          235  +  memset(pMdb, 0, sizeof(MdbDb));
          236  +
          237  +  rc = mdb_env_create(&pMdb->env);
          238  +  if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024);
          239  +  if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600);
          240  +  if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn);
          241  +  if( rc==0 ){
          242  +    rc = mdb_open(txn, NULL, 0, &pMdb->dbi);
          243  +    mdb_txn_commit(txn);
          244  +  }
          245  +
          246  +  *ppDb = (TestDb *)pMdb;
          247  +  return rc;
          248  +}
          249  +
          250  +int test_mdb_close(TestDb *pDb){
          251  +  MdbDb *pMdb = (MdbDb *)pDb;
          252  +
          253  +  mdb_close(pMdb->env, pMdb->dbi);
          254  +  mdb_env_close(pMdb->env);
          255  +  free(pMdb);
          256  +  return 0;
          257  +}
          258  +
          259  +int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){
          260  +  int rc;
          261  +  MdbDb *pMdb = (MdbDb *)pDb;
          262  +  MDB_val val;
          263  +  MDB_val key;
          264  +  MDB_txn *txn;
          265  +
          266  +  val.mv_size = nVal; 
          267  +  val.mv_data = pVal;
          268  +  key.mv_size = nKey; 
          269  +  key.mv_data = pKey;
          270  +
          271  +  rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn);
          272  +  if( rc==0 ){
          273  +    rc = mdb_put(txn, pMdb->dbi, &key, &val, 0);
          274  +    if( rc==0 ){
          275  +      rc = mdb_txn_commit(txn);
          276  +    }else{
          277  +      mdb_txn_abort(txn);
          278  +    }
          279  +  }
          280  +  
          281  +  return rc;
          282  +}
          283  +
          284  +int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){
          285  +  int rc;
          286  +  MdbDb *pMdb = (MdbDb *)pDb;
          287  +  MDB_val key;
          288  +  MDB_txn *txn;
          289  +
          290  +  key.mv_size = nKey; 
          291  +  key.mv_data = pKey;
          292  +  rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn);
          293  +  if( rc==0 ){
          294  +    rc = mdb_del(txn, pMdb->dbi, &key, 0);
          295  +    if( rc==0 ){
          296  +      rc = mdb_txn_commit(txn);
          297  +    }else{
          298  +      mdb_txn_abort(txn);
          299  +    }
          300  +  }
          301  +  
          302  +  return rc;
          303  +}
          304  +
          305  +int test_mdb_fetch(
          306  +  TestDb *pDb, 
          307  +  void *pKey, 
          308  +  int nKey, 
          309  +  void **ppVal,
          310  +  int *pnVal
          311  +){
          312  +  int rc;
          313  +  MdbDb *pMdb = (MdbDb *)pDb;
          314  +  MDB_val key;
          315  +  MDB_txn *txn;
          316  +
          317  +  key.mv_size = nKey;
          318  +  key.mv_data = pKey;
          319  +
          320  +  rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn);
          321  +  if( rc==0 ){
          322  +    MDB_val val = {0, 0};
          323  +    rc = mdb_get(txn, pMdb->dbi, &key, &val);
          324  +    if( rc==MDB_NOTFOUND ){
          325  +      rc = 0;
          326  +      *ppVal = 0;
          327  +      *pnVal = -1;
          328  +    }else{
          329  +      *ppVal = val.mv_data;
          330  +      *pnVal = val.mv_size;
          331  +    }
          332  +    mdb_txn_commit(txn);
          333  +  }
          334  +
          335  +  return rc;
          336  +}
          337  +
          338  +int test_mdb_scan(
          339  +  TestDb *pDb,                    /* Database handle */
          340  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          341  +  int bReverse,                   /* True for a reverse order scan */
          342  +  void *pKey1, int nKey1,         /* Start of search */
          343  +  void *pKey2, int nKey2,         /* End of search */
          344  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          345  +){
          346  +  MdbDb *pMdb = (MdbDb *)pDb;
          347  +  int rc;
          348  +  MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT;
          349  +  MDB_txn *txn;
          350  +
          351  +  rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn);
          352  +  if( rc==0 ){
          353  +    MDB_cursor *csr;
          354  +    MDB_val key = {0, 0};
          355  +    MDB_val val = {0, 0};
          356  +
          357  +    rc = mdb_cursor_open(txn, pMdb->dbi, &csr);
          358  +    if( rc==0 ){
          359  +      while( mdb_cursor_get(csr, &key, &val, op)==0 ){
          360  +        xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size);
          361  +      }
          362  +      mdb_cursor_close(csr);
          363  +    }
          364  +  }
          365  +
          366  +  return rc;
          367  +}
          368  +
          369  +#endif /* HAVE_MDB */
          370  +

Added ext/lsm1/lsm-test/lsmtest_tdb3.c.

            1  +
            2  +#include "lsmtest_tdb.h"
            3  +#include "lsm.h"
            4  +#include "lsmtest.h"
            5  +
            6  +#include <stdlib.h>
            7  +#include <string.h>
            8  +#include <assert.h>
            9  +#ifndef _WIN32
           10  +# include <unistd.h>
           11  +#endif
           12  +#include <stdio.h>
           13  +
           14  +#ifndef _WIN32
           15  +# include <sys/time.h>
           16  +#endif
           17  +
           18  +typedef struct LsmDb LsmDb;
           19  +typedef struct LsmWorker LsmWorker;
           20  +typedef struct LsmFile LsmFile;
           21  +
           22  +#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024)
           23  +#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024)
           24  +
           25  +#ifdef LSM_MUTEX_PTHREADS
           26  +#include <pthread.h>
           27  +
           28  +#define LSMTEST_THREAD_CKPT      1
           29  +#define LSMTEST_THREAD_WORKER    2
           30  +#define LSMTEST_THREAD_WORKER_AC 3
           31  +
           32  +/*
           33  +** There are several different types of worker threads that run in different
           34  +** test configurations, depending on the value of LsmWorker.eType.
           35  +**
           36  +**   1. Checkpointer.
           37  +**   2. Worker with auto-checkpoint.
           38  +**   3. Worker without auto-checkpoint.
           39  +*/
           40  +struct LsmWorker {
           41  +  LsmDb *pDb;                     /* Main database structure */
           42  +  lsm_db *pWorker;                /* Worker database handle */
           43  +  pthread_t worker_thread;        /* Worker thread */
           44  +  pthread_cond_t worker_cond;     /* Condition var the worker waits on */
           45  +  pthread_mutex_t worker_mutex;   /* Mutex used with worker_cond */
           46  +  int bDoWork;                    /* Set to true by client when there is work */
           47  +  int worker_rc;                  /* Store error code here */
           48  +  int eType;                      /* LSMTEST_THREAD_XXX constant */
           49  +  int bBlock;
           50  +};
           51  +#else
           52  +struct LsmWorker { int worker_rc; int bBlock; };
           53  +#endif
           54  +
           55  +static void mt_shutdown(LsmDb *);
           56  +
           57  +lsm_env *tdb_lsm_env(void){
           58  +  static int bInit = 0;
           59  +  static lsm_env env;
           60  +  if( bInit==0 ){
           61  +    memcpy(&env, lsm_default_env(), sizeof(env));
           62  +    bInit = 1;
           63  +  }
           64  +  return &env;
           65  +}
           66  +
           67  +typedef struct FileSector FileSector;
           68  +typedef struct FileData FileData;
           69  +
           70  +struct FileSector {
           71  +  u8 *aOld;                       /* Old data for this sector */
           72  +};
           73  +
           74  +struct FileData {
           75  +  int nSector;                    /* Allocated size of apSector[] array */
           76  +  FileSector *aSector;            /* Array of file sectors */
           77  +};
           78  +
           79  +/*
           80  +** bPrepareCrash:
           81  +**   If non-zero, the file wrappers maintain enough in-memory data to
           82  +**   simulate the effect of a power-failure on the file-system (i.e. that
           83  +**   unsynced sectors may be written, not written, or overwritten with
           84  +**   arbitrary data when the crash occurs).
           85  +**
           86  +** bCrashed:
           87  +**   Set to true after a crash is simulated. Once this variable is true, all
           88  +**   VFS methods other than xClose() return LSM_IOERR as soon as they are
           89  +**   called (without affecting the contents of the file-system).
           90  +**
           91  +** env:
           92  +**   The environment object used by all lsm_db* handles opened by this
           93  +**   object (i.e. LsmDb.db plus any worker connections). Variable env.pVfsCtx
           94  +**   always points to the containing LsmDb structure.
           95  +*/
           96  +struct LsmDb {
           97  +  TestDb base;                    /* Base class - methods table */
           98  +  lsm_env env;                    /* Environment used by connection db */
           99  +  char *zName;                    /* Database file name */
          100  +  lsm_db *db;                     /* LSM database handle */
          101  +
          102  +  lsm_cursor *pCsr;               /* Cursor held open during read transaction */
          103  +  void *pBuf;                     /* Buffer for tdb_fetch() output */
          104  +  int nBuf;                       /* Allocated (not used) size of pBuf */
          105  +
          106  +  /* Crash testing related state */
          107  +  int bCrashed;                   /* True once a crash has occurred */
          108  +  int nAutoCrash;                 /* Number of syncs until a crash */
          109  +  int bPrepareCrash;              /* True to store writes in memory */
          110  +
          111  +  /* Unsynced data (while crash testing) */
          112  +  int szSector;                   /* Assumed size of disk sectors (512B) */
          113  +  FileData aFile[2];              /* Database and log file data */
          114  +
          115  +  /* Other test instrumentation */
          116  +  int bNoRecovery;                /* If true, assume DMS2 is locked */
          117  +
          118  +  /* Work hook redirection */
          119  +  void (*xWork)(lsm_db *, void *);
          120  +  void *pWorkCtx;
          121  +
          122  +  /* IO logging hook */
          123  +  void (*xWriteHook)(void *, int, lsm_i64, int, int);
          124  +  void *pWriteCtx;
          125  +  
          126  +  /* Worker threads (for lsm_mt) */
          127  +  int nMtMinCkpt;
          128  +  int nMtMaxCkpt;
          129  +  int eMode;
          130  +  int nWorker;
          131  +  LsmWorker *aWorker;
          132  +};
          133  +
          134  +#define LSMTEST_MODE_SINGLETHREAD    1
          135  +#define LSMTEST_MODE_BACKGROUND_CKPT 2
          136  +#define LSMTEST_MODE_BACKGROUND_WORK 3
          137  +#define LSMTEST_MODE_BACKGROUND_BOTH 4
          138  +
          139  +/*************************************************************************
          140  +**************************************************************************
          141  +** Begin test VFS code.
          142  +*/
          143  +
          144  +struct LsmFile {
          145  +  lsm_file *pReal;                /* Real underlying file */
          146  +  int bLog;                       /* True for log file. False for db file */
          147  +  LsmDb *pDb;                     /* Database handle that uses this file */
          148  +};
          149  +
          150  +static int testEnvFullpath(
          151  +  lsm_env *pEnv,                  /* Environment for current LsmDb */
          152  +  const char *zFile,              /* Relative path name */
          153  +  char *zOut,                     /* Output buffer */
          154  +  int *pnOut                      /* IN/OUT: Size of output buffer */
          155  +){
          156  +  lsm_env *pRealEnv = tdb_lsm_env();
          157  +  return pRealEnv->xFullpath(pRealEnv, zFile, zOut, pnOut);
          158  +}
          159  +
          160  +static int testEnvOpen(
          161  +  lsm_env *pEnv,                  /* Environment for current LsmDb */
          162  +  const char *zFile,              /* Name of file to open */
          163  +  int flags,
          164  +  lsm_file **ppFile               /* OUT: New file handle object */
          165  +){
          166  +  lsm_env *pRealEnv = tdb_lsm_env();
          167  +  LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx;
          168  +  int rc;                         /* Return Code */
          169  +  LsmFile *pRet;                  /* The new file handle */
          170  +  int nFile;                      /* Length of string zFile in bytes */
          171  +
          172  +  nFile = strlen(zFile);
          173  +  pRet = (LsmFile *)testMalloc(sizeof(LsmFile));
          174  +  pRet->pDb = pDb;
          175  +  pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4));
          176  +
          177  +  rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal);
          178  +  if( rc!=LSM_OK ){
          179  +    testFree(pRet);
          180  +    pRet = 0;
          181  +  }
          182  +
          183  +  *ppFile = (lsm_file *)pRet;
          184  +  return rc;
          185  +}
          186  +
          187  +static int testEnvRead(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){
          188  +  lsm_env *pRealEnv = tdb_lsm_env();
          189  +  LsmFile *p = (LsmFile *)pFile;
          190  +  if( p->pDb->bCrashed ) return LSM_IOERR;
          191  +  return pRealEnv->xRead(p->pReal, iOff, pData, nData);
          192  +}
          193  +
          194  +static int testEnvWrite(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){
          195  +  lsm_env *pRealEnv = tdb_lsm_env();
          196  +  LsmFile *p = (LsmFile *)pFile;
          197  +  LsmDb *pDb = p->pDb;
          198  +
          199  +  if( pDb->bCrashed ) return LSM_IOERR;
          200  +
          201  +  if( pDb->bPrepareCrash ){
          202  +    FileData *pData = &pDb->aFile[p->bLog];
          203  +    int iFirst;                 
          204  +    int iLast;
          205  +    int iSector;
          206  +
          207  +    iFirst = (iOff / pDb->szSector);
          208  +    iLast =  ((iOff + nData - 1) / pDb->szSector);
          209  +
          210  +    if( pData->nSector<(iLast+1) ){
          211  +      int nNew = ( ((iLast + 1) + 63) / 64 ) * 64;
          212  +      assert( nNew>iLast );
          213  +      pData->aSector = (FileSector *)testRealloc(
          214  +          pData->aSector, nNew*sizeof(FileSector)
          215  +      );
          216  +      memset(&pData->aSector[pData->nSector], 
          217  +          0, (nNew - pData->nSector) * sizeof(FileSector)
          218  +      );
          219  +      pData->nSector = nNew;
          220  +    }
          221  +
          222  +    for(iSector=iFirst; iSector<=iLast; iSector++){
          223  +      if( pData->aSector[iSector].aOld==0 ){
          224  +        u8 *aOld = (u8 *)testMalloc(pDb->szSector);
          225  +        pRealEnv->xRead(
          226  +            p->pReal, (lsm_i64)iSector*pDb->szSector, aOld, pDb->szSector
          227  +        );
          228  +        pData->aSector[iSector].aOld = aOld;
          229  +      }
          230  +    }
          231  +  }
          232  +
          233  +  if( pDb->xWriteHook ){
          234  +    int rc;
          235  +    int nUs;
          236  +    struct timeval t1;
          237  +    struct timeval t2;
          238  +
          239  +    gettimeofday(&t1, 0);
          240  +    assert( nData>0 );
          241  +    rc = pRealEnv->xWrite(p->pReal, iOff, pData, nData);
          242  +    gettimeofday(&t2, 0);
          243  +
          244  +    nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec);
          245  +    pDb->xWriteHook(pDb->pWriteCtx, p->bLog, iOff, nData, nUs);
          246  +    return rc;
          247  +  }
          248  +
          249  +  return pRealEnv->xWrite(p->pReal, iOff, pData, nData);
          250  +}
          251  +
          252  +static void doSystemCrash(LsmDb *pDb);
          253  +
          254  +static int testEnvSync(lsm_file *pFile){
          255  +  lsm_env *pRealEnv = tdb_lsm_env();
          256  +  LsmFile *p = (LsmFile *)pFile;
          257  +  LsmDb *pDb = p->pDb;
          258  +  FileData *pData = &pDb->aFile[p->bLog];
          259  +  int i;
          260  +
          261  +  if( pDb->bCrashed ) return LSM_IOERR;
          262  +
          263  +  if( pDb->nAutoCrash ){
          264  +    pDb->nAutoCrash--;
          265  +    if( pDb->nAutoCrash==0 ){
          266  +      doSystemCrash(pDb);
          267  +      pDb->bCrashed = 1;
          268  +      return LSM_IOERR;
          269  +    }
          270  +  }
          271  +
          272  +  if( pDb->bPrepareCrash ){
          273  +    for(i=0; i<pData->nSector; i++){
          274  +      testFree(pData->aSector[i].aOld);
          275  +      pData->aSector[i].aOld = 0;
          276  +    }
          277  +  }
          278  +
          279  +  if( pDb->xWriteHook ){
          280  +    int rc;
          281  +    int nUs;
          282  +    struct timeval t1;
          283  +    struct timeval t2;
          284  +
          285  +    gettimeofday(&t1, 0);
          286  +    rc = pRealEnv->xSync(p->pReal);
          287  +    gettimeofday(&t2, 0);
          288  +
          289  +    nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec);
          290  +    pDb->xWriteHook(pDb->pWriteCtx, p->bLog, 0, 0, nUs);
          291  +    return rc;
          292  +  }
          293  +
          294  +  return pRealEnv->xSync(p->pReal);
          295  +}
          296  +
          297  +static int testEnvTruncate(lsm_file *pFile, lsm_i64 iOff){
          298  +  lsm_env *pRealEnv = tdb_lsm_env();
          299  +  LsmFile *p = (LsmFile *)pFile;
          300  +  if( p->pDb->bCrashed ) return LSM_IOERR;
          301  +  return pRealEnv->xTruncate(p->pReal, iOff);
          302  +}
          303  +
          304  +static int testEnvSectorSize(lsm_file *pFile){
          305  +  lsm_env *pRealEnv = tdb_lsm_env();
          306  +  LsmFile *p = (LsmFile *)pFile;
          307  +  return pRealEnv->xSectorSize(p->pReal);
          308  +}
          309  +
          310  +static int testEnvRemap(
          311  +  lsm_file *pFile, 
          312  +  lsm_i64 iMin, 
          313  +  void **ppOut,
          314  +  lsm_i64 *pnOut
          315  +){
          316  +  lsm_env *pRealEnv = tdb_lsm_env();
          317  +  LsmFile *p = (LsmFile *)pFile;
          318  +  return pRealEnv->xRemap(p->pReal, iMin, ppOut, pnOut);
          319  +}
          320  +
          321  +static int testEnvFileid(
          322  +  lsm_file *pFile, 
          323  +  void *ppOut,
          324  +  int *pnOut
          325  +){
          326  +  lsm_env *pRealEnv = tdb_lsm_env();
          327  +  LsmFile *p = (LsmFile *)pFile;
          328  +  return pRealEnv->xFileid(p->pReal, ppOut, pnOut);
          329  +}
          330  +
          331  +static int testEnvClose(lsm_file *pFile){
          332  +  lsm_env *pRealEnv = tdb_lsm_env();
          333  +  LsmFile *p = (LsmFile *)pFile;
          334  +
          335  +  pRealEnv->xClose(p->pReal);
          336  +  testFree(p);
          337  +  return LSM_OK;
          338  +}
          339  +
          340  +static int testEnvUnlink(lsm_env *pEnv, const char *zFile){
          341  +  lsm_env *pRealEnv = tdb_lsm_env();
          342  +  unused_parameter(pEnv);
          343  +  return pRealEnv->xUnlink(pRealEnv, zFile);
          344  +}
          345  +
          346  +static int testEnvLock(lsm_file *pFile, int iLock, int eType){
          347  +  LsmFile *p = (LsmFile *)pFile;
          348  +  lsm_env *pRealEnv = tdb_lsm_env();
          349  +
          350  +  if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){
          351  +    return LSM_BUSY;
          352  +  }
          353  +  return pRealEnv->xLock(p->pReal, iLock, eType);
          354  +}
          355  +
          356  +static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){
          357  +  LsmFile *p = (LsmFile *)pFile;
          358  +  lsm_env *pRealEnv = tdb_lsm_env();
          359  +
          360  +  if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){
          361  +    return LSM_BUSY;
          362  +  }
          363  +  return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType);
          364  +}
          365  +
          366  +static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){
          367  +  LsmFile *p = (LsmFile *)pFile;
          368  +  lsm_env *pRealEnv = tdb_lsm_env();
          369  +  return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp);
          370  +}
          371  +
          372  +static void testEnvShmBarrier(void){
          373  +}
          374  +
          375  +static int testEnvShmUnmap(lsm_file *pFile, int bDel){
          376  +  LsmFile *p = (LsmFile *)pFile;
          377  +  lsm_env *pRealEnv = tdb_lsm_env();
          378  +  return pRealEnv->xShmUnmap(p->pReal, bDel);
          379  +}
          380  +
          381  +static int testEnvSleep(lsm_env *pEnv, int us){
          382  +  lsm_env *pRealEnv = tdb_lsm_env();
          383  +  return pRealEnv->xSleep(pRealEnv, us);
          384  +}
          385  +
          386  +static void doSystemCrash(LsmDb *pDb){
          387  +  lsm_env *pEnv = tdb_lsm_env();
          388  +  int iFile;
          389  +  int iSeed = pDb->aFile[0].nSector + pDb->aFile[1].nSector;
          390  +
          391  +  char *zFile = pDb->zName;
          392  +  char *zFree = 0;
          393  +
          394  +  for(iFile=0; iFile<2; iFile++){
          395  +    lsm_file *pFile = 0;
          396  +    int i;
          397  +
          398  +    pEnv->xOpen(pEnv, zFile, 0, &pFile);
          399  +    for(i=0; i<pDb->aFile[iFile].nSector; i++){
          400  +      u8 *aOld = pDb->aFile[iFile].aSector[i].aOld;
          401  +      if( aOld ){
          402  +        int iOpt = testPrngValue(iSeed++) % 3;
          403  +        switch( iOpt ){
          404  +          case 0:
          405  +            break;
          406  +
          407  +          case 1:
          408  +            testPrngArray(iSeed++, (u32 *)aOld, pDb->szSector/4);
          409  +            /* Fall-through */
          410  +
          411  +          case 2:
          412  +            pEnv->xWrite(
          413  +                pFile, (lsm_i64)i * pDb->szSector, aOld, pDb->szSector
          414  +            );
          415  +            break;
          416  +        }
          417  +        testFree(aOld);
          418  +        pDb->aFile[iFile].aSector[i].aOld = 0;
          419  +      }
          420  +    }
          421  +    pEnv->xClose(pFile);
          422  +    zFree = zFile = sqlite3_mprintf("%s-log", pDb->zName);
          423  +  }
          424  +
          425  +  sqlite3_free(zFree);
          426  +}
          427  +/*
          428  +** End test VFS code.
          429  +**************************************************************************
          430  +*************************************************************************/
          431  +
          432  +/*************************************************************************
          433  +**************************************************************************
          434  +** Begin test compression hooks.
          435  +*/
          436  +
          437  +#ifdef HAVE_ZLIB
          438  +#include <zlib.h>
          439  +
          440  +static int testZipBound(void *pCtx, int nSrc){
          441  +  return compressBound(nSrc);
          442  +}
          443  +
          444  +static int testZipCompress(
          445  +  void *pCtx,                     /* Context pointer */
          446  +  char *aOut, int *pnOut,         /* OUT: Buffer containing compressed data */
          447  +  const char *aIn, int nIn        /* Buffer containing input data */
          448  +){
          449  +  uLongf n = *pnOut;              /* In/out buffer size for compress() */
          450  +  int rc;                         /* compress() return code */
          451  + 
          452  +  rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn);
          453  +  *pnOut = n;
          454  +  return (rc==Z_OK ? 0 : LSM_ERROR);
          455  +}
          456  +
          457  +static int testZipUncompress(
          458  +  void *pCtx,                     /* Context pointer */
          459  +  char *aOut, int *pnOut,         /* OUT: Buffer containing uncompressed data */
          460  +  const char *aIn, int nIn        /* Buffer containing input data */
          461  +){
          462  +  uLongf n = *pnOut;              /* In/out buffer size for uncompress() */
          463  +  int rc;                         /* uncompress() return code */
          464  +
          465  +  rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn);
          466  +  *pnOut = n;
          467  +  return (rc==Z_OK ? 0 : LSM_ERROR);
          468  +}
          469  +
          470  +static int testConfigureCompression(lsm_db *pDb){
          471  +  static lsm_compress zip = {
          472  +    0,                            /* Context pointer (unused) */
          473  +    1,                            /* Id value */
          474  +    testZipBound,                 /* xBound method */
          475  +    testZipCompress,              /* xCompress method */
          476  +    testZipUncompress             /* xUncompress method */
          477  +  };
          478  +  return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip);
          479  +}
          480  +#endif /* ifdef HAVE_ZLIB */
          481  +
          482  +/*
          483  +** End test compression hooks.
          484  +**************************************************************************
          485  +*************************************************************************/
          486  +
          487  +static int test_lsm_close(TestDb *pTestDb){
          488  +  int i;
          489  +  int rc = LSM_OK;
          490  +  LsmDb *pDb = (LsmDb *)pTestDb;
          491  +
          492  +  lsm_csr_close(pDb->pCsr);
          493  +  lsm_close(pDb->db);
          494  +
          495  +  /* If this is a multi-threaded database, wait on the worker threads. */
          496  +  mt_shutdown(pDb);
          497  +  for(i=0; i<pDb->nWorker && rc==LSM_OK; i++){
          498  +    rc = pDb->aWorker[i].worker_rc;
          499  +  }
          500  +
          501  +  for(i=0; i<pDb->aFile[0].nSector; i++){
          502  +    testFree(pDb->aFile[0].aSector[i].aOld);
          503  +  }
          504  +  testFree(pDb->aFile[0].aSector);
          505  +  for(i=0; i<pDb->aFile[1].nSector; i++){
          506  +    testFree(pDb->aFile[1].aSector[i].aOld);
          507  +  }
          508  +  testFree(pDb->aFile[1].aSector);
          509  +
          510  +  memset(pDb, sizeof(LsmDb), 0x11);
          511  +  testFree((char *)pDb->pBuf);
          512  +  testFree((char *)pDb);
          513  +  return rc;
          514  +}
          515  +
          516  +static void mt_signal_worker(LsmDb*, int);
          517  +
          518  +static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){
          519  +  int nSleep = 0;
          520  +  int nKB;
          521  +  int rc;
          522  +
          523  +  do {
          524  +    nKB = 0;
          525  +    rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB);
          526  +    if( rc!=LSM_OK || nKB<pDb->nMtMaxCkpt ) break;
          527  +#ifdef LSM_MUTEX_PTHREADS
          528  +    mt_signal_worker(pDb, 
          529  +        (pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ? 0 : 1)
          530  +    );
          531  +#endif
          532  +    usleep(5000);
          533  +    nSleep += 5;
          534  +  }while( 1 );
          535  +
          536  +#if 0
          537  +    if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep);
          538  +#endif
          539  +
          540  +  return rc;
          541  +}
          542  +
          543  +static int waitOnWorker(LsmDb *pDb){
          544  +  int rc;
          545  +  int nLimit = -1;
          546  +  int nSleep = 0;
          547  +
          548  +  rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit);
          549  +  do {
          550  +    int nOld, nNew, rc;
          551  +    rc = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew);
          552  +    if( rc!=LSM_OK ) return rc;
          553  +    if( nOld==0 || nNew<(nLimit/2) ) break;
          554  +#ifdef LSM_MUTEX_PTHREADS
          555  +    mt_signal_worker(pDb, 0);
          556  +#endif
          557  +    usleep(5000);
          558  +    nSleep += 5;
          559  +  }while( 1 );
          560  +
          561  +#if 0
          562  +  if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep);
          563  +#endif
          564  +
          565  +  return rc;
          566  +}
          567  +
          568  +static int test_lsm_write(
          569  +  TestDb *pTestDb, 
          570  +  void *pKey, 
          571  +  int nKey, 
          572  +  void *pVal,
          573  +  int nVal
          574  +){
          575  +  LsmDb *pDb = (LsmDb *)pTestDb;
          576  +  int rc = LSM_OK;
          577  +
          578  +  if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){
          579  +    rc = waitOnCheckpointer(pDb, pDb->db);
          580  +  }else if( 
          581  +      pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK
          582  +   || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH 
          583  +  ){
          584  +    rc = waitOnWorker(pDb);
          585  +  }
          586  +
          587  +  if( rc==LSM_OK ){
          588  +    rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal);
          589  +  }
          590  +  return rc;
          591  +}
          592  +
          593  +static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){
          594  +  LsmDb *pDb = (LsmDb *)pTestDb;
          595  +  return lsm_delete(pDb->db, pKey, nKey);
          596  +}
          597  +
          598  +static int test_lsm_delete_range(
          599  +  TestDb *pTestDb, 
          600  +  void *pKey1, int nKey1,
          601  +  void *pKey2, int nKey2
          602  +){
          603  +  LsmDb *pDb = (LsmDb *)pTestDb;
          604  +  return lsm_delete_range(pDb->db, pKey1, nKey1, pKey2, nKey2);
          605  +}
          606  +
          607  +static int test_lsm_fetch(
          608  +  TestDb *pTestDb, 
          609  +  void *pKey, 
          610  +  int nKey, 
          611  +  void **ppVal, 
          612  +  int *pnVal
          613  +){
          614  +  int rc;
          615  +  LsmDb *pDb = (LsmDb *)pTestDb;
          616  +  lsm_cursor *csr;
          617  +
          618  +  if( pKey==0 ) return LSM_OK;
          619  +
          620  +  rc = lsm_csr_open(pDb->db, &csr);
          621  +  if( rc!=LSM_OK ) return rc;
          622  +
          623  +  rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ);
          624  +  if( rc==LSM_OK ){
          625  +    if( lsm_csr_valid(csr) ){
          626  +      const void *pVal; int nVal;
          627  +      rc = lsm_csr_value(csr, &pVal, &nVal);
          628  +      if( nVal>pDb->nBuf ){
          629  +        testFree(pDb->pBuf);
          630  +        pDb->pBuf = testMalloc(nVal*2);
          631  +        pDb->nBuf = nVal*2;
          632  +      }
          633  +      memcpy(pDb->pBuf, pVal, nVal);
          634  +      *ppVal = pDb->pBuf;
          635  +      *pnVal = nVal;
          636  +    }else{
          637  +      *ppVal = 0;
          638  +      *pnVal = -1;
          639  +    }
          640  +  }
          641  +  lsm_csr_close(csr);
          642  +  return rc;
          643  +}
          644  +
          645  +static int test_lsm_scan(
          646  +  TestDb *pTestDb,
          647  +  void *pCtx,
          648  +  int bReverse,
          649  +  void *pFirst, int nFirst,
          650  +  void *pLast, int nLast,
          651  +  void (*xCallback)(void *, void *, int , void *, int)
          652  +){
          653  +  LsmDb *pDb = (LsmDb *)pTestDb;
          654  +  lsm_cursor *csr;
          655  +  int rc;
          656  +
          657  +  rc = lsm_csr_open(pDb->db, &csr);
          658  +  if( rc!=LSM_OK ) return rc;
          659  +
          660  +  if( bReverse ){
          661  +    if( pLast ){
          662  +      rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_LE);
          663  +    }else{
          664  +      rc = lsm_csr_last(csr);
          665  +    }
          666  +  }else{
          667  +    if( pFirst ){
          668  +      rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_GE);
          669  +    }else{
          670  +      rc = lsm_csr_first(csr);
          671  +    }
          672  +  }
          673  +
          674  +  while( rc==LSM_OK && lsm_csr_valid(csr) ){
          675  +    const void *pKey; int nKey;
          676  +    const void *pVal; int nVal;
          677  +    int cmp;
          678  +
          679  +    lsm_csr_key(csr, &pKey, &nKey);
          680  +    lsm_csr_value(csr, &pVal, &nVal);
          681  +
          682  +    if( bReverse && pFirst ){
          683  +      cmp = memcmp(pFirst, pKey, MIN(nKey, nFirst));
          684  +      if( cmp>0 || (cmp==0 && nFirst>nKey) ) break;
          685  +    }else if( bReverse==0 && pLast ){
          686  +      cmp = memcmp(pLast, pKey, MIN(nKey, nLast));
          687  +      if( cmp<0 || (cmp==0 && nLast<nKey) ) break;
          688  +    }
          689  +
          690  +    xCallback(pCtx, (void *)pKey, nKey, (void *)pVal, nVal);
          691  +
          692  +    if( bReverse ){
          693  +      rc = lsm_csr_prev(csr);
          694  +    }else{
          695  +      rc = lsm_csr_next(csr);
          696  +    }
          697  +  }
          698  +
          699  +  lsm_csr_close(csr);
          700  +  return rc;
          701  +}
          702  +
          703  +static int test_lsm_begin(TestDb *pTestDb, int iLevel){
          704  +  int rc = LSM_OK;
          705  +  LsmDb *pDb = (LsmDb *)pTestDb;
          706  +
          707  +  /* iLevel==0 is a no-op. */
          708  +  if( iLevel==0 ) return 0;
          709  +
          710  +  if( pDb->pCsr==0 ) rc = lsm_csr_open(pDb->db, &pDb->pCsr);
          711  +  if( rc==LSM_OK && iLevel>1 ){
          712  +    rc = lsm_begin(pDb->db, iLevel-1);
          713  +  }
          714  +
          715  +  return rc;
          716  +}
          717  +static int test_lsm_commit(TestDb *pTestDb, int iLevel){
          718  +  LsmDb *pDb = (LsmDb *)pTestDb;
          719  +
          720  +  /* If iLevel==0, close any open read transaction */
          721  +  if( iLevel==0 && pDb->pCsr ){
          722  +    lsm_csr_close(pDb->pCsr);
          723  +    pDb->pCsr = 0;
          724  +  }
          725  +
          726  +  /* If iLevel==0, close any open read transaction */
          727  +  return lsm_commit(pDb->db, MAX(0, iLevel-1));
          728  +}
          729  +static int test_lsm_rollback(TestDb *pTestDb, int iLevel){
          730  +  LsmDb *pDb = (LsmDb *)pTestDb;
          731  +
          732  +  /* If iLevel==0, close any open read transaction */
          733  +  if( iLevel==0 && pDb->pCsr ){
          734  +    lsm_csr_close(pDb->pCsr);
          735  +    pDb->pCsr = 0;
          736  +  }
          737  +
          738  +  return lsm_rollback(pDb->db, MAX(0, iLevel-1));
          739  +}
          740  +
          741  +/*
          742  +** A log message callback registered with lsm connections. Prints all 
          743  +** messages to stderr.
          744  +*/
          745  +static void xLog(void *pCtx, int rc, const char *z){
          746  +  unused_parameter(rc);
          747  +  /* fprintf(stderr, "lsm: rc=%d \"%s\"\n", rc, z); */
          748  +  if( pCtx ) fprintf(stderr, "%s: ", (char *)pCtx);
          749  +  fprintf(stderr, "%s\n", z);
          750  +  fflush(stderr);
          751  +}
          752  +
          753  +static void xWorkHook(lsm_db *db, void *pArg){
          754  +  LsmDb *p = (LsmDb *)pArg;
          755  +  if( p->xWork ) p->xWork(db, p->pWorkCtx);
          756  +}
          757  +
          758  +#define TEST_NO_RECOVERY -1
          759  +#define TEST_COMPRESSION -3
          760  +
          761  +#define TEST_MT_MODE     -2
          762  +#define TEST_MT_MIN_CKPT -4
          763  +#define TEST_MT_MAX_CKPT -5
          764  +
          765  +int test_lsm_config_str(
          766  +  LsmDb *pLsm,
          767  +  lsm_db *db, 
          768  +  int bWorker,
          769  +  const char *zStr,
          770  +  int *pnThread
          771  +){
          772  +  struct CfgParam {
          773  +    const char *zParam;
          774  +    int bWorker;
          775  +    int eParam;
          776  +  } aParam[] = {
          777  +    { "autoflush",        0, LSM_CONFIG_AUTOFLUSH },
          778  +    { "page_size",        0, LSM_CONFIG_PAGE_SIZE },
          779  +    { "block_size",       0, LSM_CONFIG_BLOCK_SIZE },
          780  +    { "safety",           0, LSM_CONFIG_SAFETY },
          781  +    { "autowork",         0, LSM_CONFIG_AUTOWORK },
          782  +    { "autocheckpoint",   0, LSM_CONFIG_AUTOCHECKPOINT },
          783  +    { "mmap",             0, LSM_CONFIG_MMAP },
          784  +    { "use_log",          0, LSM_CONFIG_USE_LOG },
          785  +    { "automerge",        0, LSM_CONFIG_AUTOMERGE },
          786  +    { "max_freelist",     0, LSM_CONFIG_MAX_FREELIST },
          787  +    { "multi_proc",       0, LSM_CONFIG_MULTIPLE_PROCESSES },
          788  +    { "worker_automerge", 1, LSM_CONFIG_AUTOMERGE },
          789  +    { "test_no_recovery", 0, TEST_NO_RECOVERY },
          790  +    { "bg_min_ckpt",      0, TEST_NO_RECOVERY },
          791  +
          792  +    { "mt_mode",          0, TEST_MT_MODE },
          793  +    { "mt_min_ckpt",      0, TEST_MT_MIN_CKPT },
          794  +    { "mt_max_ckpt",      0, TEST_MT_MAX_CKPT },
          795  +
          796  +#ifdef HAVE_ZLIB
          797  +    { "compression",      0, TEST_COMPRESSION },
          798  +#endif
          799  +    { 0, 0 }
          800  +  };
          801  +  const char *z = zStr;
          802  +  int nThread = 1;
          803  +
          804  +  if( zStr==0 ) return 0;
          805  +
          806  +  assert( db );
          807  +  while( z[0] ){
          808  +    const char *zStart;
          809  +
          810  +    /* Skip whitespace */
          811  +    while( *z==' ' ) z++;
          812  +    zStart = z;
          813  +
          814  +    while( *z && *z!='=' ) z++;
          815  +    if( *z ){
          816  +      int eParam;
          817  +      int i;
          818  +      int iVal;
          819  +      int iMul = 1;
          820  +      int rc;
          821  +      char zParam[32];
          822  +      int nParam = z-zStart;
          823  +      if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error;
          824  +
          825  +      memcpy(zParam, zStart, nParam);
          826  +      zParam[nParam] = '\0';
          827  +      rc = testArgSelect(aParam, "param", zParam, &i);
          828  +      if( rc!=0 ) return rc;
          829  +      eParam = aParam[i].eParam;
          830  +
          831  +      z++;
          832  +      zStart = z;
          833  +      while( *z>='0' && *z<='9' ) z++;
          834  +      if( *z=='k' || *z=='K' ){
          835  +        iMul = 1;
          836  +        z++;
          837  +      }else if( *z=='M' || *z=='M' ){
          838  +        iMul = 1024;
          839  +        z++;
          840  +      }
          841  +      nParam = z-zStart;
          842  +      if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error;
          843  +      memcpy(zParam, zStart, nParam);
          844  +      zParam[nParam] = '\0';
          845  +      iVal = atoi(zParam) * iMul;
          846  +
          847  +      if( eParam>0 ){
          848  +        if( bWorker || aParam[i].bWorker==0 ){
          849  +          lsm_config(db, eParam, &iVal);
          850  +        }
          851  +      }else{
          852  +        switch( eParam ){
          853  +          case TEST_NO_RECOVERY:
          854  +            if( pLsm ) pLsm->bNoRecovery = iVal;
          855  +            break;
          856  +          case TEST_MT_MODE:
          857  +            if( pLsm ) nThread = iVal;
          858  +            break;
          859  +          case TEST_MT_MIN_CKPT:
          860  +            if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal*1024;
          861  +            break;
          862  +          case TEST_MT_MAX_CKPT:
          863  +            if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal*1024;
          864  +            break;
          865  +#ifdef HAVE_ZLIB
          866  +          case TEST_COMPRESSION:
          867  +            testConfigureCompression(db);
          868  +            break;
          869  +#endif
          870  +        }
          871  +      }
          872  +    }else if( z!=zStart ){
          873  +      goto syntax_error;
          874  +    }
          875  +  }
          876  +
          877  +  if( pnThread ) *pnThread = nThread;
          878  +  if( pLsm && pLsm->nMtMaxCkpt < pLsm->nMtMinCkpt ){
          879  +    pLsm->nMtMinCkpt = pLsm->nMtMaxCkpt;
          880  +  }
          881  +
          882  +  return 0;
          883  + syntax_error:
          884  +  testPrintError("syntax error at: \"%s\"\n", z);
          885  +  return 1;
          886  +}
          887  +
          888  +int tdb_lsm_config_str(TestDb *pDb, const char *zStr){
          889  +  int rc = 0;
          890  +  if( tdb_lsm(pDb) ){
          891  +    int i;
          892  +    LsmDb *pLsm = (LsmDb *)pDb;
          893  +
          894  +    rc = test_lsm_config_str(pLsm, pLsm->db, 0, zStr, 0);
          895  +#ifdef LSM_MUTEX_PTHREADS
          896  +    for(i=0; rc==0 && i<pLsm->nWorker; i++){
          897  +      rc = test_lsm_config_str(0, pLsm->aWorker[i].pWorker, 1, zStr, 0);
          898  +    }
          899  +#endif
          900  +  }
          901  +  return rc;
          902  +}
          903  +
          904  +int tdb_lsm_configure(lsm_db *db, const char *zConfig){
          905  +  return test_lsm_config_str(0, db, 0, zConfig, 0);
          906  +}
          907  +
          908  +static int testLsmStartWorkers(LsmDb *, int, const char *, const char *);
          909  +
          910  +static int testLsmOpen(
          911  +  const char *zCfg,
          912  +  const char *zFilename, 
          913  +  int bClear, 
          914  +  TestDb **ppDb
          915  +){
          916  +  static const DatabaseMethods LsmMethods = {
          917  +    test_lsm_close,
          918  +    test_lsm_write,
          919  +    test_lsm_delete,
          920  +    test_lsm_delete_range,
          921  +    test_lsm_fetch,
          922  +    test_lsm_scan,
          923  +    test_lsm_begin,
          924  +    test_lsm_commit,
          925  +    test_lsm_rollback
          926  +  };
          927  +
          928  +  int rc;
          929  +  int nFilename;
          930  +  LsmDb *pDb;
          931  +
          932  +  /* If the bClear flag is set, delete any existing database. */
          933  +  assert( zFilename);
          934  +  if( bClear ) testDeleteLsmdb(zFilename);
          935  +  nFilename = strlen(zFilename);
          936  +
          937  +  pDb = (LsmDb *)testMalloc(sizeof(LsmDb) + nFilename + 1);
          938  +  memset(pDb, 0, sizeof(LsmDb));
          939  +  pDb->base.pMethods = &LsmMethods;
          940  +  pDb->zName = (char *)&pDb[1];
          941  +  memcpy(pDb->zName, zFilename, nFilename + 1);
          942  +
          943  +  /* Default the sector size used for crash simulation to 512 bytes. 
          944  +  ** Todo: There should be an OS method to obtain this value - just as
          945  +  ** there is in SQLite. For now, LSM assumes that it is smaller than
          946  +  ** the page size (default 4KB).
          947  +  */
          948  +  pDb->szSector = 256;
          949  +
          950  +  /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */
          951  +  pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT;
          952  +  pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT;
          953  +
          954  +  memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env));
          955  +  pDb->env.pVfsCtx = (void *)pDb;
          956  +  pDb->env.xFullpath = testEnvFullpath;
          957  +  pDb->env.xOpen = testEnvOpen;
          958  +  pDb->env.xRead = testEnvRead;
          959  +  pDb->env.xWrite = testEnvWrite;
          960  +  pDb->env.xTruncate = testEnvTruncate;
          961  +  pDb->env.xSync = testEnvSync;
          962  +  pDb->env.xSectorSize = testEnvSectorSize;
          963  +  pDb->env.xRemap = testEnvRemap;
          964  +  pDb->env.xFileid = testEnvFileid;
          965  +  pDb->env.xClose = testEnvClose;
          966  +  pDb->env.xUnlink = testEnvUnlink;
          967  +  pDb->env.xLock = testEnvLock;
          968  +  pDb->env.xTestLock = testEnvTestLock;
          969  +  pDb->env.xShmBarrier = testEnvShmBarrier;
          970  +  pDb->env.xShmMap = testEnvShmMap;
          971  +  pDb->env.xShmUnmap = testEnvShmUnmap;
          972  +  pDb->env.xSleep = testEnvSleep;
          973  +
          974  +  rc = lsm_new(&pDb->env, &pDb->db);
          975  +  if( rc==LSM_OK ){
          976  +    int nThread = 1;
          977  +    lsm_config_log(pDb->db, xLog, 0);
          978  +    lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb);
          979  +
          980  +    rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread);
          981  +    if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename);
          982  +
          983  +    pDb->eMode = nThread;
          984  +#ifdef LSM_MUTEX_PTHREADS
          985  +    if( rc==LSM_OK && nThread>1 ){
          986  +      testLsmStartWorkers(pDb, nThread, zFilename, zCfg);
          987  +    }
          988  +#endif
          989  +
          990  +    if( rc!=LSM_OK ){
          991  +      test_lsm_close((TestDb *)pDb);
          992  +      pDb = 0;
          993  +    }
          994  +  }
          995  +
          996  +  *ppDb = (TestDb *)pDb;
          997  +  return rc;
          998  +}
          999  +
         1000  +int test_lsm_open(
         1001  +  const char *zSpec, 
         1002  +  const char *zFilename, 
         1003  +  int bClear, 
         1004  +  TestDb **ppDb
         1005  +){
         1006  +  return testLsmOpen(zSpec, zFilename, bClear, ppDb);
         1007  +}
         1008  +
         1009  +int test_lsm_small_open(
         1010  +  const char *zSpec, 
         1011  +  const char *zFile, 
         1012  +  int bClear, 
         1013  +  TestDb **ppDb
         1014  +){
         1015  +  const char *zCfg = "page_size=256 block_size=64 mmap=1024";
         1016  +  return testLsmOpen(zCfg, zFile, bClear, ppDb);
         1017  +}
         1018  +
         1019  +int test_lsm_lomem_open(
         1020  +  const char *zSpec, 
         1021  +  const char *zFilename, 
         1022  +  int bClear, 
         1023  +  TestDb **ppDb
         1024  +){
         1025  +    /* "max_freelist=4 autocheckpoint=32" */
         1026  +  const char *zCfg = 
         1027  +    "page_size=256 block_size=64 autoflush=16 "
         1028  +    "autocheckpoint=32"
         1029  +    "mmap=0 "
         1030  +  ;
         1031  +  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
         1032  +}
         1033  +
         1034  +int test_lsm_zip_open(
         1035  +  const char *zSpec, 
         1036  +  const char *zFilename, 
         1037  +  int bClear, 
         1038  +  TestDb **ppDb
         1039  +){
         1040  +  const char *zCfg = 
         1041  +    "page_size=256 block_size=64 autoflush=16 "
         1042  +    "autocheckpoint=32 compression=1 mmap=0 "
         1043  +  ;
         1044  +  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
         1045  +}
         1046  +
         1047  +lsm_db *tdb_lsm(TestDb *pDb){
         1048  +  if( pDb->pMethods->xClose==test_lsm_close ){
         1049  +    return ((LsmDb *)pDb)->db;
         1050  +  }
         1051  +  return 0;
         1052  +}
         1053  +
         1054  +int tdb_lsm_multithread(TestDb *pDb){
         1055  +  int ret = 0;
         1056  +  if( tdb_lsm(pDb) ){
         1057  +    ret = ((LsmDb*)pDb)->eMode!=LSMTEST_MODE_SINGLETHREAD;
         1058  +  }
         1059  +  return ret;
         1060  +}
         1061  +
         1062  +void tdb_lsm_enable_log(TestDb *pDb, int bEnable){
         1063  +  lsm_db *db = tdb_lsm(pDb);
         1064  +  if( db ){
         1065  +    lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client");
         1066  +  }
         1067  +}
         1068  +
         1069  +void tdb_lsm_application_crash(TestDb *pDb){
         1070  +  if( tdb_lsm(pDb) ){
         1071  +    LsmDb *p = (LsmDb *)pDb;
         1072  +    p->bCrashed = 1;
         1073  +  }
         1074  +}
         1075  +
         1076  +void tdb_lsm_prepare_system_crash(TestDb *pDb){
         1077  +  if( tdb_lsm(pDb) ){
         1078  +    LsmDb *p = (LsmDb *)pDb;
         1079  +    p->bPrepareCrash = 1;
         1080  +  }
         1081  +}
         1082  +
         1083  +void tdb_lsm_system_crash(TestDb *pDb){
         1084  +  if( tdb_lsm(pDb) ){
         1085  +    LsmDb *p = (LsmDb *)pDb;
         1086  +    p->bCrashed = 1;
         1087  +    doSystemCrash(p);
         1088  +  }
         1089  +}
         1090  +
         1091  +void tdb_lsm_safety(TestDb *pDb, int eMode){
         1092  +  assert( eMode==LSM_SAFETY_OFF 
         1093  +       || eMode==LSM_SAFETY_NORMAL 
         1094  +       || eMode==LSM_SAFETY_FULL 
         1095  +  );
         1096  +  if( tdb_lsm(pDb) ){
         1097  +    int iParam = eMode;
         1098  +    LsmDb *p = (LsmDb *)pDb;
         1099  +    lsm_config(p->db, LSM_CONFIG_SAFETY, &iParam);
         1100  +  }
         1101  +}
         1102  +
         1103  +void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync){
         1104  +  assert( iSync>0 );
         1105  +  if( tdb_lsm(pDb) ){
         1106  +    LsmDb *p = (LsmDb *)pDb;
         1107  +    p->nAutoCrash = iSync;
         1108  +    p->bPrepareCrash = 1;
         1109  +  }
         1110  +}
         1111  +
         1112  +void tdb_lsm_config_work_hook(
         1113  +  TestDb *pDb, 
         1114  +  void (*xWork)(lsm_db *, void *), 
         1115  +  void *pWorkCtx
         1116  +){
         1117  +  if( tdb_lsm(pDb) ){
         1118  +    LsmDb *p = (LsmDb *)pDb;
         1119  +    p->xWork = xWork;
         1120  +    p->pWorkCtx = pWorkCtx;
         1121  +  }
         1122  +}
         1123  +
         1124  +void tdb_lsm_write_hook(
         1125  +  TestDb *pDb, 
         1126  +  void (*xWrite)(void *, int, lsm_i64, int, int),
         1127  +  void *pWriteCtx
         1128  +){
         1129  +  if( tdb_lsm(pDb) ){
         1130  +    LsmDb *p = (LsmDb *)pDb;
         1131  +    p->xWriteHook = xWrite;
         1132  +    p->pWriteCtx = pWriteCtx;
         1133  +  }
         1134  +}
         1135  +
         1136  +int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb){
         1137  +  return testLsmOpen(zCfg, zDb, bClear, ppDb);
         1138  +}
         1139  +
         1140  +#ifdef LSM_MUTEX_PTHREADS
         1141  +
         1142  +/*
         1143  +** Signal worker thread iWorker that there may be work to do.
         1144  +*/
         1145  +static void mt_signal_worker(LsmDb *pDb, int iWorker){
         1146  +  LsmWorker *p = &pDb->aWorker[iWorker];
         1147  +  pthread_mutex_lock(&p->worker_mutex);
         1148  +  p->bDoWork = 1;
         1149  +  pthread_cond_signal(&p->worker_cond);
         1150  +  pthread_mutex_unlock(&p->worker_mutex);
         1151  +}
         1152  +
         1153  +/*
         1154  +** This routine is used as the main() for all worker threads.
         1155  +*/
         1156  +static void *worker_main(void *pArg){
         1157  +  LsmWorker *p = (LsmWorker *)pArg;
         1158  +  lsm_db *pWorker;                /* Connection to access db through */
         1159  +
         1160  +  pthread_mutex_lock(&p->worker_mutex);
         1161  +  while( (pWorker = p->pWorker) ){
         1162  +    int rc = LSM_OK;
         1163  +
         1164  +    /* Do some work. If an error occurs, exit. */
         1165  +
         1166  +    pthread_mutex_unlock(&p->worker_mutex);
         1167  +    if( p->eType==LSMTEST_THREAD_CKPT ){
         1168  +      int nKB = 0;
         1169  +      rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB);
         1170  +      if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){
         1171  +        rc = lsm_checkpoint(pWorker, 0);
         1172  +      }
         1173  +    }else{
         1174  +      int nWrite;
         1175  +      do {
         1176  +
         1177  +        if( p->eType==LSMTEST_THREAD_WORKER ){
         1178  +          waitOnCheckpointer(p->pDb, pWorker);
         1179  +        }
         1180  +
         1181  +        nWrite = 0;
         1182  +        rc = lsm_work(pWorker, 0, 256, &nWrite);
         1183  +
         1184  +        if( p->eType==LSMTEST_THREAD_WORKER && nWrite ){
         1185  +          mt_signal_worker(p->pDb, 1);
         1186  +        }
         1187  +      }while( nWrite && p->pWorker );
         1188  +    }
         1189  +    pthread_mutex_lock(&p->worker_mutex);
         1190  +
         1191  +    if( rc!=LSM_OK && rc!=LSM_BUSY ){
         1192  +      p->worker_rc = rc;
         1193  +      break;
         1194  +    }
         1195  +
         1196  +    /* The thread will wake up when it is signaled either because another
         1197  +    ** thread has created some work for this one or because the connection
         1198  +    ** is being closed.  */
         1199  +    if( p->pWorker && p->bDoWork==0 ){
         1200  +      pthread_cond_wait(&p->worker_cond, &p->worker_mutex);
         1201  +    }
         1202  +    p->bDoWork = 0;
         1203  +  }
         1204  +  pthread_mutex_unlock(&p->worker_mutex);
         1205  +  
         1206  +  return 0;
         1207  +}
         1208  +
         1209  +
         1210  +static void mt_stop_worker(LsmDb *pDb, int iWorker){
         1211  +  LsmWorker *p = &pDb->aWorker[iWorker];
         1212  +  if( p->pWorker ){
         1213  +    void *pDummy;
         1214  +    lsm_db *pWorker;
         1215  +
         1216  +    /* Signal the worker to stop */
         1217  +    pthread_mutex_lock(&p->worker_mutex);
         1218  +    pWorker = p->pWorker;
         1219  +    p->pWorker = 0;
         1220  +    pthread_cond_signal(&p->worker_cond);
         1221  +    pthread_mutex_unlock(&p->worker_mutex);
         1222  +
         1223  +    /* Join the worker thread. */
         1224  +    pthread_join(p->worker_thread, &pDummy);
         1225  +
         1226  +    /* Free resources allocated in mt_start_worker() */
         1227  +    pthread_cond_destroy(&p->worker_cond);
         1228  +    pthread_mutex_destroy(&p->worker_mutex);
         1229  +    lsm_close(pWorker);
         1230  +  }
         1231  +}
         1232  +
         1233  +static void mt_shutdown(LsmDb *pDb){
         1234  +  int i;
         1235  +  for(i=0; i<pDb->nWorker; i++){
         1236  +    mt_stop_worker(pDb, i);
         1237  +  }
         1238  +}
         1239  +
         1240  +/*
         1241  +** This callback is invoked by LSM when the client database writes to
         1242  +** the database file (i.e. to flush the contents of the in-memory tree).
         1243  +** This implies there may be work to do on the database, so signal
         1244  +** the worker threads.
         1245  +*/
         1246  +static void mt_client_work_hook(lsm_db *db, void *pArg){
         1247  +  LsmDb *pDb = (LsmDb *)pArg;     /* LsmDb database handle */
         1248  +
         1249  +  /* Invoke the user level work-hook, if any. */
         1250  +  if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx);
         1251  +
         1252  +  /* Wake up worker thread 0. */
         1253  +  mt_signal_worker(pDb, 0);
         1254  +}
         1255  +
         1256  +static void mt_worker_work_hook(lsm_db *db, void *pArg){
         1257  +  LsmDb *pDb = (LsmDb *)pArg;     /* LsmDb database handle */
         1258  +
         1259  +  /* Invoke the user level work-hook, if any. */
         1260  +  if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx);
         1261  +}
         1262  +
         1263  +/*
         1264  +** Launch worker thread iWorker for database connection pDb.
         1265  +*/
         1266  +static int mt_start_worker(
         1267  +  LsmDb *pDb,                     /* Main database structure */
         1268  +  int iWorker,                    /* Worker number to start */
         1269  +  const char *zFilename,          /* File name of database to open */
         1270  +  const char *zCfg,               /* Connection configuration string */
         1271  +  int eType                       /* Type of worker thread */
         1272  +){
         1273  +  int rc = 0;                     /* Return code */
         1274  +  LsmWorker *p;                   /* Object to initialize */
         1275  +
         1276  +  assert( iWorker<pDb->nWorker );
         1277  +  assert( eType==LSMTEST_THREAD_CKPT 
         1278  +       || eType==LSMTEST_THREAD_WORKER 
         1279  +       || eType==LSMTEST_THREAD_WORKER_AC 
         1280  +  );
         1281  +
         1282  +  p = &pDb->aWorker[iWorker];
         1283  +  p->eType = eType;
         1284  +  p->pDb = pDb;
         1285  +
         1286  +  /* Open the worker connection */
         1287  +  if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker);
         1288  +  if( zCfg ){
         1289  +    test_lsm_config_str(pDb, p->pWorker, 1, zCfg, 0);
         1290  +  }
         1291  +  if( rc==0 ) rc = lsm_open(p->pWorker, zFilename);
         1292  +  lsm_config_log(p->pWorker, xLog, (void *)"worker");
         1293  +
         1294  +  /* Configure the work-hook */
         1295  +  if( rc==0 ){
         1296  +    lsm_config_work_hook(p->pWorker, mt_worker_work_hook, (void *)pDb);
         1297  +  }
         1298  +
         1299  +  if( eType==LSMTEST_THREAD_WORKER ){
         1300  +    test_lsm_config_str(0, p->pWorker, 1, "autocheckpoint=0", 0);
         1301  +  }
         1302  +
         1303  +  /* Kick off the worker thread. */
         1304  +  if( rc==0 ) rc = pthread_cond_init(&p->worker_cond, 0);
         1305  +  if( rc==0 ) rc = pthread_mutex_init(&p->worker_mutex, 0);
         1306  +  if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p);
         1307  +
         1308  +  return rc;
         1309  +}
         1310  +
         1311  +
         1312  +static int testLsmStartWorkers(
         1313  +  LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg
         1314  +){
         1315  +  int rc;
         1316  +
         1317  +  if( eModel<1 || eModel>4 ) return 1;
         1318  +  if( eModel==1 ) return 0;
         1319  +
         1320  +  /* Configure a work-hook for the client connection. Worker 0 is signalled
         1321  +  ** every time the users connection writes to the database.  */
         1322  +  lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb);
         1323  +
         1324  +  /* Allocate space for two worker connections. They may not both be
         1325  +  ** used, but both are allocated.  */
         1326  +  pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2);
         1327  +  memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2);
         1328  +
         1329  +  switch( eModel ){
         1330  +    case LSMTEST_MODE_BACKGROUND_CKPT:
         1331  +      pDb->nWorker = 1;
         1332  +      test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0);
         1333  +      rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT);
         1334  +      break;
         1335  +
         1336  +    case LSMTEST_MODE_BACKGROUND_WORK:
         1337  +      pDb->nWorker = 1;
         1338  +      test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0);
         1339  +      rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER_AC);
         1340  +      break;
         1341  +
         1342  +    case LSMTEST_MODE_BACKGROUND_BOTH:
         1343  +      pDb->nWorker = 2;
         1344  +      test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0);
         1345  +      rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER);
         1346  +      if( rc==0 ){
         1347  +        rc = mt_start_worker(pDb, 1, zFilename, zCfg, LSMTEST_THREAD_CKPT);
         1348  +      }
         1349  +      break;
         1350  +  }
         1351  +
         1352  +  return rc;
         1353  +}
         1354  +
         1355  +
         1356  +int test_lsm_mt2(
         1357  +  const char *zSpec, 
         1358  +  const char *zFilename, 
         1359  +  int bClear, 
         1360  +  TestDb **ppDb
         1361  +){
         1362  +  const char *zCfg = "mt_mode=2";
         1363  +  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
         1364  +}
         1365  +
         1366  +int test_lsm_mt3(
         1367  +  const char *zSpec, 
         1368  +  const char *zFilename, 
         1369  +  int bClear, 
         1370  +  TestDb **ppDb
         1371  +){
         1372  +  const char *zCfg = "mt_mode=4";
         1373  +  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
         1374  +}
         1375  +
         1376  +#else
         1377  +static void mt_shutdown(LsmDb *pDb) { 
         1378  +  unused_parameter(pDb); 
         1379  +}
         1380  +int test_lsm_mt(const char *zFilename, int bClear, TestDb **ppDb){
         1381  +  unused_parameter(zFilename);
         1382  +  unused_parameter(bClear);
         1383  +  unused_parameter(ppDb);
         1384  +  testPrintError("threads unavailable - recompile with LSM_MUTEX_PTHREADS\n");
         1385  +  return 1;
         1386  +}
         1387  +#endif

Added ext/lsm1/lsm-test/lsmtest_tdb4.c.

            1  +
            2  +/*
            3  +** This file contains the TestDb bt wrapper.
            4  +*/
            5  +
            6  +#include "lsmtest_tdb.h"
            7  +#include "lsmtest.h"
            8  +#include <unistd.h>
            9  +#include "bt.h"
           10  +
           11  +#include <pthread.h>
           12  +
           13  +typedef struct BtDb BtDb;
           14  +typedef struct BtFile BtFile;
           15  +
           16  +/* Background checkpointer interface (see implementations below). */
           17  +typedef struct bt_ckpter bt_ckpter;
           18  +static int bgc_attach(BtDb *pDb, const char*);
           19  +static int bgc_detach(BtDb *pDb);
           20  +
           21  +/*
           22  +** Each database or log file opened by a database handle is wrapped by
           23  +** an object of the following type.
           24  +*/
           25  +struct BtFile {
           26  +  BtDb *pBt;                      /* Database handle that opened this file */
           27  +  bt_env *pVfs;                   /* Underlying VFS */
           28  +  bt_file *pFile;                 /* File handle belonging to underlying VFS */
           29  +  int nSectorSize;                /* Size of sectors in bytes */
           30  +  int nSect