/ Check-in [213c61cb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge all the latest trunk enhancements.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent
Files: files | file ages | folders
SHA3-256: 213c61cb82d4ee51cc638fc5bb2b54cd51b963c40d088b2f01caf77032ea32fb
User & Date: drh 2017-07-20 17:47:12
Wiki:begin-concurrent
Context
2017-07-20
18:28
Get the build working with SQLITE_OMIT_CONCURRENT. check-in: a29401e9 user: drh tags: begin-concurrent
17:47
Merge all the latest trunk enhancements. check-in: 213c61cb user: drh tags: begin-concurrent
15:08
Enhance the built-in date/time functions so that they can be used in CHECK constraints, in the WHERE clause or partial indexes, and index expressions, provided that none of the non-deterministic keywords ("now", "localtime", "utc") are used as arguments. check-in: a90c062d user: drh tags: trunk
2017-06-10
17:23
Remove sqlite3_log() and abort() calls added to this branch to debug the pointer-map problem ([fda22108]). check-in: 79544fc2 user: dan tags: begin-concurrent
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   177    177            icu.lo insert.lo json1.lo legacy.lo loadext.lo \
   178    178            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   179    179            memjournal.lo \
   180    180            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   181    181            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   182    182            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   183    183            random.lo resolve.lo rowset.lo rtree.lo \
   184         -         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
          184  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
   185    185            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   186    186            update.lo util.lo vacuum.lo \
   187    187            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   188    188            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   189    189            utf.lo vtab.lo
   190    190   
   191    191   # Object files for the amalgamation.
................................................................................
   346    346   SRC += \
   347    347     $(TOP)/ext/session/sqlite3session.c \
   348    348     $(TOP)/ext/session/sqlite3session.h
   349    349   SRC += \
   350    350     $(TOP)/ext/rbu/sqlite3rbu.h \
   351    351     $(TOP)/ext/rbu/sqlite3rbu.c
   352    352   SRC += \
   353         -  $(TOP)/ext/misc/json1.c
          353  +  $(TOP)/ext/misc/json1.c \
          354  +  $(TOP)/ext/misc/stmt.c
   354    355   
   355    356   # Generated source code files
   356    357   #
   357    358   SRC += \
   358    359     keywordhash.h \
   359    360     opcodes.c \
   360    361     opcodes.h \
................................................................................
   430    431     $(TOP)/ext/misc/nextchar.c \
   431    432     $(TOP)/ext/misc/percentile.c \
   432    433     $(TOP)/ext/misc/regexp.c \
   433    434     $(TOP)/ext/misc/remember.c \
   434    435     $(TOP)/ext/misc/series.c \
   435    436     $(TOP)/ext/misc/spellfix.c \
   436    437     $(TOP)/ext/misc/totype.c \
          438  +  $(TOP)/ext/misc/unionvtab.c \
   437    439     $(TOP)/ext/misc/wholenumber.c
   438    440   
   439    441   # Source code to the library files needed by the test fixture
   440    442   #
   441    443   TESTSRC2 = \
   442    444     $(TOP)/src/attach.c \
   443    445     $(TOP)/src/backup.c \
................................................................................
   479    481     $(TOP)/ext/fts3/fts3.c \
   480    482     $(TOP)/ext/fts3/fts3_aux.c \
   481    483     $(TOP)/ext/fts3/fts3_expr.c \
   482    484     $(TOP)/ext/fts3/fts3_term.c \
   483    485     $(TOP)/ext/fts3/fts3_tokenizer.c \
   484    486     $(TOP)/ext/fts3/fts3_write.c \
   485    487     $(TOP)/ext/async/sqlite3async.c \
   486         -  $(TOP)/ext/session/sqlite3session.c 
          488  +  $(TOP)/ext/session/sqlite3session.c \
          489  +  $(TOP)/ext/misc/stmt.c 
   487    490   
   488    491   # Header files used by all library source files.
   489    492   #
   490    493   HDR = \
   491    494      $(TOP)/src/btree.h \
   492    495      $(TOP)/src/btreeInt.h \
   493    496      $(TOP)/src/hash.h \
................................................................................
   560    563   
   561    564   # Extra compiler options for various shell tools
   562    565   #
   563    566   SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
   564    567   # SHELL_OPT += -DSQLITE_ENABLE_FTS5
   565    568   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   566    569   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
          570  +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   567    571   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   568    572   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
   569    573   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
   570    574   FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
   571    575   DBFUZZ_OPT = 
   572    576   
   573    577   # This is the default Makefile target.  The objects listed here
................................................................................
  1030   1034   
  1031   1035   sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
  1032   1036   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
  1033   1037   
  1034   1038   json1.lo:	$(TOP)/ext/misc/json1.c
  1035   1039   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
  1036   1040   
         1041  +stmt.lo:	$(TOP)/ext/misc/stmt.c
         1042  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
         1043  +
  1037   1044   # FTS5 things
  1038   1045   #
  1039   1046   FTS5_SRC = \
  1040   1047      $(TOP)/ext/fts5/fts5.h \
  1041   1048      $(TOP)/ext/fts5/fts5Int.h \
  1042   1049      $(TOP)/ext/fts5/fts5_aux.c \
  1043   1050      $(TOP)/ext/fts5/fts5_buffer.c \
................................................................................
  1079   1086   # hidden when the library is built via the amalgamation).
  1080   1087   #
  1081   1088   TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1082   1089   TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
  1083   1090   TESTFIXTURE_FLAGS += -DBUILD_sqlite
  1084   1091   TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  1085   1092   TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
         1093  +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
  1086   1094   
  1087   1095   TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
  1088   1096   TESTFIXTURE_SRC1 = sqlite3.c
  1089   1097   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
  1090   1098   TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
  1091   1099   
  1092   1100   testfixture$(TEXE):	$(TESTFIXTURE_SRC)

Changes to Makefile.msc.

  1290   1290     $(TOP)\ext\fts3\fts3_unicode.c \
  1291   1291     $(TOP)\ext\fts3\fts3_unicode2.c \
  1292   1292     $(TOP)\ext\fts3\fts3_write.c \
  1293   1293     $(TOP)\ext\icu\icu.c \
  1294   1294     $(TOP)\ext\rtree\rtree.c \
  1295   1295     $(TOP)\ext\session\sqlite3session.c \
  1296   1296     $(TOP)\ext\rbu\sqlite3rbu.c \
  1297         -  $(TOP)\ext\misc\json1.c
         1297  +  $(TOP)\ext\misc\json1.c \
         1298  +  $(TOP)\ext\misc\stmt.c
  1298   1299   
  1299   1300   # Extension header files, part 1.
  1300   1301   #
  1301   1302   SRC08 = \
  1302   1303     $(TOP)\ext\fts1\fts1.h \
  1303   1304     $(TOP)\ext\fts1\fts1_hash.h \
  1304   1305     $(TOP)\ext\fts1\fts1_tokenizer.h \
................................................................................
  1413   1414     $(TOP)\ext\misc\nextchar.c \
  1414   1415     $(TOP)\ext\misc\percentile.c \
  1415   1416     $(TOP)\ext\misc\regexp.c \
  1416   1417     $(TOP)\ext\misc\remember.c \
  1417   1418     $(TOP)\ext\misc\series.c \
  1418   1419     $(TOP)\ext\misc\spellfix.c \
  1419   1420     $(TOP)\ext\misc\totype.c \
         1421  +  $(TOP)\ext\misc\unionvtab.c \
  1420   1422     $(TOP)\ext\misc\wholenumber.c
  1421   1423   
  1422   1424   # Source code to the library files needed by the test fixture
  1423   1425   # (non-amalgamation)
  1424   1426   #
  1425   1427   TESTSRC2 = \
  1426   1428     $(SRC00) \
................................................................................
  1501   1503     $(TOP)\test\fuzzdata5.db
  1502   1504   # <</mark>>
  1503   1505   
  1504   1506   # Additional compiler options for the shell.  These are only effective
  1505   1507   # when the shell is not being dynamically linked.
  1506   1508   #
  1507   1509   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1508         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
         1510  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
  1509   1511   !ENDIF
  1510   1512   
  1511   1513   # <<mark>>
  1512   1514   # Extra compiler options for various test tools.
  1513   1515   #
  1514   1516   MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
  1515   1517   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
................................................................................
  2085   2087   # hidden when the library is built via the amalgamation).
  2086   2088   #
  2087   2089   TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  2088   2090   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  2089   2091   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
  2090   2092   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  2091   2093   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
         2094  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
  2092   2095   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
  2093   2096   
  2094   2097   TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  2095   2098   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  2096   2099   !IF $(USE_AMALGAMATION)==0
  2097   2100   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  2098   2101   !ELSE
................................................................................
  2238   2241   dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
  2239   2242   	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
  2240   2243   
  2241   2244   rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
  2242   2245   	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
  2243   2246   		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2244   2247   
         2248  +LSMDIR=$(TOP)\ext\lsm1
         2249  +!INCLUDE $(LSMDIR)\Makefile.msc
         2250  +
  2245   2251   moreclean:	clean
  2246   2252   	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
  2247   2253   # <</mark>>
  2248   2254   
  2249   2255   clean:
  2250   2256   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  2251   2257   	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
................................................................................
  2257   2263   	del /Q mkkeywordhash.* keywordhash.h 2>NUL
  2258   2264   	del /Q notasharedlib.* 2>NUL
  2259   2265   	-rmdir /Q/S .deps 2>NUL
  2260   2266   	-rmdir /Q/S .libs 2>NUL
  2261   2267   	-rmdir /Q/S tsrc 2>NUL
  2262   2268   	del /Q .target_source 2>NUL
  2263   2269   	del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL
         2270  +	del /Q lsm.dll lsmtest.exe 2>NUL
  2264   2271   	del /Q testloadext.dll 2>NUL
  2265   2272   	del /Q testfixture.exe test.db 2>NUL
  2266   2273   	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL
  2267   2274   	del /Q changeset.exe 2>NUL
  2268   2275   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  2269   2276   	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2270   2277   	del /Q sqlite3.c sqlite3-*.c 2>NUL

Changes to README.md.

    30     30        to locate the check-in desired, click on its information page link,
    31     31        then click on the "Tarball" or "ZIP Archive" links on the information
    32     32        page.
    33     33   
    34     34   If you do want to use Fossil to check out the source tree, 
    35     35   first install Fossil version 2.0 or later.
    36     36   (Source tarballs and precompiled binaries available
    37         -[here](https://www.fossil-scm.org/fossil/uv/download.html).)
           37  +[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
           38  +a stand-alone program.  To install, simply download or build the single 
           39  +executable file and put that file someplace on your $PATH.)
    38     40   Then run commands like this:
    39     41   
    40     42           mkdir ~/sqlite
    41     43           cd ~/sqlite
    42     44           fossil clone https://www.sqlite.org/src sqlite.fossil
    43     45           fossil open sqlite.fossil
    44     46       
................................................................................
   102    104   SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
   103    105   is required by the makefiles (including those for MSVC).  SQLite contains
   104    106   a lot of generated code and Tcl is used to do much of that code generation.
   105    107   The makefiles also require AWK.
   106    108   
   107    109   ## Source Code Tour
   108    110   
   109         -Most of the core source files are in the **src/** subdirectory.  But
   110         -src/ also contains files used to build the "testfixture" test harness;
   111         -those file all begin with "test".  And src/ contains the "shell.c" file
   112         -which is the main program for the "sqlite3.exe" command-line shell and
   113         -the "tclsqlite.c" file which implements the bindings to SQLite from the
   114         -Tcl programming language.  (Historical note:  SQLite began as a Tcl
          111  +Most of the core source files are in the **src/** subdirectory.  The
          112  +**src/** folder also contains files used to build the "testfixture" test
          113  +harness. The names of the source files used by "testfixture" all begin
          114  +with "test".
          115  +The **src/** also contains the "shell.c" file
          116  +which is the main program for the "sqlite3.exe"
          117  +[command-line shell](https://sqlite.org/cli.html) and
          118  +the "tclsqlite.c" file which implements the
          119  +[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
          120  +(Historical note:  SQLite began as a Tcl
   115    121   extension and only later escaped to the wild as an independent library.)
   116    122   
   117    123   Test scripts and programs are found in the **test/** subdirectory.
   118         -There are other test suites for SQLite (see
   119         -[How SQLite Is Tested](http://www.sqlite.org/testing.html))
   120         -but those other test suites are
   121         -in separate source repositories.
          124  +Addtional test code is found in other source repositories.
          125  +See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
          126  +additional information.
   122    127   
   123    128   The **ext/** subdirectory contains code for extensions.  The
   124    129   Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
   125    130   **ext/rtree**.  The **ext/misc** subdirectory contains a number of
   126    131   smaller, single-file extensions, such as a REGEXP operator.
   127    132   
   128    133   The **tool/** subdirectory contains various scripts and programs used
................................................................................
   138    143   The "target&#95;source" make target will create a subdirectory "tsrc/" and
   139    144   fill it with all the source files needed to build SQLite, both
   140    145   manually-edited files and automatically-generated files.
   141    146   
   142    147   The SQLite interface is defined by the **sqlite3.h** header file, which is
   143    148   generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
   144    149   [Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
   145         -The manifest.uuid file contains the SHA1 hash of the particular check-in
          150  +The manifest.uuid file contains the SHA3 hash of the particular check-in
   146    151   and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
   147    152   contains the current SQLite version number.  The sqlite3.h header is really
   148    153   just a copy of src/sqlite.h.in with the source-id and version number inserted
   149    154   at just the right spots. Note that comment text in the sqlite3.h file is
   150    155   used to generate much of the SQLite API documentation.  The Tcl scripts
   151    156   used to generate that documentation are in a separate source repository.
   152    157   
   153    158   The SQL language parser is **parse.c** which is generate from a grammar in
   154    159   the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
   155    160   by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
   156         -for lemon is at tool/lemon.c.  Lemon uses a
   157         -template for generating its parser.  A generic template is in tool/lempar.c,
   158         -but SQLite uses a slightly modified template found in src/lempar.c.
          161  +for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
          162  +template for generating its parser.
   159    163   
   160    164   Lemon also generates the **parse.h** header file, at the same time it
   161    165   generates parse.c. But the parse.h header file is
   162    166   modified further (to add additional symbols) using the ./addopcodes.awk
   163    167   AWK script.
   164    168   
   165    169   The **opcodes.h** header file contains macros that define the numbers
................................................................................
   171    175   opcode-number to opcode-name that is used for EXPLAIN output.
   172    176   
   173    177   The **keywordhash.h** header file contains the definition of a hash table
   174    178   that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
   175    179   the numeric codes used by the parse.c parser.  The keywordhash.h file is
   176    180   generated by a C-language program at tool mkkeywordhash.c.
   177    181   
          182  +The **pragma.h** header file contains various definitions used to parse
          183  +and implement the PRAGMA statements.  The header is generated by a
          184  +script **tool/mkpragmatab.tcl**. If you want to add a new PRAGMA, edit
          185  +the **tool/mkpragmatab.tcl** file to insert the information needed by the
          186  +parser for your new PRAGMA, then run the script to regenerate the
          187  +**pragma.h** header file.
          188  +
   178    189   ### The Amalgamation
   179    190   
   180    191   All of the individual C source code and header files (both manually-edited
   181    192   and automatically-generated) can be combined into a single big source file
   182    193   **sqlite3.c** called "the amalgamation".  The amalgamation is the recommended
   183    194   way of using SQLite in a larger application.  Combining all individual
   184    195   source code files into a single big source code file allows the C compiler
................................................................................
   188    199   
   189    200   The amalgamation is generated from the tool/mksqlite3c.tcl Tcl script.
   190    201   First, all of the individual source files must be gathered into the tsrc/
   191    202   subdirectory (using the equivalent of "make target_source") then the
   192    203   tool/mksqlite3c.tcl script is run to copy them all together in just the
   193    204   right order while resolving internal "#include" references.
   194    205   
   195         -The amalgamation source file is more than 100K lines long.  Some symbolic
          206  +The amalgamation source file is more than 200K lines long.  Some symbolic
   196    207   debuggers (most notably MSVC) are unable to deal with files longer than 64K
   197    208   lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
   198    209   can be run on the amalgamation to break it up into a single small C file
   199    210   called **sqlite3-all.c** that does #include on about five other files
   200    211   named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
   201    212   all of the source code is contained within a single translation unit so
   202    213   that the compiler can do extra cross-procedure optimization, but no
................................................................................
   205    216   ## How It All Fits Together
   206    217   
   207    218   SQLite is modular in design.
   208    219   See the [architectural description](http://www.sqlite.org/arch.html)
   209    220   for details. Other documents that are useful in
   210    221   (helping to understand how SQLite works include the
   211    222   [file format](http://www.sqlite.org/fileformat2.html) description,
   212         -the [virtual machine](http://www.sqlite.org/vdbe.html) that runs
          223  +the [virtual machine](http://www.sqlite.org/opcode.html) that runs
   213    224   prepared statements, the description of
   214    225   [how transactions work](http://www.sqlite.org/atomiccommit.html), and
   215    226   the [overview of the query planner](http://www.sqlite.org/optoverview.html).
   216    227   
   217         -Unfortunately, years of effort have gone into optimizating SQLite, both
          228  +Years of effort have gone into optimizating SQLite, both
   218    229   for small size and high performance.  And optimizations tend to result in
   219         -complex code.  So there is a lot of complexity in the SQLite implementation.
          230  +complex code.  So there is a lot of complexity in the current SQLite
          231  +implementation.  It will not be the easiest library in the world to hack.
   220    232   
   221    233   Key files:
   222    234   
   223    235     *  **sqlite.h.in** - This file defines the public interface to the SQLite
   224    236        library.  Readers will need to be familiar with this interface before
   225    237        trying to understand how the library works internally.
   226    238   
   227    239     *  **sqliteInt.h** - this header file defines many of the data objects
   228    240        used internally by SQLite.
   229    241   
   230         -  *  **parse.y** - This file describes the LALR(1) grammer that SQLite uses
          242  +  *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
   231    243        to parse SQL statements, and the actions that are taken at each step
   232    244        in the parsing process.
   233    245   
   234    246     *  **vdbe.c** - This file implements the virtual machine that runs
   235    247        prepared statements.  There are various helper files whose names
   236    248        begin with "vdbe".  The VDBE has access to the vdbeInt.h header file
   237    249        which defines internal data objects.  The rest of SQLite interacts
................................................................................
   256    268        is the file that, when linked against sqlite3.a, generates the
   257    269        "sqlite3.exe" command-line shell.
   258    270   
   259    271     *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
   260    272        is not part of the core SQLite library.  But as most of the tests in this
   261    273        repository are written in Tcl, the Tcl language bindings are important.
   262    274   
   263         -There are many other source files.  Each has a suscinct header comment that
          275  +There are many other source files.  Each has a succinct header comment that
   264    276   describes its purpose and role within the larger system.
   265    277   
   266    278   
   267    279   ## Contacts
   268    280   
   269    281   The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
   270         -with geographically distributed backup servers at
          282  +with geographically distributed backups at
   271    283   [http://www2.sqlite.org/](http://www2.sqlite.org) and
   272    284   [http://www3.sqlite.org/](http://www3.sqlite.org).

Changes to VERSION.

     1         -3.19.3
            1  +3.20.0

Changes to autoconf/Makefile.am.

     8      8   bin_PROGRAMS = sqlite3
     9      9   sqlite3_SOURCES = shell.c sqlite3.h
    10     10   EXTRA_sqlite3_SOURCES = sqlite3.c
    11     11   sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
    12     12   sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
    13     13   sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
    14     14   
    15         -include_HEADERS = sqlite3.h sqlite3ext.h msvc.h
           15  +include_HEADERS = sqlite3.h sqlite3ext.h
    16     16   
    17     17   EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
    18     18   pkgconfigdir = ${libdir}/pkgconfig
    19     19   pkgconfig_DATA = sqlite3.pc
    20     20   
    21     21   man_MANS = sqlite3.1

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.19.3.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.20.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.19.3'
   730         -PACKAGE_STRING='sqlite 3.19.3'
          729  +PACKAGE_VERSION='3.20.0'
          730  +PACKAGE_STRING='sqlite 3.20.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1459   1459   #
  1460   1460   # Report the --help message.
  1461   1461   #
  1462   1462   if test "$ac_init_help" = "long"; then
  1463   1463     # Omit some internal or obsolete options to make the list less imposing.
  1464   1464     # This message is too long to be a string in the A/UX 3.1 sh.
  1465   1465     cat <<_ACEOF
  1466         -\`configure' configures sqlite 3.19.3 to adapt to many kinds of systems.
         1466  +\`configure' configures sqlite 3.20.0 to adapt to many kinds of systems.
  1467   1467   
  1468   1468   Usage: $0 [OPTION]... [VAR=VALUE]...
  1469   1469   
  1470   1470   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1471   1471   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1472   1472   
  1473   1473   Defaults for the options are specified in brackets.
................................................................................
  1524   1524     --build=BUILD     configure for building on BUILD [guessed]
  1525   1525     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1526   1526   _ACEOF
  1527   1527   fi
  1528   1528   
  1529   1529   if test -n "$ac_init_help"; then
  1530   1530     case $ac_init_help in
  1531         -     short | recursive ) echo "Configuration of sqlite 3.19.3:";;
         1531  +     short | recursive ) echo "Configuration of sqlite 3.20.0:";;
  1532   1532      esac
  1533   1533     cat <<\_ACEOF
  1534   1534   
  1535   1535   Optional Features:
  1536   1536     --disable-option-checking  ignore unrecognized --enable/--with options
  1537   1537     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1538   1538     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1648   1648       cd "$ac_pwd" || { ac_status=$?; break; }
  1649   1649     done
  1650   1650   fi
  1651   1651   
  1652   1652   test -n "$ac_init_help" && exit $ac_status
  1653   1653   if $ac_init_version; then
  1654   1654     cat <<\_ACEOF
  1655         -sqlite configure 3.19.3
         1655  +sqlite configure 3.20.0
  1656   1656   generated by GNU Autoconf 2.69
  1657   1657   
  1658   1658   Copyright (C) 2012 Free Software Foundation, Inc.
  1659   1659   This configure script is free software; the Free Software Foundation
  1660   1660   gives unlimited permission to copy, distribute and modify it.
  1661   1661   _ACEOF
  1662   1662     exit
................................................................................
  2067   2067     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2068   2068   
  2069   2069   } # ac_fn_c_check_header_mongrel
  2070   2070   cat >config.log <<_ACEOF
  2071   2071   This file contains any messages produced by compilers while
  2072   2072   running configure, to aid debugging if configure makes a mistake.
  2073   2073   
  2074         -It was created by sqlite $as_me 3.19.3, which was
         2074  +It was created by sqlite $as_me 3.20.0, which was
  2075   2075   generated by GNU Autoconf 2.69.  Invocation command line was
  2076   2076   
  2077   2077     $ $0 $@
  2078   2078   
  2079   2079   _ACEOF
  2080   2080   exec 5>>config.log
  2081   2081   {
................................................................................
 12147  12147   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12148  12148   
 12149  12149   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12150  12150   # Save the log message, to keep $0 and so on meaningful, and to
 12151  12151   # report actual input values of CONFIG_FILES etc. instead of their
 12152  12152   # values after options handling.
 12153  12153   ac_log="
 12154         -This file was extended by sqlite $as_me 3.19.3, which was
        12154  +This file was extended by sqlite $as_me 3.20.0, which was
 12155  12155   generated by GNU Autoconf 2.69.  Invocation command line was
 12156  12156   
 12157  12157     CONFIG_FILES    = $CONFIG_FILES
 12158  12158     CONFIG_HEADERS  = $CONFIG_HEADERS
 12159  12159     CONFIG_LINKS    = $CONFIG_LINKS
 12160  12160     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12161  12161     $ $0 $@
................................................................................
 12213  12213   
 12214  12214   Report bugs to the package provider."
 12215  12215   
 12216  12216   _ACEOF
 12217  12217   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12218  12218   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12219  12219   ac_cs_version="\\
 12220         -sqlite config.status 3.19.3
        12220  +sqlite config.status 3.20.0
 12221  12221   configured by $0, generated by GNU Autoconf 2.69,
 12222  12222     with options \\"\$ac_cs_config\\"
 12223  12223   
 12224  12224   Copyright (C) 2012 Free Software Foundation, Inc.
 12225  12225   This config.status script is free software; the Free Software Foundation
 12226  12226   gives unlimited permission to copy, distribute and modify it."
 12227  12227   

Changes to doc/lemon.html.

    18     18   Lemon also implements features that can be used
    19     19   to eliminate resource leaks, making is suitable for use
    20     20   in long-running programs such as graphical user interfaces
    21     21   or embedded controllers.</p>
    22     22   
    23     23   <p>This document is an introduction to the Lemon
    24     24   parser generator.</p>
           25  +
           26  +<h2>Security Note</h2>
           27  +
           28  +<p>The language parser code created by Lemon is very robust and
           29  +is well-suited for use in internet-facing applications that need to
           30  +safely process maliciously crafted inputs.
           31  +
           32  +<p>The "lemon.exe" command-line tool itself works great when given a valid
           33  +input grammar file and almost always gives helpful
           34  +error messages for malformed inputs.  However,  it is possible for
           35  +a malicious user to craft a grammar file that will cause 
           36  +lemon.exe to crash.
           37  +We do not see this as a problem, as lemon.exe is not intended to be used
           38  +with hostile inputs.
           39  +To summarize:</p>
           40  +
           41  +<ul>
           42  +<li>Parser code generated by lemon &rarr; Robust and secure
           43  +<li>The "lemon.exe" command line tool itself &rarr; Not so much
           44  +</ul>
    25     45   
    26     46   <h2>Theory of Operation</h2>
    27     47   
    28     48   <p>The main goal of Lemon is to translate a context free grammar (CFG)
    29     49   for a particular language into C code that implements a parser for
    30     50   that language.
    31     51   The program has two inputs:

Changes to ext/fts3/fts3.c.

  1702   1702         sqlite3_reset(pCsr->pStmt);
  1703   1703         pCsr->pStmt = 0;
  1704   1704       }
  1705   1705       pCsr->bSeekStmt = 0;
  1706   1706     }
  1707   1707     sqlite3_finalize(pCsr->pStmt);
  1708   1708   }
         1709  +
         1710  +/*
         1711  +** Free all resources currently held by the cursor passed as the only
         1712  +** argument.
         1713  +*/
         1714  +static void fts3ClearCursor(Fts3Cursor *pCsr){
         1715  +  fts3CursorFinalizeStmt(pCsr);
         1716  +  sqlite3Fts3FreeDeferredTokens(pCsr);
         1717  +  sqlite3_free(pCsr->aDoclist);
         1718  +  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
         1719  +  sqlite3Fts3ExprFree(pCsr->pExpr);
         1720  +  memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
         1721  +}
  1709   1722   
  1710   1723   /*
  1711   1724   ** Close the cursor.  For additional information see the documentation
  1712   1725   ** on the xClose method of the virtual table interface.
  1713   1726   */
  1714   1727   static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
  1715   1728     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  1716   1729     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1717         -  fts3CursorFinalizeStmt(pCsr);
  1718         -  sqlite3Fts3ExprFree(pCsr->pExpr);
  1719         -  sqlite3Fts3FreeDeferredTokens(pCsr);
  1720         -  sqlite3_free(pCsr->aDoclist);
  1721         -  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
         1730  +  fts3ClearCursor(pCsr);
  1722   1731     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1723   1732     sqlite3_free(pCsr);
  1724   1733     return SQLITE_OK;
  1725   1734   }
  1726   1735   
  1727   1736   /*
  1728   1737   ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
................................................................................
  1740   1749       char *zSql;
  1741   1750       if( p->pSeekStmt ){
  1742   1751         pCsr->pStmt = p->pSeekStmt;
  1743   1752         p->pSeekStmt = 0;
  1744   1753       }else{
  1745   1754         zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
  1746   1755         if( !zSql ) return SQLITE_NOMEM;
  1747         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         1756  +      rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
  1748   1757         sqlite3_free(zSql);
  1749   1758       }
  1750   1759       if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
  1751   1760     }
  1752   1761     return rc;
  1753   1762   }
  1754   1763   
................................................................................
  3215   3224     if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++];
  3216   3225     if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++];
  3217   3226     if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
  3218   3227     if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
  3219   3228     assert( iIdx==nVal );
  3220   3229   
  3221   3230     /* In case the cursor has been used before, clear it now. */
  3222         -  fts3CursorFinalizeStmt(pCsr);
  3223         -  sqlite3_free(pCsr->aDoclist);
  3224         -  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
  3225         -  sqlite3Fts3ExprFree(pCsr->pExpr);
  3226         -  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
         3231  +  fts3ClearCursor(pCsr);
  3227   3232   
  3228   3233     /* Set the lower and upper bounds on docids to return */
  3229   3234     pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
  3230   3235     pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64);
  3231   3236   
  3232   3237     if( idxStr ){
  3233   3238       pCsr->bDesc = (idxStr[0]=='D');
................................................................................
  3277   3282         );
  3278   3283       }else{
  3279   3284         zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", 
  3280   3285             p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC")
  3281   3286         );
  3282   3287       }
  3283   3288       if( zSql ){
  3284         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         3289  +      rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
  3285   3290         sqlite3_free(zSql);
  3286   3291       }else{
  3287   3292         rc = SQLITE_NOMEM;
  3288   3293       }
  3289   3294     }else if( eSearch==FTS3_DOCID_SEARCH ){
  3290   3295       rc = fts3CursorSeekStmt(pCsr);
  3291   3296       if( rc==SQLITE_OK ){
................................................................................
  3298   3303   }
  3299   3304   
  3300   3305   /* 
  3301   3306   ** This is the xEof method of the virtual table. SQLite calls this 
  3302   3307   ** routine to find out if it has reached the end of a result set.
  3303   3308   */
  3304   3309   static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
  3305         -  return ((Fts3Cursor *)pCursor)->isEof;
         3310  +  Fts3Cursor *pCsr = (Fts3Cursor*)pCursor;
         3311  +  if( pCsr->isEof ){
         3312  +    fts3ClearCursor(pCsr);
         3313  +    pCsr->isEof = 1;
         3314  +  }
         3315  +  return pCsr->isEof;
  3306   3316   }
  3307   3317   
  3308   3318   /* 
  3309   3319   ** This is the xRowid method. The SQLite core calls this routine to
  3310   3320   ** retrieve the rowid for the current row of the result set. fts3
  3311   3321   ** exposes %_content.docid as the rowid for the virtual table. The
  3312   3322   ** rowid should be written to *pRowid.
................................................................................
  3339   3349   
  3340   3350     /* The column value supplied by SQLite must be in range. */
  3341   3351     assert( iCol>=0 && iCol<=p->nColumn+2 );
  3342   3352   
  3343   3353     switch( iCol-p->nColumn ){
  3344   3354       case 0:
  3345   3355         /* The special 'table-name' column */
  3346         -      sqlite3_result_blob(pCtx, &pCsr, sizeof(Fts3Cursor*), SQLITE_TRANSIENT);
  3347         -      sqlite3_result_subtype(pCtx, SQLITE_BLOB);
         3356  +      sqlite3_result_pointer(pCtx, pCsr, "fts3cursor");
  3348   3357         break;
  3349   3358   
  3350   3359       case 1:
  3351   3360         /* The docid column */
  3352   3361         sqlite3_result_int64(pCtx, pCsr->iPrevId);
  3353   3362         break;
  3354   3363   
................................................................................
  3558   3567   */
  3559   3568   static int fts3FunctionArg(
  3560   3569     sqlite3_context *pContext,      /* SQL function call context */
  3561   3570     const char *zFunc,              /* Function name */
  3562   3571     sqlite3_value *pVal,            /* argv[0] passed to function */
  3563   3572     Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
  3564   3573   ){
  3565         -  int rc = SQLITE_OK;
  3566         -  if( sqlite3_value_subtype(pVal)==SQLITE_BLOB ){
  3567         -    *ppCsr = *(Fts3Cursor**)sqlite3_value_blob(pVal);
         3574  +  int rc;
         3575  +  *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor");
         3576  +  if( (*ppCsr)!=0 ){
         3577  +    rc = SQLITE_OK;
  3568   3578     }else{
  3569   3579       char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
  3570   3580       sqlite3_result_error(pContext, zErr, -1);
  3571   3581       sqlite3_free(zErr);
  3572   3582       rc = SQLITE_ERROR;
  3573   3583     }
  3574   3584     return rc;

Changes to ext/fts3/fts3_write.c.

   403    403         zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
   404    404       }else{
   405    405         zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
   406    406       }
   407    407       if( !zSql ){
   408    408         rc = SQLITE_NOMEM;
   409    409       }else{
   410         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
          410  +      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
          411  +                              &pStmt, NULL);
   411    412         sqlite3_free(zSql);
   412    413         assert( rc==SQLITE_OK || pStmt==0 );
   413    414         p->aStmt[eStmt] = pStmt;
   414    415       }
   415    416     }
   416    417     if( apVal ){
   417    418       int i;

Changes to ext/fts5/fts5_hash.c.

   167    167   
   168    168     apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*));
   169    169     if( !apNew ) return SQLITE_NOMEM;
   170    170     memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));
   171    171   
   172    172     for(i=0; i<pHash->nSlot; i++){
   173    173       while( apOld[i] ){
   174         -      int iHash;
          174  +      unsigned int iHash;
   175    175         Fts5HashEntry *p = apOld[i];
   176    176         apOld[i] = p->pHashNext;
   177         -      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), strlen(fts5EntryKey(p)));
          177  +      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
          178  +                          (int)strlen(fts5EntryKey(p)));
   178    179         p->pHashNext = apNew[iHash];
   179    180         apNew[iHash] = p;
   180    181       }
   181    182     }
   182    183   
   183    184     sqlite3_free(apOld);
   184    185     pHash->nSlot = nNew;
................................................................................
   473    474   int sqlite3Fts5HashQuery(
   474    475     Fts5Hash *pHash,                /* Hash table to query */
   475    476     const char *pTerm, int nTerm,   /* Query term */
   476    477     const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
   477    478     int *pnDoclist                  /* OUT: Size of doclist in bytes */
   478    479   ){
   479    480     unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
   480         -  char *zKey;
          481  +  char *zKey = 0;
   481    482     Fts5HashEntry *p;
   482    483   
   483    484     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   484    485       zKey = fts5EntryKey(p);
   485    486       if( memcmp(zKey, pTerm, nTerm)==0 && zKey[nTerm]==0 ) break;
   486    487     }
   487    488   

Changes to ext/fts5/fts5_index.c.

   724    724   static int fts5IndexPrepareStmt(
   725    725     Fts5Index *p,
   726    726     sqlite3_stmt **ppStmt,
   727    727     char *zSql
   728    728   ){
   729    729     if( p->rc==SQLITE_OK ){
   730    730       if( zSql ){
   731         -      p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
          731  +      p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1,
          732  +                                 SQLITE_PREPARE_PERSISTENT, ppStmt, 0);
   732    733       }else{
   733    734         p->rc = SQLITE_NOMEM;
   734    735       }
   735    736     }
   736    737     sqlite3_free(zSql);
   737    738     return p->rc;
   738    739   }
................................................................................
   773    774       char *zSql = sqlite3_mprintf(
   774    775           "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", 
   775    776             pConfig->zDb, pConfig->zName
   776    777       );
   777    778       if( zSql==0 ){
   778    779         rc = SQLITE_NOMEM;
   779    780       }else{
   780         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0);
          781  +      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
          782  +                              SQLITE_PREPARE_PERSISTENT, &p->pDeleter, 0);
   781    783         sqlite3_free(zSql);
   782    784       }
   783    785       if( rc!=SQLITE_OK ){
   784    786         p->rc = rc;
   785    787         return;
   786    788       }
   787    789     }

Changes to ext/fts5/fts5_main.c.

   879    879     va_list ap;
   880    880   
   881    881     va_start(ap, zFmt);
   882    882     zSql = sqlite3_vmprintf(zFmt, ap);
   883    883     if( zSql==0 ){
   884    884       rc = SQLITE_NOMEM; 
   885    885     }else{
   886         -    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
          886  +    rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 
          887  +                            SQLITE_PREPARE_PERSISTENT, &pRet, 0);
   887    888       if( rc!=SQLITE_OK ){
   888    889         *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
   889    890       }
   890    891       sqlite3_free(zSql);
   891    892     }
   892    893   
   893    894     va_end(ap);
................................................................................
  1015   1016     const char *zRank = pCsr->zRank;
  1016   1017     const char *zRankArgs = pCsr->zRankArgs;
  1017   1018   
  1018   1019     if( zRankArgs ){
  1019   1020       char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
  1020   1021       if( zSql ){
  1021   1022         sqlite3_stmt *pStmt = 0;
  1022         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0);
         1023  +      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
         1024  +                              SQLITE_PREPARE_PERSISTENT, &pStmt, 0);
  1023   1025         sqlite3_free(zSql);
  1024   1026         assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
  1025   1027         if( rc==SQLITE_OK ){
  1026   1028           if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1027   1029             int nByte;
  1028   1030             pCsr->nRankArg = sqlite3_column_count(pStmt);
  1029   1031             nByte = sizeof(sqlite3_value*)*pCsr->nRankArg;
................................................................................
  2603   2605   
  2604   2606     sqlite3_free(pGlobal);
  2605   2607   }
  2606   2608   
  2607   2609   static void fts5Fts5Func(
  2608   2610     sqlite3_context *pCtx,          /* Function call context */
  2609   2611     int nArg,                       /* Number of args */
  2610         -  sqlite3_value **apUnused        /* Function arguments */
         2612  +  sqlite3_value **apArg           /* Function arguments */
  2611   2613   ){
  2612   2614     Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
  2613         -  char buf[8];
  2614         -  UNUSED_PARAM2(nArg, apUnused);
  2615         -  assert( nArg==0 );
  2616         -  assert( sizeof(buf)>=sizeof(pGlobal) );
  2617         -  memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
  2618         -  sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
         2615  +  fts5_api **ppApi;
         2616  +  UNUSED_PARAM(nArg);
         2617  +  assert( nArg==1 );
         2618  +  ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr");
         2619  +  if( ppApi ) *ppApi = &pGlobal->api;
  2619   2620   }
  2620   2621   
  2621   2622   /*
  2622   2623   ** Implementation of fts5_source_id() function.
  2623   2624   */
  2624   2625   static void fts5SourceIdFunc(
  2625   2626     sqlite3_context *pCtx,          /* Function call context */
................................................................................
  2676   2677       if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
  2677   2678       if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
  2678   2679       if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
  2679   2680       if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
  2680   2681       if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
  2681   2682       if( rc==SQLITE_OK ){
  2682   2683         rc = sqlite3_create_function(
  2683         -          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
         2684  +          db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
  2684   2685         );
  2685   2686       }
  2686   2687       if( rc==SQLITE_OK ){
  2687   2688         rc = sqlite3_create_function(
  2688   2689             db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0
  2689   2690         );
  2690   2691       }

Changes to ext/fts5/fts5_storage.c.

   132    132           zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
   133    133           break;
   134    134       }
   135    135   
   136    136       if( zSql==0 ){
   137    137         rc = SQLITE_NOMEM;
   138    138       }else{
   139         -      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
          139  +      rc = sqlite3_prepare_v3(pC->db, zSql, -1,
          140  +                              SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
   140    141         sqlite3_free(zSql);
   141    142         if( rc!=SQLITE_OK && pzErrMsg ){
   142    143           *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
   143    144         }
   144    145       }
   145    146     }
   146    147   

Changes to ext/fts5/fts5_tcl.c.

    95     95     int rc = f5tDbPointer(interp, pObj, &db);
    96     96     if( rc!=TCL_OK ){
    97     97       return TCL_ERROR;
    98     98     }else{
    99     99       sqlite3_stmt *pStmt = 0;
   100    100       fts5_api *pApi = 0;
   101    101   
   102         -    rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0);
          102  +    rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
   103    103       if( rc!=SQLITE_OK ){
   104    104         Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
   105    105         return TCL_ERROR;
   106    106       }
   107         -
   108         -    if( SQLITE_ROW==sqlite3_step(pStmt) ){
   109         -      const void *pPtr = sqlite3_column_blob(pStmt, 0);
   110         -      memcpy((void*)&pApi, pPtr, sizeof(pApi));
   111         -    }
          107  +    sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr");
          108  +    sqlite3_step(pStmt);
   112    109   
   113    110       if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
   114    111         Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
   115    112         return TCL_ERROR;
   116    113       }
   117    114   
   118    115       *ppDb = db;

Changes to ext/fts5/fts5_test_mi.c.

    69     69   ** handle (accessible using sqlite3_errcode()/errmsg()).
    70     70   */
    71     71   static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
    72     72     sqlite3_stmt *pStmt = 0;
    73     73     int rc;
    74     74   
    75     75     *ppApi = 0;
    76         -  rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
           76  +  rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
    77     77     if( rc==SQLITE_OK ){
    78         -    if( SQLITE_ROW==sqlite3_step(pStmt) 
    79         -        && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
    80         -      ){
    81         -      memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
    82         -    }
           78  +    sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr");
           79  +    (void)sqlite3_step(pStmt);
    83     80       rc = sqlite3_finalize(pStmt);
    84     81     }
    85     82   
    86     83     return rc;
    87     84   }
    88     85   
    89     86   
................................................................................
   418    415     /* Register the implementation of matchinfo() */
   419    416     rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
   420    417   
   421    418     return rc;
   422    419   }
   423    420   
   424    421   #endif /* SQLITE_ENABLE_FTS5 */
   425         -

Changes to ext/fts5/fts5_test_tok.c.

   178    178     sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
   179    179     char **pzErr                    /* OUT: sqlite3_malloc'd error message */
   180    180   ){
   181    181     fts5_api *pApi = (fts5_api*)pCtx;
   182    182     Fts5tokTable *pTab = 0;
   183    183     int rc;
   184    184     char **azDequote = 0;
   185         -  int nDequote;
          185  +  int nDequote = 0;
   186    186   
   187    187     rc = sqlite3_declare_vtab(db, 
   188    188          "CREATE TABLE x(input HIDDEN, token, start, end, position)"
   189    189     );
   190    190   
   191    191     if( rc==SQLITE_OK ){
   192    192       nDequote = argc-3;

Changes to ext/fts5/test/fts5aa.test.

   437    437   # exception. But since bm25() can now used the cached structure record,
   438    438   # it never sees the corruption introduced by funk() and so the following 
   439    439   # statement no longer fails.
   440    440   #
   441    441   do_catchsql_test 16.2 {
   442    442     SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
   443    443   } {0 {{} -1e-06 {}}}
   444         -# {1 {SQL logic error or missing database}}
          444  +# {1 {SQL logic error}}
   445    445   
   446    446   #-------------------------------------------------------------------------
   447    447   #
   448    448   reset_db
   449    449   do_execsql_test 17.1 {
   450    450     CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
   451    451     INSERT INTO b2 VALUES('a');
................................................................................
   591    591     SELECT rowid FROM t9('a*')
   592    592   } {1}
   593    593   
   594    594   }
   595    595   
   596    596   
   597    597   finish_test
   598         -
   599         -

Changes to ext/fts5/test/fts5ab.test.

   290    290     INSERT INTO x1 VALUES($doc);
   291    291   }
   292    292   
   293    293   } ;# foreach_detail_mode...
   294    294   
   295    295   
   296    296   finish_test
   297         -

Changes to ext/fts5/test/fts5ac.test.

   272    272   } {
   273    273     do_execsql_test 2.3.$tn {
   274    274       SELECT fts5_expr_tcl($expr, 'N $x')
   275    275     } [list $tclexpr]
   276    276   }
   277    277   
   278    278   finish_test
   279         -

Changes to ext/fts5/test/fts5ad.test.

   227    227         28 {a f*} 29 {a* f*} 30 {a* fghij*}
   228    228       } {
   229    229         set res [prefix_query $prefix]
   230    230         if {$bAsc} {
   231    231           set res [lsort -integer -increasing $res]
   232    232         }
   233    233         set n [llength $res]
   234         -      if {$T==5} breakpoint 
   235    234         do_execsql_test $T.$bAsc.$tn.$n $sql $res
   236    235       }
   237    236     }
   238    237   
   239    238     catchsql COMMIT
   240    239   }
   241    240   
   242    241   }
   243    242   
   244    243   finish_test
   245         -

Changes to ext/fts5/test/fts5ae.test.

   305    305       SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
   306    306     } $cnt
   307    307   }
   308    308   
   309    309   }
   310    310   
   311    311   finish_test
   312         -

Changes to ext/fts5/test/fts5af.test.

   174    174   do_execsql_test 5.1 {
   175    175     SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
   176    176   } {{[x] a a a a a...}}
   177    177   
   178    178   } ;# foreach_detail_mode 
   179    179   
   180    180   finish_test
   181         -

Changes to ext/fts5/test/fts5ag.test.

   138    138     }
   139    139   }
   140    140   
   141    141   } ;# foreach_detail_mode
   142    142   
   143    143   
   144    144   finish_test
   145         -

Changes to ext/fts5/test/fts5ah.test.

   163    163   } {10000}
   164    164   
   165    165   } ;# foreach_detail_mode
   166    166   
   167    167   #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
   168    168   
   169    169   finish_test
   170         -

Changes to ext/fts5/test/fts5ai.test.

    51     51   do_execsql_test 1.2 {
    52     52     INSERT INTO t1(t1) VALUES('integrity-check');
    53     53   }
    54     54   }
    55     55   
    56     56   
    57     57   finish_test
    58         -

Changes to ext/fts5/test/fts5aj.test.

    62     62     }
    63     63   }
    64     64   
    65     65   do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }
    66     66   
    67     67   
    68     68   finish_test
    69         -

Changes to ext/fts5/test/fts5ak.test.

   143    143     {[a b c] [c d e]}
   144    144     {[a b c d e]}
   145    145   }
   146    146   
   147    147   }
   148    148   
   149    149   finish_test
   150         -

Changes to ext/fts5/test/fts5al.test.

    73     73     1 ""
    74     74     2 "fname"
    75     75     3 "fname(X'234ab')"
    76     76     4 "myfunc(-1.,'abc')"
    77     77   } {
    78     78     do_test 2.2.$tn {
    79     79       catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
    80         -  } {1 {SQL logic error or missing database}}
           80  +  } {1 {SQL logic error}}
    81     81   }
    82     82   
    83     83   #-------------------------------------------------------------------------
    84     84   # Assorted tests of the tcl interface for creating extension functions.
    85     85   #
    86     86   
    87     87   do_execsql_test 3.1 {
................................................................................
   293    293     SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
   294    294   } {1 {parse error in rank function: }}
   295    295   
   296    296   } ;# foreach_detail_mode
   297    297   
   298    298   
   299    299   finish_test
   300         -

Changes to ext/fts5/test/fts5alter.test.

    85     85   
    86     86   do_execsql_test 3.1 {
    87     87     CREATE VIRTUAL TABLE abc USING fts5(a);
    88     88     INSERT INTO abc(rowid, a) VALUES(1, 'a');
    89     89     BEGIN;
    90     90       INSERT INTO abc(rowid, a) VALUES(2, 'a');
    91     91   }
    92         -breakpoint
    93     92   do_execsql_test 3.2 {
    94     93       SELECT rowid FROM abc WHERE abc MATCH 'a';
    95     94   } {1 2}
    96     95   
    97     96   do_execsql_test 3.3 {
    98     97     COMMIT;
    99     98     SELECT rowid FROM abc WHERE abc MATCH 'a';
   100     99   } {1 2}
   101    100   
   102    101   finish_test
   103         -

Changes to ext/fts5/test/fts5auto.test.

   338    338   } {
   339    339     do_auto_test 4.$tn yy $expr
   340    340   }
   341    341   
   342    342   
   343    343   
   344    344   finish_test
   345         -

Changes to ext/fts5/test/fts5aux.test.

   236    236     4  {"a a a" "b" "a d"} {"[a] [a] [a]" "[a] d"}
   237    237     1  {"b d" "a b"}       {"[b] [d]" "[a] b"}
   238    238     2  {"d b" "a d"}       {"[d] [b]" "[a] d"}
   239    239     3  {"a a d"}           {"[a] [a] d"}
   240    240   } {
   241    241     execsql { DELETE FROM x1 }
   242    242     foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } }
   243         -  breakpoint
   244    243     do_execsql_test 8.$tn {
   245    244       SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)';
   246    245     } $res
   247    246   }
   248    247   
   249    248   #-------------------------------------------------------------------------
   250    249   # Test the built-in bm25() demo.
................................................................................
   275    274   } {
   276    275     9 10
   277    276   }
   278    277   
   279    278   
   280    279   
   281    280   finish_test
   282         -

Changes to ext/fts5/test/fts5auxdata.test.

   108    108   db eval { 
   109    109     SELECT aux_function_2(f1, 2, 'A'), aux_function_2(f1, 2, 'B') 
   110    110     FROM f1 WHERE f1 MATCH 'a'
   111    111     ORDER BY rowid ASC
   112    112   }
   113    113   
   114    114   finish_test
   115         -

Changes to ext/fts5/test/fts5bigpl.test.

    57     57       set doc [string repeat "$t " 150000000]
    58     58       execsql { INSERT INTO t1 VALUES($doc) }
    59     59     }
    60     60     execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    61     61   } {}
    62     62   
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5bigtok.test.

    60     60       do_execsql_test 2.[string range $v 0 0] {
    61     61         SELECT rowid FROM t1($v) ORDER BY rowid DESC
    62     62       } [lsort -integer -decr $res]
    63     63     }
    64     64   }
    65     65   
    66     66   finish_test
    67         -
    68         -

Changes to ext/fts5/test/fts5colset.test.

    80     80     do_catchsql_test 4.1 {
    81     81       SELECT * FROM t1 WHERE rowid MATCH 'a'
    82     82     } {1 {unable to use function MATCH in the requested context}}
    83     83   }
    84     84   
    85     85   
    86     86   finish_test
    87         -
    88         -

Changes to ext/fts5/test/fts5columnsize.test.

   139    139   #
   140    140   do_execsql_test 4.1.1 {
   141    141     CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0);
   142    142     INSERT INTO t5 VALUES('1 2 3 4');
   143    143     INSERT INTO t5 VALUES('2 4 6 8');
   144    144   }
   145    145   
   146         -breakpoint
   147    146   do_execsql_test 4.1.2 {
   148    147     INSERT INTO t5(t5) VALUES('integrity-check');
   149    148   }
   150    149   
   151    150   finish_test

Changes to ext/fts5/test/fts5config.test.

    62     62     5 "f1(x':;')"
    63     63     6 "f1(x'[]')"
    64     64     7 "f1(x'{}')"
    65     65     8 "f1('abc)"
    66     66   } {
    67     67     do_catchsql_test 3.$tn {
    68     68       INSERT INTO t1(t1, rank) VALUES('rank', $val);
    69         -  } {1 {SQL logic error or missing database}}
           69  +  } {1 {SQL logic error}}
    70     70   }
    71     71   
    72     72   #-------------------------------------------------------------------------
    73     73   # The parsing of SQL literals specified as part of 'rank' options.
    74     74   #
    75     75   do_execsql_test 4.0 {
    76     76     CREATE VIRTUAL TABLE zzz USING fts5(one);
................................................................................
   106    106   #-------------------------------------------------------------------------
   107    107   # Misquoting in tokenize= and other options. 
   108    108   #
   109    109   do_catchsql_test 5.1 {
   110    110     CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii");
   111    111   } {1 {parse error in tokenize directive}} 
   112    112   
   113         -breakpoint
   114    113   do_catchsql_test 5.2 {
   115    114     CREATE VIRTUAL TABLE xx USING fts5(x, [y[]);
   116    115   } {0 {}}
   117    116   
   118    117   do_catchsql_test 5.3 {
   119    118     CREATE VIRTUAL TABLE yy USING fts5(x, [y]]);
   120    119   } {1 {unrecognized token: "]"}}
................................................................................
   165    164   #   9.5.* 'hashsize' options.
   166    165   #
   167    166   do_execsql_test 9.0 {
   168    167     CREATE VIRTUAL TABLE abc USING fts5(a, b);
   169    168   } {}
   170    169   do_catchsql_test 9.1.1 {
   171    170     INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
   172         -} {1 {SQL logic error or missing database}}
          171  +} {1 {SQL logic error}}
   173    172   do_catchsql_test 9.1.2 {
   174    173     INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
   175         -} {1 {SQL logic error or missing database}}
          174  +} {1 {SQL logic error}}
   176    175   do_catchsql_test 9.1.3 {
   177    176     INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
   178         -} {1 {SQL logic error or missing database}}
          177  +} {1 {SQL logic error}}
   179    178   
   180    179   do_catchsql_test 9.2.1 {
   181    180     INSERT INTO abc(abc, rank) VALUES('automerge', -5);
   182         -} {1 {SQL logic error or missing database}}
          181  +} {1 {SQL logic error}}
   183    182   do_catchsql_test 9.2.2 {
   184    183     INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
   185         -} {1 {SQL logic error or missing database}}
          184  +} {1 {SQL logic error}}
   186    185   do_catchsql_test 9.2.3 {
   187    186     INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
   188         -} {1 {SQL logic error or missing database}}
          187  +} {1 {SQL logic error}}
   189    188   do_execsql_test 9.2.4 {
   190    189     INSERT INTO abc(abc, rank) VALUES('automerge', 1);
   191    190   } {}
   192    191   
   193    192   do_catchsql_test 9.3.1 {
   194    193     INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
   195         -} {1 {SQL logic error or missing database}}
          194  +} {1 {SQL logic error}}
   196    195   do_catchsql_test 9.3.2 {
   197    196     INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
   198         -} {1 {SQL logic error or missing database}}
          197  +} {1 {SQL logic error}}
   199    198   do_execsql_test 9.3.3 {
   200    199     INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
   201    200   } {}
   202    201   do_execsql_test 9.3.4 {
   203    202     INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
   204    203   } {}
   205    204   
   206    205   do_catchsql_test 9.4.1 {
   207    206     INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
   208         -} {1 {SQL logic error or missing database}}
          207  +} {1 {SQL logic error}}
   209    208   
   210    209   do_catchsql_test 9.5.1 {
   211    210     INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
   212         -} {1 {SQL logic error or missing database}}
          211  +} {1 {SQL logic error}}
   213    212   do_catchsql_test 9.5.2 {
   214    213     INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
   215         -} {1 {SQL logic error or missing database}}
          214  +} {1 {SQL logic error}}
   216    215   do_catchsql_test 9.5.3 {
   217    216     INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
   218    217   } {0 {}}
   219    218   
   220    219   #-------------------------------------------------------------------------
   221    220   # Too many prefix indexes. Maximum allowed is 31.
   222    221   #
................................................................................
   241    240   } {
   242    241     set res [list 1 {malformed detail=... directive}]
   243    242     do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
   244    243   }
   245    244   
   246    245   do_catchsql_test 12.1 {
   247    246     INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
   248         -} {1 {SQL logic error or missing database}}
          247  +} {1 {SQL logic error}}
   249    248   
   250    249   #-------------------------------------------------------------------------
   251    250   # errors in the 'usermerge' option
   252    251   #
   253    252   do_execsql_test 13.0 {
   254    253     CREATE VIRTUAL TABLE tt USING fts5(ttt);
   255    254   }
................................................................................
   256    255   foreach {tn val} {
   257    256     1     -1
   258    257     2     4.2
   259    258     3     17
   260    259     4     1
   261    260   } {
   262    261     set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
   263         -  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
          262  +  do_catchsql_test 13.$tn $sql {1 {SQL logic error}}
   264    263   }
   265    264   
   266    265   finish_test
   267         -

Changes to ext/fts5/test/fts5conflict.test.

    62     62     REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1');
    63     63     DELETE FROM tbl WHERE a=100;
    64     64   
    65     65     INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
    66     66   }
    67     67   
    68     68   finish_test
    69         -
    70         -

Changes to ext/fts5/test/fts5content.test.

   251    251   do_execsql_test 6.2 {
   252    252     DROP TABLE xx;
   253    253     SELECT name FROM sqlite_master;
   254    254   } {}
   255    255   
   256    256   
   257    257   finish_test
   258         -

Changes to ext/fts5/test/fts5corrupt.test.

    92     92   
    93     93   do_catchsql_test 3.1 {
    94     94     DELETE FROM t3_content WHERE rowid = 3;
    95     95     SELECT * FROM t3 WHERE t3 MATCH 'o';
    96     96   } {1 {database disk image is malformed}}
    97     97   
    98     98   finish_test
    99         -

Changes to ext/fts5/test/fts5corrupt2.test.

   265    265   do_catchsql_test 6.2 {
   266    266     SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a'
   267    267   } {1 SQLITE_CORRUPT_VTAB}
   268    268   
   269    269   
   270    270   sqlite3_fts5_may_be_corrupt 0
   271    271   finish_test
   272         -

Changes to ext/fts5/test/fts5corrupt3.test.

   405    405   } {}
   406    406   do_catchsql_test 9.2.2 {
   407    407     SELECT * FROM t1('one AND two');
   408    408   } {1 {database disk image is malformed}}
   409    409   
   410    410   sqlite3_fts5_may_be_corrupt 0
   411    411   finish_test
   412         -

Changes to ext/fts5/test/fts5delete.test.

    47     47     for {set i 0} {$i < 5} {incr i} {
    48     48       execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
    49     49       execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    50     50     }
    51     51   } {}
    52     52   
    53     53   finish_test
    54         -

Changes to ext/fts5/test/fts5detail.test.

   237    237       (SELECT sum(length(block)) from t2_data) <
   238    238       (SELECT sum(length(block)) from t3_data)
   239    239   } {1}
   240    240   
   241    241   
   242    242   
   243    243   finish_test
   244         -

Changes to ext/fts5/test/fts5determin.test.

    59     59     } {}
    60     60   
    61     61     do_determin_test 1.4
    62     62   }
    63     63   
    64     64   
    65     65   finish_test
    66         -
    67         -

Changes to ext/fts5/test/fts5dlidx.test.

    62     62           append doc " y" 
    63     63         }
    64     64       }
    65     65       execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) }
    66     66     }
    67     67     execsql COMMIT
    68     68   
    69         -  breakpoint
    70     69     do_test $tn.1 {
    71     70       execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    72     71     } {}
    73     72     
    74     73     do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
    75     74     do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
    76     75     
................................................................................
   120    119       INSERT INTO t1(rowid,x) SELECT i, $str FROM iii;
   121    120       COMMIT;
   122    121     }
   123    122   
   124    123     do_execsql_test $tn.1 {
   125    124       SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a'
   126    125     } {1}
   127         -  breakpoint
   128    126     do_execsql_test $tn.2 {
   129    127       SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
   130    128     } {1}
   131    129   }
   132    130   
   133    131   do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
   134    132   
................................................................................
   193    191   }
   194    192   
   195    193   } ;# foreach_detail_mode
   196    194   
   197    195   
   198    196   
   199    197   finish_test
   200         -

Changes to ext/fts5/test/fts5doclist.test.

    40     40   
    41     41   do_execsql_test 1.2 {
    42     42     INSERT INTO ccc(ccc) VALUES('integrity-check');
    43     43   }
    44     44   
    45     45   
    46     46   finish_test
    47         -

Changes to ext/fts5/test/fts5eb.test.

    77     77   do_execsql_test 3.3 {
    78     78     SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
    79     79   } {1 -1e-06}
    80     80   
    81     81   
    82     82   
    83     83   finish_test
    84         -
    85         -
    86         -

Changes to ext/fts5/test/fts5fault1.test.

   347    347       if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" }
   348    348     }
   349    349   }
   350    350   
   351    351   
   352    352   
   353    353   finish_test
   354         -

Changes to ext/fts5/test/fts5fault2.test.

   133    133       );
   134    134     }
   135    135   } -test {
   136    136     faultsim_test_result {0 {}}
   137    137   }
   138    138   
   139    139   finish_test
   140         -

Changes to ext/fts5/test/fts5fault3.test.

   106    106   } -test {
   107    107     faultsim_test_result [list 0 {}]
   108    108   }
   109    109   
   110    110   
   111    111   
   112    112   finish_test
   113         -

Changes to ext/fts5/test/fts5fault4.test.

   391    391   } -body {
   392    392     db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" }
   393    393   } -test {
   394    394     faultsim_test_result {0 {}}
   395    395   }
   396    396   
   397    397   finish_test
   398         -

Changes to ext/fts5/test/fts5fault5.test.

   101    101     db eval {
   102    102       SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
   103    103     }
   104    104   } -test {
   105    105     faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
   106    106   }
   107    107   
   108         -breakpoint
   109    108   do_execsql_test 3.3.0 {
   110    109     SELECT * FROM tv2;
   111    110   } {
   112    111     0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        
   113    112     14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     
   114    113     2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   
   115    114     9 x 1 {}
................................................................................
   126    125         9 x 1 {}
   127    126     ]]
   128    127   }
   129    128   
   130    129   
   131    130   
   132    131   finish_test
   133         -

Changes to ext/fts5/test/fts5fault6.test.

   276    276     }
   277    277   } -test {
   278    278     faultsim_test_result {0 1}
   279    279   }
   280    280   
   281    281   #-------------------------------------------------------------------------
   282    282   catch { db close }
   283         -breakpoint
   284    283   do_faultsim_test 6 -faults oom* -prep {
   285    284     sqlite_orig db test.db
   286    285     sqlite3_db_config_lookaside db 0 0 0
   287    286   } -test {
   288    287     faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }}
   289    288     if {$testrc==0} {
   290    289       db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) }
   291    290     }
   292    291     db close
   293    292   }
   294    293   finish_test
   295         -

Changes to ext/fts5/test/fts5fault7.test.

   112    112   do_faultsim_test 2.2 -faults oom-* -body {
   113    113     db eval { SELECT * FROM xy('""') }
   114    114   } -test {
   115    115     faultsim_test_result {0 {}}
   116    116   }
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5fault8.test.

    78     78     execsql { INSERT INTO x2(x2) VALUES('optimize') }
    79     79   } -test {
    80     80     faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
    81     81   }
    82     82   
    83     83   
    84     84   finish_test
    85         -

Changes to ext/fts5/test/fts5fault9.test.

   149    149     faultsim_test_result [list 0 {1 3}]
   150    150   }
   151    151   
   152    152   
   153    153   } ;# foreach_detail_mode...
   154    154   
   155    155   finish_test
   156         -

Changes to ext/fts5/test/fts5faultA.test.

    57     57     sqlite3 db test.db
    58     58   } -body {
    59     59     execsql { SELECT rowid FROM o2('a+b+c NOT xyz') }
    60     60   } -test {
    61     61     faultsim_test_result {0 {1 2}}
    62     62   }
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5faultB.test.

   128    128     execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
   129    129   } -test {
   130    130     faultsim_test_result {0 {2 3}}
   131    131   }
   132    132   
   133    133   
   134    134   finish_test
   135         -

Changes to ext/fts5/test/fts5full.test.

    36     36         execsql { INSERT INTO x8 VALUES( rnddoc(5) ); }
    37     37       }
    38     38     } msg] $msg
    39     39   } {1 {database or disk is full}}
    40     40   
    41     41   
    42     42   finish_test
    43         -

Changes to ext/fts5/test/fts5fuzz1.test.

    86     86   reset_db
    87     87   do_catchsql_test 4.1 {
    88     88     CREATE VIRTUAL TABLE f2 USING fts5(o, t);
    89     89     SELECT * FROM f2('(8 AND 9)`AND 10');
    90     90   } {1 {fts5: syntax error near "`"}}
    91     91   
    92     92   finish_test
    93         -

Changes to ext/fts5/test/fts5hash.test.

   117    117       set hash [sqlite3_fts5_token_hash 1024 $big]
   118    118       while {1} {
   119    119         set small [random_token]
   120    120         if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break
   121    121       }
   122    122   
   123    123       execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) }
   124         -breakpoint
   125    124       execsql {
   126    125         INSERT INTO t2 VALUES($small || ' ' || $big);
   127    126       }
   128    127     } {}
   129    128   
   130    129   } ;# foreach_detail_mode
   131    130   
   132    131   finish_test
   133         -

Changes to ext/fts5/test/fts5integrity.test.

   206    206         if {$res == [lsort -integer $res2]} { incr ok }
   207    207       }
   208    208       set ok
   209    209     } {1000}
   210    210   }
   211    211   
   212    212   finish_test
   213         -

Changes to ext/fts5/test/fts5lastrowid.test.

    66     66   do_execsql_test 1.6 {
    67     67     INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
    68     68     SELECT last_insert_rowid();
    69     69   } {14}
    70     70   
    71     71   
    72     72   finish_test
    73         -

Added ext/fts5/test/fts5leftjoin.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5leftjoin
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE vt USING fts5(x);
           26  +  INSERT INTO vt VALUES('abc');
           27  +  INSERT INTO vt VALUES('xyz');
           28  +
           29  +  CREATE TABLE t1(a INTEGER PRIMARY KEY);
           30  +  INSERT INTO t1 VALUES(1), (2);
           31  +}
           32  +
           33  +do_execsql_test 1.1 {
           34  +  SELECT * FROM t1 LEFT JOIN (
           35  +    SELECT rowid AS rrr, * FROM vt WHERE vt MATCH 'abc'
           36  +  ) ON t1.a = rrr
           37  +} {1 1 abc 2 {} {}}
           38  +
           39  +do_execsql_test 1.2 {
           40  +  SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc')
           41  +} {1 abc 2 abc}
           42  +
           43  +finish_test

Changes to ext/fts5/test/fts5matchinfo.test.

   468    468   } ;# foreach_detail_mode
   469    469   
   470    470   #-------------------------------------------------------------------------
   471    471   # Test that a bad fts5() return is detected
   472    472   #
   473    473   reset_db
   474    474   proc xyz {} {}
   475         -db func fts5 -argcount 0 xyz
          475  +db func fts5 -argcount 1 xyz
   476    476   do_test 13.1 {
   477    477     list [catch { sqlite3_fts5_register_matchinfo db } msg] $msg
   478    478   } {1 SQLITE_ERROR}
   479    479   
   480    480   #-------------------------------------------------------------------------
   481    481   # Test that an invalid matchinfo() flag is detected
   482    482   #
................................................................................
   488    488   } {}
   489    489   
   490    490   do_catchsql_test 14.2 {
   491    491     SELECT matchinfo(x1, 'd') FROM x1('a b c');
   492    492   } {1 {unrecognized matchinfo flag: d}}
   493    493   
   494    494   finish_test
   495         -

Changes to ext/fts5/test/fts5merge.test.

   237    237   do_execsql_test 6.3 {
   238    238     INSERT INTO g1(g1) VALUES('integrity-check');
   239    239   }
   240    240   
   241    241   
   242    242   
   243    243   finish_test
   244         -

Changes to ext/fts5/test/fts5merge2.test.

    51     51   do_execsql_test 1.2 {
    52     52     INSERT INTO t1(t1) VALUES('integrity-check');
    53     53   }
    54     54   
    55     55   }
    56     56   
    57     57   finish_test
    58         -

Changes to ext/fts5/test/fts5multiclient.test.

    41     41       sql1 { INSERT INTO t1 VALUES('a b c') }
    42     42       sql3 { INSERT INTO t1(t1) VALUES('integrity-check') }
    43     43     } {}
    44     44   
    45     45   };# do_multiclient_test
    46     46   };# foreach_detail_mode
    47     47   finish_test
    48         -

Changes to ext/fts5/test/fts5near.test.

    64     64   do_near_test 1.23 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 4) } 0
    65     65   
    66     66   do_near_test 1.24 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 5) } 1
    67     67   do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0
    68     68   
    69     69   
    70     70   finish_test
    71         -

Changes to ext/fts5/test/fts5onepass.test.

   174    174       UPDATE ttt SET x = 'A B C' WHERE rowid = 4;
   175    175       INSERT INTO ttt(rowid, x) VALUES(6, 'd e f');
   176    176     COMMIT;
   177    177   } {}
   178    178   do_test 4.2.2 { fts5_level_segs ttt } {3}
   179    179   
   180    180   finish_test
   181         -

Changes to ext/fts5/test/fts5optimize.test.

   102    102     do_execsql_test 2.$tn.5 {
   103    103       INSERT INTO t1(t1) VALUES('integrity-check');
   104    104     }
   105    105   
   106    106     do_test 2.$tn.6 { fts5_segcount t1 } 1
   107    107   }
   108    108   finish_test
   109         -

Changes to ext/fts5/test/fts5phrase.test.

   112    112     FROM t3('a:f+f')
   113    113   } {
   114    114     31 {h *f f*} {i j g e c} {j j f c a i j} 
   115    115     50 {*f f* c} {f f b i i} {f f a j e c i}
   116    116   }
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5plan.test.

    60     60     0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
    61     61   }
    62     62   
    63     63   
    64     64   
    65     65   
    66     66   finish_test
    67         -

Changes to ext/fts5/test/fts5porter.test.

 11799  11799       lindex [sqlite3_fts5_tokenize db porter $in] 0
 11800  11800     } $out
 11801  11801     incr i
 11802  11802   }
 11803  11803   
 11804  11804   
 11805  11805   finish_test
 11806         -

Changes to ext/fts5/test/fts5porter2.test.

    63     63       lindex [sqlite3_fts5_tokenize db porter $in] 0
    64     64     } $out
    65     65     incr i
    66     66   }
    67     67   
    68     68   
    69     69   finish_test
    70         -

Changes to ext/fts5/test/fts5prefix.test.

   337    337       do_execsql_test 7.$tn {
   338    338         SELECT md5sum(id, block) FROM tt_data
   339    339       } [list $::checksum]
   340    340     }
   341    341   }
   342    342   
   343    343   finish_test
   344         -
   345         -

Changes to ext/fts5/test/fts5query.test.

    75     75       } {}
    76     76       incr ret
    77     77     }
    78     78   }
    79     79   
    80     80   
    81     81   finish_test
    82         -
    83         -

Changes to ext/fts5/test/fts5rank.test.

    86     86     execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
    87     87   } {1 3 2}
    88     88   
    89     89   do_test 2.7 {
    90     90     execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
    91     91   } {1 3 2}
    92     92   
           93  +db2 close
    93     94   
    94     95   #--------------------------------------------------------------------------
    95     96   # At one point there was a problem with queries such as:
    96     97   #
    97     98   #   ... MATCH 'x OR y' ORDER BY rank;
    98     99   #
    99    100   # if there were zero occurrences of token 'y' in the dataset. The
................................................................................
   147    148     VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
   148    149   } {{wrinkle in time} {Bill Smith}}
   149    150   
   150    151   
   151    152   
   152    153   
   153    154   finish_test
   154         -

Changes to ext/fts5/test/fts5rebuild.test.

    60     60     CREATE VIRTUAL TABLE nc USING fts5(doc, content=);
    61     61   }
    62     62   
    63     63   do_catchsql_test 2.2 {
    64     64     INSERT INTO nc(nc) VALUES('rebuild');
    65     65   } {1 {'rebuild' may not be used with a contentless fts5 table}}
    66     66   finish_test
    67         -

Changes to ext/fts5/test/fts5restart.test.

   145    145     }
   146    146     set res
   147    147   } {500 400 300}
   148    148   
   149    149   
   150    150   
   151    151   finish_test
   152         -

Changes to ext/fts5/test/fts5rowid.test.

   212    212   } {36}
   213    213   
   214    214   #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r}
   215    215   
   216    216   
   217    217   
   218    218   finish_test
   219         -

Changes to ext/fts5/test/fts5simple.test.

   407    407   
   408    408   do_catchsql_test 19.2 {
   409    409     SELECT * FROM x1 WHERE x1 MATCH 'c0 AND (c1 AND (c2 AND (c3 AND (c4 AND (c5 AND (c6 AND (c7 AND (c8 AND (c9 AND (c10 AND (c11 AND (c12 AND (c13 AND (c14 AND (c15 AND (c16 AND (c17 AND (c18 AND (c19 AND (c20 AND (c21 AND (c22 AND (c23 AND (c24 AND (c25 AND (c26 AND (c27 AND (c28 AND (c29 AND (c30 AND (c31 AND (c32 AND (c33 AND (c34 AND (c35 AND (c36 AND (c37 AND (c38 AND (c39 AND (c40 AND (c41 AND (c42 AND (c43 AND (c44 AND (c45 AND (c46 AND (c47 AND (c48 AND (c49 AND (c50 AND (c51 AND (c52 AND (c53 AND (c54 AND (c55 AND (c56 AND (c57 AND (c58 AND (c59 AND (c60 AND (c61 AND (c62 AND (c63 AND (c64 AND (c65 AND (c66 AND (c67 AND (c68 AND (c69 AND (c70 AND (c71 AND (c72 AND (c73 AND (c74 AND (c75 AND (c76 AND (c77 AND (c78 AND (c79 AND (c80 AND (c81 AND (c82 AND (c83 AND (c84 AND (c85 AND (c86 AND (c87 AND (c88 AND (c89 AND (c90 AND (c91 AND (c92 AND (c93 AND (c94 AND (c95 AND (c96 AND (c97 AND (c98 AND (c99 AND (c100 AND (c101 AND (c102 AND (c103 AND (c104 AND (c105 AND (c106 AND (c107 AND (c108 AND (c109 AND (c110 AND (c111 AND (c112 AND (c113 AND (c114 AND (c115 AND (c116 AND (c117 AND (c118 AND (c119 AND (c120 AND (c121 AND (c122 AND (c123 AND (c124 AND (c125 AND (c126 AND (c127 AND (c128 AND (c129 AND (c130 AND (c131 AND (c132 AND (c133 AND (c134 AND (c135 AND (c136 AND (c137 AND (c138 AND (c139 AND (c140 AND (c141 AND (c142 AND (c143 AND (c144 AND (c145 AND (c146 AND (c147 AND (c148 AND (c149 AND (c150 AND (c151 AND (c152 AND (c153 AND (c154 AND (c155 AND (c156 AND (c157 AND (c158 AND (c159 AND (c160 AND (c161 AND (c162 AND (c163 AND (c164 AND (c165 AND (c166 AND (c167 AND (c168 AND (c169 AND (c170 AND (c171 AND (c172 AND (c173 AND (c174 AND (c175 AND (c176 AND (c177 AND (c178 AND (c179 AND (c180 AND (c181 AND (c182 AND (c183 AND (c184 AND (c185 AND (c186 AND (c187 AND (c188 AND (c189 AND (c190 AND (c191 AND (c192 AND (c193 AND (c194 AND (c195 AND (c196 AND (c197 AND (c198 AND (c199 AND c200)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))';
   410    410   } {1 {fts5: parser stack overflow}}
   411    411   
   412    412   #-------------------------------------------------------------------------
   413    413   reset_db
   414         -breakpoint
   415    414   do_execsql_test 20.0 {
   416    415     CREATE VIRTUAL TABLE x1 USING fts5(x);
   417    416     INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
   418    417     INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree');
   419    418   }
   420    419   do_test 20.1 {
   421    420     for {set i 1} {$i <= 200} {incr i} {

Changes to ext/fts5/test/fts5simple2.test.

   366    366   do_execsql_test 17.6 { 
   367    367     SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55
   368    368   }
   369    369   
   370    370   #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
   371    371     
   372    372   finish_test
   373         -

Changes to ext/fts5/test/fts5simple3.test.

   112    112   } 
   113    113   do_execsql_test 4.6 {
   114    114     SELECT * FROM t2('ab + xyz');
   115    115   }
   116    116   
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5synonym.test.

   417    417   do_execsql_test 7.1.2 {
   418    418     INSERT INTO t2(t2) VALUES('integrity-check');
   419    419   }
   420    420   
   421    421   } ;# foreach_detail_mode
   422    422   
   423    423   finish_test
   424         -

Changes to ext/fts5/test/fts5synonym2.test.

   157    157   
   158    158   }
   159    159   
   160    160   }
   161    161   }
   162    162   
   163    163   finish_test
   164         -

Changes to ext/fts5/test/fts5tok1.test.

   105    105   do_catchsql_test 2.0 {
   106    106     CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer);
   107    107   } {1 {vtable constructor failed: tX}}
   108    108   
   109    109   do_catchsql_test 2.1 {
   110    110     CREATE VIRTUAL TABLE t4 USING fts5tokenize;
   111    111     SELECT * FROM t4;
   112         -} {1 {SQL logic error or missing database}}
          112  +} {1 {SQL logic error}}
   113    113   
   114    114   
   115    115   finish_test

Changes to ext/fts5/test/fts5tokenizer.test.

   298    298   
   299    299   set ::flags [list]
   300    300   do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
   301    301   do_test 9.5.2 { set ::flags } {query}
   302    302   
   303    303   
   304    304   finish_test
   305         -

Changes to ext/fts5/test/fts5unicode.test.

    46     46     CREATE VIRTUAL TABLE t1 USING fts5(x);
    47     47     CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
    48     48     CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii);
    49     49     INSERT INTO t1 VALUES('\xC0\xC8\xCC');
    50     50     INSERT INTO t2 VALUES('\xC0\xC8\xCC');
    51     51     INSERT INTO t3 VALUES('\xC0\xC8\xCC');
    52     52   "
    53         -breakpoint
    54     53   do_execsql_test 2.1 "
    55     54     SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC';
    56     55     SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC';
    57     56     SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
    58     57   " {t1 t2}
    59     58   
    60     59   
    61     60   finish_test
    62         -

Changes to ext/fts5/test/fts5unicode2.test.

   277    277       INSERT INTO t9(a) VALUES('abc%88def %89ghi%90');
   278    278     }
   279    279   } {0 {}}
   280    280   
   281    281   
   282    282   #-------------------------------------------------------------------------
   283    283   
   284         -breakpoint
   285    284   do_unicode_token_test3 5.1 {tokenchars {}} {
   286    285     sqlite3_reset sqlite3_column_int
   287    286   } {
   288    287     sqlite3 sqlite3 
   289    288     reset reset 
   290    289     sqlite3 sqlite3 
   291    290     column column 

Changes to ext/fts5/test/fts5unicode3.test.

   122    122     }
   123    123     append str {'");}
   124    124     execsql $str
   125    125   } {}
   126    126   
   127    127   
   128    128   finish_test
   129         -

Changes to ext/fts5/test/fts5unindexed.test.

    72     72     INSERT INTO t4(t4, rowid, a, b, c) VALUES('delete', 20, 'j k l', '', 'p q r');
    73     73     DELETE FROM x4 WHERE rowid=20;
    74     74     INSERT INTO t4(t4) VALUES('integrity-check');
    75     75   } {}
    76     76   
    77     77   
    78     78   finish_test
    79         -

Changes to ext/fts5/test/fts5update.test.

   113    113   } {}
   114    114   do_execsql_test 2.2.integrity {
   115    115     INSERT INTO x2(x2) VALUES('integrity-check');
   116    116   }
   117    117   
   118    118   }
   119    119   finish_test
   120         -
   121         -

Changes to ext/fts5/test/fts5version.test.

    57     57     db close
    58     58     sqlite3 db test.db
    59     59     catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
    60     60   } {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
    61     61   
    62     62   
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5vocab.test.

   206    206     INSERT INTO temp.t1 VALUES('1 5 3');
   207    207   
   208    208     INSERT INTO aux.t1 VALUES('x y z');
   209    209     INSERT INTO aux.t1 VALUES('m n o');
   210    210     INSERT INTO aux.t1 VALUES('x n z');
   211    211   }
   212    212   
   213         -breakpoint
   214    213   do_execsql_test 5.1 {
   215    214     CREATE VIRTUAL TABLE temp.vm  USING fts5vocab(main, t1, row);
   216    215     CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row);
   217    216     CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row);
   218    217     CREATE VIRTUAL TABLE temp.va  USING fts5vocab(aux, t1, row);
   219    218   }
   220    219   

Added ext/lsm1/Makefile.

            1  +#
            2  +# This Makefile is designed for use with main.mk in the root directory of
            3  +# this project. After including main.mk, the users makefile should contain:
            4  +#
            5  +#    LSMDIR=$(TOP)/ext/lsm1/
            6  +#    LSMOPTS=-fPIC
            7  +#    include $(LSMDIR)/Makefile
            8  +#
            9  +# The most useful targets are [lsmtest] and [lsm.so].
           10  +#
           11  +
           12  +LSMOBJ    = \
           13  +  lsm_ckpt.o \
           14  +  lsm_file.o \
           15  +  lsm_log.o \
           16  +  lsm_main.o \
           17  +  lsm_mem.o \
           18  +  lsm_mutex.o \
           19  +  lsm_shared.o \
           20  +  lsm_sorted.o \
           21  +  lsm_str.o \
           22  +  lsm_tree.o \
           23  +  lsm_unix.o \
           24  +  lsm_win32.o \
           25  +  lsm_varint.o \
           26  +  lsm_vtab.o
           27  +
           28  +LSMHDR   = \
           29  +  $(LSMDIR)/lsm.h \
           30  +  $(LSMDIR)/lsmInt.h
           31  +
           32  +LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c     \
           33  +             $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c     \
           34  +             $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c     \
           35  +             $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c     \
           36  +             $(LSMDIR)/lsm-test/lsmtest9.c                                   \
           37  +             $(LSMDIR)/lsm-test/lsmtest_datasource.c \
           38  +             $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c  \
           39  +             $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \
           40  +             $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \
           41  +             $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c
           42  +
           43  +
           44  +# all: lsm.so
           45  +
           46  +LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
           47  +
           48  +lsm.so:	$(LSMOBJ)
           49  +	$(TCCX) -shared -o lsm.so $(LSMOBJ)
           50  +
           51  +%.o:	$(LSMDIR)/%.c $(LSMHDR) sqlite3.h
           52  +	$(TCCX) $(LSMOPTS) -c $<
           53  +	
           54  +lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
           55  +	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
           56  +	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)

Added ext/lsm1/Makefile.msc.

            1  +#
            2  +# This Makefile is designed for use with Makefile.msc in the root directory
            3  +# of this project.  The Makefile.msc should contain:
            4  +#
            5  +#    LSMDIR=$(TOP)\ext\lsm1
            6  +#    !INCLUDE $(LSMDIR)\Makefile.msc
            7  +#
            8  +# The most useful targets are [lsmtest.exe] and [lsm.dll].
            9  +#
           10  +
           11  +LSMOBJ    = \
           12  +  lsm_ckpt.lo \
           13  +  lsm_file.lo \
           14  +  lsm_log.lo \
           15  +  lsm_main.lo \
           16  +  lsm_mem.lo \
           17  +  lsm_mutex.lo \
           18  +  lsm_shared.lo \
           19  +  lsm_sorted.lo \
           20  +  lsm_str.lo \
           21  +  lsm_tree.lo \
           22  +  lsm_unix.lo \
           23  +  lsm_win32.lo \
           24  +  lsm_varint.lo \
           25  +  lsm_vtab.lo
           26  +
           27  +LSMHDR   = \
           28  +  $(LSMDIR)\lsm.h \
           29  +  $(LSMDIR)\lsmInt.h
           30  +
           31  +LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c     \
           32  +             $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c     \
           33  +             $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c     \
           34  +             $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c     \
           35  +             $(LSMDIR)\lsm-test\lsmtest9.c                                   \
           36  +             $(LSMDIR)\lsm-test\lsmtest_datasource.c \
           37  +             $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c  \
           38  +             $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \
           39  +             $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \
           40  +             $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c
           41  +
           42  +# all: lsm.dll lsmtest.exe
           43  +
           44  +LSMOPTS = $(NO_WARN) -DLSM_MUTEX_WIN32=1 -I$(LSMDIR)
           45  +
           46  +!IF $(DEBUG)>2
           47  +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1
           48  +!ENDIF
           49  +
           50  +!IF $(MEMDEBUG)!=0
           51  +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1
           52  +!ENDIF
           53  +
           54  +lsm_ckpt.lo:	$(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H)
           55  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c
           56  +
           57  +lsm_file.lo:	$(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H)
           58  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c
           59  +
           60  +lsm_log.lo:	$(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H)
           61  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c
           62  +
           63  +lsm_main.lo:	$(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H)
           64  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c
           65  +
           66  +lsm_mem.lo:	$(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H)
           67  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c
           68  +
           69  +lsm_mutex.lo:	$(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H)
           70  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c
           71  +
           72  +lsm_shared.lo:	$(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H)
           73  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c
           74  +
           75  +lsm_sorted.lo:	$(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H)
           76  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c
           77  +
           78  +lsm_str.lo:	$(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H)
           79  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c
           80  +
           81  +lsm_tree.lo:	$(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H)
           82  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c
           83  +
           84  +lsm_unix.lo:	$(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H)
           85  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c
           86  +
           87  +lsm_win32.lo:	$(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H)
           88  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c
           89  +
           90  +lsm_varint.lo:	$(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H)
           91  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c
           92  +
           93  +lsm_vtab.lo:	$(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H)
           94  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c
           95  +
           96  +lsm.dll:	$(LSMOBJ)
           97  +	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ)
           98  +	copy /Y $@ $(LSMDIR)\$@
           99  +
          100  +lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJ)
          101  +	$(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJ)
          102  +	copy /Y $@ $(LSMDIR)\$@

Added ext/lsm1/lsm-test/README.

            1  +
            2  +
            3  +Organization of test case files:
            4  +
            5  +  lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a 
            6  +              database file, then verify that the contents of the database can
            7  +              be queried.
            8  +
            9  +  lsmtest2.c: Crash tests. Tests that attempt to verify that the database 
           10  +              recovers correctly following an application or system crash.
           11  +
           12  +  lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of
           13  +              transactions and sub-transactions.
           14  +
           15  +  lsmtest4.c: Multi-client tests.
           16  +
           17  +  lsmtest5.c: Multi-client tests with a different thread for each client.
           18  +
           19  +  lsmtest6.c: OOM injection tests.
           20  +
           21  +  lsmtest7.c: API tests.
           22  +
           23  +  lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that
           24  +              the system recovers and other clients proceed unaffected if
           25  +              a process fails in the middle of a write transaction.
           26  +
           27  +              The difference from lsmtest2.c is that this file tests
           28  +              live-recovery (recovery from a failure that occurs while other
           29  +              clients are still running) whereas lsmtest2.c tests recovery
           30  +              from a system or power failure.
           31  +
           32  +  lsmtest9.c: More data tests. These focus on testing that calling
           33  +              lsm_work(nMerge=1) to compact the database does not corrupt it.
           34  +              In other words, that databases containing block-redirects
           35  +              can be read and written.
           36  +
           37  +
           38  +
           39  +
           40  +

Added ext/lsm1/lsm-test/lsmtest.h.

            1  +
            2  +#ifndef __WRAPPER_INT_H_
            3  +#define __WRAPPER_INT_H_
            4  +
            5  +#include "lsmtest_tdb.h"
            6  +#include "sqlite3.h"
            7  +#include "lsm.h"
            8  +
            9  +#include <assert.h>
           10  +#include <stdarg.h>
           11  +#include <stdlib.h>
           12  +#include <string.h>
           13  +#include <stdio.h>
           14  +#ifndef _WIN32
           15  +# include <unistd.h>
           16  +#endif
           17  +#include <sys/types.h>
           18  +#include <sys/stat.h>
           19  +#include <fcntl.h>
           20  +#include <ctype.h>
           21  +#include <stdlib.h>
           22  +#include <errno.h>
           23  +
           24  +#ifdef __cplusplus
           25  +extern "C" {
           26  +#endif
           27  +
           28  +#ifdef _WIN32
           29  +# include "windows.h"
           30  +# define gettimeofday win32GetTimeOfDay
           31  +# define F_OK  (0)
           32  +# define sleep(sec) Sleep(1000 * (sec))
           33  +# define usleep(usec) Sleep(((usec) + 999) / 1000)
           34  +# ifdef _MSC_VER
           35  +#  include <io.h>
           36  +#  define snprintf _snprintf
           37  +#  define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
           38  +#  define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
           39  +#  define __va_copy(dst,src) ((dst) = (src))
           40  +#  define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1)
           41  +# else
           42  +#  error Unsupported C compiler for Windows.
           43  +# endif
           44  +int win32GetTimeOfDay(struct timeval *, void *);
           45  +#endif
           46  +
           47  +#ifndef _LSM_INT_H
           48  +typedef unsigned int  u32;
           49  +typedef unsigned char u8;
           50  +typedef long long int i64;
           51  +typedef unsigned long long int u64;
           52  +#endif
           53  +
           54  +
           55  +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0])))
           56  +
           57  +#define MIN(x,y) ((x)<(y) ? (x) : (y))
           58  +#define MAX(x,y) ((x)>(y) ? (x) : (y))
           59  +
           60  +#define unused_parameter(x) (void)(x)
           61  +
           62  +#define TESTDB_DEFAULT_PAGE_SIZE   4096
           63  +#define TESTDB_DEFAULT_CACHE_SIZE  2048
           64  +
           65  +#ifndef _O_BINARY
           66  +# define _O_BINARY (0)
           67  +#endif
           68  +
           69  +/*
           70  +** Ideally, these should be in wrapper.c. But they are here instead so that 
           71  +** they can be used by the C++ database wrappers in wrapper2.cc.
           72  +*/
           73  +typedef struct DatabaseMethods DatabaseMethods;
           74  +struct TestDb {
           75  +  DatabaseMethods const *pMethods;          /* Database methods */
           76  +  const char *zLibrary;                     /* Library name for tdb_open() */
           77  +};
           78  +struct DatabaseMethods {
           79  +  int (*xClose)(TestDb *);
           80  +  int (*xWrite)(TestDb *, void *, int , void *, int);
           81  +  int (*xDelete)(TestDb *, void *, int);
           82  +  int (*xDeleteRange)(TestDb *, void *, int, void *, int);
           83  +  int (*xFetch)(TestDb *, void *, int, void **, int *);
           84  +  int (*xScan)(TestDb *, void *, int, void *, int, void *, int,
           85  +    void (*)(void *, void *, int , void *, int)
           86  +  );
           87  +  int (*xBegin)(TestDb *, int);
           88  +  int (*xCommit)(TestDb *, int);
           89  +  int (*xRollback)(TestDb *, int);
           90  +};
           91  +
           92  +/* 
           93  +** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the
           94  +** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but
           95  +** the primary interface is the C++ API.
           96  +*/
           97  +int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb);
           98  +int test_kc_close(TestDb *);
           99  +int test_kc_write(TestDb *, void *, int , void *, int);
          100  +int test_kc_delete(TestDb *, void *, int);
          101  +int test_kc_delete_range(TestDb *, void *, int, void *, int);
          102  +int test_kc_fetch(TestDb *, void *, int, void **, int *);
          103  +int test_kc_scan(TestDb *, void *, int, void *, int, void *, int,
          104  +  void (*)(void *, void *, int , void *, int)
          105  +);
          106  +
          107  +int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          108  +int test_mdb_close(TestDb *);
          109  +int test_mdb_write(TestDb *, void *, int , void *, int);
          110  +int test_mdb_delete(TestDb *, void *, int);
          111  +int test_mdb_fetch(TestDb *, void *, int, void **, int *);
          112  +int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
          113  +  void (*)(void *, void *, int , void *, int)
          114  +);
          115  +
          116  +/* 
          117  +** Functions in wrapper3.c. This file contains the tdb wrapper for lsm.
          118  +** The wrapper for lsm is a bit more involved than the others, as it 
          119  +** includes code for a couple of different lsm configurations, and for
          120  +** various types of fault injection and robustness testing.
          121  +*/
          122  +int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          123  +int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
          124  +int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
          125  +int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
          126  +int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
          127  +int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
          128  +
          129  +int tdb_lsm_configure(lsm_db *, const char *);
          130  +
          131  +/* Functions in lsmtest_tdb4.c */
          132  +int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          133  +int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          134  +int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          135  +
          136  +
          137  +/* Functions in testutil.c. */
          138  +int  testPrngInit(void);
          139  +u32  testPrngValue(u32 iVal);
          140  +void testPrngArray(u32 iVal, u32 *aOut, int nOut);
          141  +void testPrngString(u32 iVal, char *aOut, int nOut);
          142  +
          143  +void testErrorInit(int argc, char **);
          144  +void testPrintError(const char *zFormat, ...);
          145  +void testPrintUsage(const char *zArgs);
          146  +void testPrintFUsage(const char *zFormat, ...);
          147  +void testTimeInit(void);
          148  +int  testTimeGet(void);
          149  +
          150  +/* Functions in testmem.c. */
          151  +void testMallocInstall(lsm_env *pEnv);
          152  +void testMallocUninstall(lsm_env *pEnv);
          153  +void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *);
          154  +void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *);
          155  +void testMallocOomEnable(lsm_env *pEnv, int);
          156  +
          157  +/* lsmtest.c */
          158  +TestDb *testOpen(const char *zSystem, int, int *pRc);
          159  +void testReopen(TestDb **ppDb, int *pRc);
          160  +void testClose(TestDb **ppDb);
          161  +
          162  +void testFetch(TestDb *, void *, int, void *, int, int *);
          163  +void testWrite(TestDb *, void *, int, void *, int, int *);
          164  +void testDelete(TestDb *, void *, int, int *);
          165  +void testDeleteRange(TestDb *, void *, int, void *, int, int *);
          166  +void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc);
          167  +void testFetchStr(TestDb *, const char *, const char *, int *pRc);
          168  +
          169  +void testBegin(TestDb *pDb, int iTrans, int *pRc);
          170  +void testCommit(TestDb *pDb, int iTrans, int *pRc);
          171  +
          172  +void test_failed(void);
          173  +
          174  +char *testMallocPrintf(const char *zFormat, ...);
          175  +char *testMallocVPrintf(const char *zFormat, va_list ap);
          176  +int testGlobMatch(const char *zPattern, const char *zStr);
          177  +
          178  +void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *);
          179  +void testFetchCompare(TestDb *, TestDb *, void *, int, int *);
          180  +
          181  +void *testMalloc(int);
          182  +void *testMallocCopy(void *pCopy, int nByte);
          183  +void *testRealloc(void *, int);
          184  +void testFree(void *);
          185  +
          186  +/* lsmtest_bt.c */
          187  +int do_bt(int nArg, char **azArg);
          188  +
          189  +/* testio.c */
          190  +int testVfsConfigureDb(TestDb *pDb);
          191  +
          192  +/* testfunc.c */
          193  +int do_show(int nArg, char **azArg);
          194  +int do_work(int nArg, char **azArg);
          195  +
          196  +/* testio.c */
          197  +int do_io(int nArg, char **azArg);
          198  +
          199  +/* lsmtest2.c */
          200  +void do_crash_test(const char *zPattern, int *pRc);
          201  +int do_rollback_test(int nArg, char **azArg);
          202  +
          203  +/* test3.c */
          204  +void test_rollback(const char *zSystem, const char *zPattern, int *pRc);
          205  +
          206  +/* test4.c */
          207  +void test_mc(const char *zSystem, const char *zPattern, int *pRc);
          208  +
          209  +/* test5.c */
          210  +void test_mt(const char *zSystem, const char *zPattern, int *pRc);
          211  +
          212  +/* lsmtest6.c */
          213  +void test_oom(const char *zPattern, int *pRc);
          214  +void testDeleteLsmdb(const char *zFile);
          215  +
          216  +void testSaveDb(const char *zFile, const char *zAuxExt);
          217  +void testRestoreDb(const char *zFile, const char *zAuxExt);
          218  +void testCopyLsmdb(const char *zFrom, const char *zTo);
          219  +
          220  +/* lsmtest7.c */
          221  +void test_api(const char *zPattern, int *pRc);
          222  +
          223  +/* lsmtest8.c */
          224  +void do_writer_crash_test(const char *zPattern, int *pRc);
          225  +
          226  +/*************************************************************************
          227  +** Interface to functionality in test_datasource.c.
          228  +*/
          229  +typedef struct Datasource Datasource;
          230  +typedef struct DatasourceDefn DatasourceDefn;
          231  +
          232  +struct DatasourceDefn {
          233  +  int eType;                      /* A TEST_DATASOURCE_* value */
          234  +  int nMinKey;                    /* Minimum key size */
          235  +  int nMaxKey;                    /* Maximum key size */
          236  +  int nMinVal;                    /* Minimum value size */
          237  +  int nMaxVal;                    /* Maximum value size */
          238  +};
          239  +
          240  +#define TEST_DATASOURCE_RANDOM    1
          241  +#define TEST_DATASOURCE_SEQUENCE  2
          242  +
          243  +char *testDatasourceName(const DatasourceDefn *);
          244  +Datasource *testDatasourceNew(const DatasourceDefn *);
          245  +void testDatasourceFree(Datasource *);
          246  +void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *);
          247  +/* End of test_datasource.c interface.
          248  +*************************************************************************/
          249  +void testDatasourceFetch(
          250  +  TestDb *pDb,                    /* Database handle */
          251  +  Datasource *pData,
          252  +  int iKey,
          253  +  int *pRc                        /* IN/OUT: Error code */
          254  +);
          255  +
          256  +void testWriteDatasource(TestDb *, Datasource *, int, int *);
          257  +void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *);
          258  +void testDeleteDatasource(TestDb *, Datasource *, int, int *);
          259  +void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *);
          260  +
          261  +
          262  +/* test1.c */
          263  +void test_data_1(const char *, const char *, int *pRc);
          264  +void test_data_2(const char *, const char *, int *pRc);
          265  +void test_data_3(const char *, const char *, int *pRc);
          266  +void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *);
          267  +void testCaseProgress(int, int, int, int *);
          268  +int testCaseNDot(void);
          269  +
          270  +void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *);
          271  +int testControlDb(TestDb **ppDb);
          272  +
          273  +typedef struct CksumDb CksumDb;
          274  +CksumDb *testCksumArrayNew(Datasource *, int, int, int);
          275  +char *testCksumArrayGet(CksumDb *, int);
          276  +void testCksumArrayFree(CksumDb *);
          277  +void testCaseStart(int *pRc, char *zFmt, ...);
          278  +void testCaseFinish(int rc);
          279  +void testCaseSkip(void);
          280  +int testCaseBegin(int *, const char *, const char *, ...);
          281  +
          282  +#define TEST_CKSUM_BYTES 29
          283  +int testCksumDatabase(TestDb *pDb, char *zOut);
          284  +int testCountDatabase(TestDb *pDb);
          285  +void testCompareInt(int, int, int *);
          286  +void testCompareStr(const char *z1, const char *z2, int *pRc);
          287  +
          288  +/* lsmtest9.c */
          289  +void test_data_4(const char *, const char *, int *pRc);
          290  +
          291  +
          292  +/*
          293  +** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function.
          294  +*/
          295  +#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z)
          296  +int testArgSelectX(void *, const char *, int, const char *, int *);
          297  +
          298  +#ifdef __cplusplus
          299  +}  /* End of the 'extern "C"' block */
          300  +#endif
          301  +
          302  +#endif

Added ext/lsm1/lsm-test/lsmtest1.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE
            5  +#define DATA_RANDOM     TEST_DATASOURCE_RANDOM
            6  +
            7  +typedef struct Datatest1 Datatest1;
            8  +typedef struct Datatest2 Datatest2;
            9  +
           10  +/*
           11  +** An instance of the following structure contains parameters used to
           12  +** customize the test function in this file. Test procedure:
           13  +**
           14  +**   1. Create a data-source based on the "datasource definition" vars.
           15  +**
           16  +**   2. Insert nRow key value pairs into the database.
           17  +**
           18  +**   3. Delete all keys from the database. Deletes are done in the same 
           19  +**      order as the inserts.
           20  +**
           21  +** During steps 2 and 3 above, after each Datatest1.nVerify inserts or
           22  +** deletes, the following:
           23  +**
           24  +**   a. Run Datasource.nTest key lookups and check the results are as expected.
           25  +**
           26  +**   b. If Datasource.bTestScan is true, run a handful (8) of range
           27  +**      queries (scanning forwards and backwards). Check that the results
           28  +**      are as expected.
           29  +**
           30  +**   c. Close and reopen the database. Then run (a) and (b) again.
           31  +*/
           32  +struct Datatest1 {
           33  +  /* Datasource definition */
           34  +  DatasourceDefn defn;
           35  +
           36  +  /* Test procedure parameters */
           37  +  int nRow;                       /* Number of rows to insert then delete */
           38  +  int nVerify;                    /* How often to verify the db contents */
           39  +  int nTest;                      /* Number of keys to test (0==all) */
           40  +  int bTestScan;                  /* True to do scan tests */
           41  +};
           42  +
           43  +/*
           44  +** An instance of the following data structure is used to describe the
           45  +** second type of test case in this file. The chief difference between 
           46  +** these tests and those described by Datatest1 is that these tests also
           47  +** experiment with range-delete operations. Tests proceed as follows:
           48  +**
           49  +**     1. Open the datasource described by Datatest2.defn. 
           50  +**
           51  +**     2. Open a connection on an empty database.
           52  +**
           53  +**     3. Do this Datatest2.nIter times:
           54  +**
           55  +**        a) Insert Datatest2.nWrite key-value pairs from the datasource.
           56  +**
           57  +**        b) Select two pseudo-random keys and use them as the start
           58  +**           and end points of a range-delete operation.
           59  +**
           60  +**        c) Verify that the contents of the database are as expected (see
           61  +**           below for details).
           62  +**
           63  +**        d) Close and then reopen the database handle.
           64  +**
           65  +**        e) Verify that the contents of the database are still as expected.
           66  +**
           67  +** The inserts and range deletes are run twice - once on the database being
           68  +** tested and once using a control system (sqlite3, kc etc. - something that 
           69  +** works). In order to verify that the contents of the db being tested are
           70  +** correct, the test runs a bunch of scans and lookups on both the test and
           71  +** control databases. If the results are the same, the test passes.
           72  +*/
           73  +struct Datatest2 {
           74  +  DatasourceDefn defn;
           75  +  int nRange;
           76  +  int nWrite;                     /* Number of writes per iteration */
           77  +  int nIter;                      /* Total number of iterations to run */
           78  +};
           79  +
           80  +/*
           81  +** Generate a unique name for the test case pTest with database system
           82  +** zSystem.
           83  +*/
           84  +static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){
           85  +  char *zRet;
           86  +  char *zData;
           87  +  zData = testDatasourceName(&pTest->defn);
           88  +  zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", 
           89  +      zSystem, zData, bRecover, pTest->nRow, pTest->nVerify
           90  +  );
           91  +  testFree(zData);
           92  +  return zRet;
           93  +}
           94  +
           95  +int testControlDb(TestDb **ppDb){
           96  +#ifdef HAVE_KYOTOCABINET
           97  +  return tdb_open("kyotocabinet", "tmp.db", 1, ppDb);
           98  +#else
           99  +  return tdb_open("sqlite3", "", 1, ppDb);
          100  +#endif
          101  +}
          102  +
          103  +void testDatasourceFetch(
          104  +  TestDb *pDb,                    /* Database handle */
          105  +  Datasource *pData,
          106  +  int iKey,
          107  +  int *pRc                        /* IN/OUT: Error code */
          108  +){
          109  +  void *pKey; int nKey;           /* Database key to query for */
          110  +  void *pVal; int nVal;           /* Expected result of query */
          111  +
          112  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          113  +  testFetch(pDb, pKey, nKey, pVal, nVal, pRc);
          114  +}
          115  +
          116  +/*
          117  +** This function is called to test that the contents of database pDb
          118  +** are as expected. In this case, expected is defined as containing
          119  +** key-value pairs iFirst through iLast, inclusive, from data source 
          120  +** pData. In other words, a loop like the following could be used to
          121  +** construct a database with identical contents from scratch.
          122  +**
          123  +**   for(i=iFirst; i<=iLast; i++){
          124  +**     testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
          125  +**     // insert (pKey, nKey) -> (pVal, nVal) into database
          126  +**   }
          127  +**
          128  +** The key domain consists of keys 0 to (nRow-1), inclusive, from
          129  +** data source pData. For both scan and lookup tests, keys are selected
          130  +** pseudo-randomly from within this set.
          131  +**
          132  +** This function runs nLookupTest lookup tests and nScanTest scan tests.
          133  +**
          134  +** A lookup test consists of selecting a key from the domain and querying
          135  +** pDb for it. The test fails if the presence of the key and, if present,
          136  +** the associated value do not match the expectations defined above.
          137  +**
          138  +** A scan test involves selecting a key from the domain and running
          139  +** the following queries:
          140  +**
          141  +**   1. Scan all keys equal to or greater than the key, in ascending order.
          142  +**   2. Scan all keys equal to or smaller than the key, in descending order.
          143  +**
          144  +** Additionally, if nLookupTest is greater than zero, the following are
          145  +** run once:
          146  +**
          147  +**   1. Scan all keys in the db, in ascending order.
          148  +**   2. Scan all keys in the db, in descending order.
          149  +**
          150  +** As you would assume, the test fails if the returned values do not match
          151  +** expectations.
          152  +*/
          153  +void testDbContents(
          154  +  TestDb *pDb,                    /* Database handle being tested */
          155  +  Datasource *pData,              /* pDb contains data from here */
          156  +  int nRow,                       /* Size of key domain */
          157  +  int iFirst,                     /* Index of first key from pData in pDb */
          158  +  int iLast,                      /* Index of last key from pData in pDb */
          159  +  int nLookupTest,                /* Number of lookup tests to run */
          160  +  int nScanTest,                  /* Number of scan tests to run */
          161  +  int *pRc                        /* IN/OUT: Error code */
          162  +){
          163  +  int j;
          164  +  int rc = *pRc;
          165  +
          166  +  if( rc==0 && nScanTest ){
          167  +    TestDb *pDb2 = 0;
          168  +
          169  +    /* Open a control db (i.e. one that we assume works) */
          170  +    rc = testControlDb(&pDb2);
          171  +
          172  +    for(j=iFirst; rc==0 && j<=iLast; j++){
          173  +      void *pKey; int nKey;         /* Database key to insert */
          174  +      void *pVal; int nVal;         /* Database value to insert */
          175  +      testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal);
          176  +      rc = tdb_write(pDb2, pKey, nKey, pVal, nVal);
          177  +    }
          178  +
          179  +    if( rc==0 ){
          180  +      int iKey1;
          181  +      int iKey2;
          182  +      void *pKey1; int nKey1;       /* Start key */
          183  +      void *pKey2; int nKey2;       /* Final key */
          184  +
          185  +      iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow;
          186  +      iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow;
          187  +      testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
          188  +      pKey1 = testMalloc(nKey1+1);
          189  +      memcpy(pKey1, pKey2, nKey1+1);
          190  +      testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
          191  +
          192  +      testScanCompare(pDb2, pDb, 0, 0, 0,         0, 0,         &rc);
          193  +      testScanCompare(pDb2, pDb, 0, 0, 0,         pKey2, nKey2, &rc);
          194  +      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0,         &rc);
          195  +      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc);
          196  +      testScanCompare(pDb2, pDb, 1, 0, 0,         0, 0,         &rc);
          197  +      testScanCompare(pDb2, pDb, 1, 0, 0,         pKey2, nKey2, &rc);
          198  +      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0,         &rc);
          199  +      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc);
          200  +      testFree(pKey1);
          201  +    }
          202  +    tdb_close(pDb2);
          203  +  }
          204  +
          205  +  /* Test some lookups. */
          206  +  for(j=0; rc==0 && j<nLookupTest; j++){
          207  +    int iKey;                     /* Datasource key to test */
          208  +    void *pKey; int nKey;         /* Database key to query for */
          209  +    void *pVal; int nVal;         /* Expected result of query */
          210  +
          211  +    if( nLookupTest>=nRow ){
          212  +      iKey = j;
          213  +    }else{
          214  +      iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow;
          215  +    }
          216  +
          217  +    testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          218  +    if( iFirst>iKey || iKey>iLast ){
          219  +      pVal = 0;
          220  +      nVal = -1;
          221  +    }
          222  +
          223  +    testFetch(pDb, pKey, nKey, pVal, nVal, &rc);
          224  +  }
          225  +
          226  +  *pRc = rc;
          227  +}
          228  +
          229  +/*
          230  +** This function should be called during long running test cases to output
          231  +** the progress dots (...) to stdout.
          232  +*/
          233  +void testCaseProgress(int i, int n, int nDot, int *piDot){
          234  +  int iDot = *piDot;
          235  +  while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){
          236  +    printf(".");
          237  +    fflush(stdout);
          238  +    iDot++;
          239  +  }
          240  +  *piDot = iDot;
          241  +}
          242  +
          243  +int testCaseNDot(void){ return 20; }
          244  +
          245  +#if 0
          246  +static void printScanCb(
          247  +    void *pCtx, void *pKey, int nKey, void *pVal, int nVal
          248  +){
          249  +  printf("%s\n", (char *)pKey);
          250  +  fflush(stdout);
          251  +}
          252  +#endif
          253  +
          254  +void testReopenRecover(TestDb **ppDb, int *pRc){
          255  +  if( *pRc==0 ){
          256  +    const char *zLib = tdb_library_name(*ppDb);
          257  +    const char *zDflt = tdb_default_db(zLib);
          258  +    testCopyLsmdb(zDflt, "bak.db");
          259  +    testClose(ppDb);
          260  +    testCopyLsmdb("bak.db", zDflt);
          261  +    *pRc = tdb_open(zLib, 0, 0, ppDb);
          262  +  }
          263  +}
          264  +
          265  +
          266  +static void doDataTest1(
          267  +  const char *zSystem,            /* Database system to test */
          268  +  int bRecover,
          269  +  Datatest1 *p,                   /* Structure containing test parameters */
          270  +  int *pRc                        /* OUT: Error code */
          271  +){
          272  +  int i;
          273  +  int iDot;
          274  +  int rc = LSM_OK;
          275  +  Datasource *pData;
          276  +  TestDb *pDb;
          277  +
          278  +  /* Start the test case, open a database and allocate the datasource. */
          279  +  pDb = testOpen(zSystem, 1, &rc);
          280  +  pData = testDatasourceNew(&p->defn);
          281  +
          282  +  i = 0;
          283  +  iDot = 0;
          284  +  while( rc==LSM_OK && i<p->nRow ){
          285  +
          286  +    /* Insert some data */
          287  +    testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
          288  +    i += p->nVerify;
          289  +
          290  +    /* Check that the db content is correct. */
          291  +    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
          292  +
          293  +    if( bRecover ){
          294  +      testReopenRecover(&pDb, &rc);
          295  +    }else{
          296  +      testReopen(&pDb, &rc);
          297  +    }
          298  +
          299  +    /* Check that the db content is still correct. */
          300  +    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
          301  +
          302  +    /* Update the progress dots... */
          303  +    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
          304  +  }
          305  +
          306  +  i = 0;
          307  +  iDot = 0;
          308  +  while( rc==LSM_OK && i<p->nRow ){
          309  +
          310  +    /* Delete some entries */
          311  +    testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
          312  +    i += p->nVerify;
          313  +
          314  +    /* Check that the db content is correct. */
          315  +    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
          316  +
          317  +    /* Close and reopen the database. */
          318  +    if( bRecover ){
          319  +      testReopenRecover(&pDb, &rc);
          320  +    }else{
          321  +      testReopen(&pDb, &rc);
          322  +    }
          323  +
          324  +    /* Check that the db content is still correct. */
          325  +    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
          326  +
          327  +    /* Update the progress dots... */
          328  +    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
          329  +  }
          330  +
          331  +  /* Free the datasource, close the database and finish the test case. */
          332  +  testDatasourceFree(pData);
          333  +  tdb_close(pDb);
          334  +  testCaseFinish(rc);
          335  +  *pRc = rc;
          336  +}
          337  +
          338  +
          339  +void test_data_1(
          340  +  const char *zSystem,            /* Database system name */
          341  +  const char *zPattern,           /* Run test cases that match this pattern */
          342  +  int *pRc                        /* IN/OUT: Error code */
          343  +){
          344  +  Datatest1 aTest[] = {
          345  +    { {DATA_RANDOM,     500,600,   1000,2000},     1000,  100,  10,  0},
          346  +    { {DATA_RANDOM,     20,25,     100,200},       1000,  250, 1000, 1},
          347  +    { {DATA_RANDOM,     8,10,      100,200},       1000,  250, 1000, 1},
          348  +    { {DATA_RANDOM,     8,10,      10,20},         1000,  250, 1000, 1},
          349  +    { {DATA_RANDOM,     8,10,      1000,2000},     1000,  250, 1000, 1},
          350  +    { {DATA_RANDOM,     8,100,     10000,20000},    100,   25,  100, 1},
          351  +    { {DATA_RANDOM,     80,100,    10,20},         1000,  250, 1000, 1},
          352  +    { {DATA_RANDOM,     5000,6000, 10,20},          100,   25,  100, 1},
          353  +    { {DATA_SEQUENTIAL, 5,10,      10,20},         1000,  250, 1000, 1},
          354  +    { {DATA_SEQUENTIAL, 5,10,      100,200},       1000,  250, 1000, 1},
          355  +    { {DATA_SEQUENTIAL, 5,10,      1000,2000},     1000,  250, 1000, 1},
          356  +    { {DATA_SEQUENTIAL, 5,100,     10000,20000},    100,   25,  100, 1},
          357  +    { {DATA_RANDOM,     10,10,     100,100},     100000, 1000,  100, 0},
          358  +    { {DATA_SEQUENTIAL, 10,10,     100,100},     100000, 1000,  100, 0},
          359  +  };
          360  +
          361  +  int i;
          362  +  int bRecover;
          363  +
          364  +  for(bRecover=0; bRecover<2; bRecover++){
          365  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          366  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          367  +      char *zName = getName(zSystem, bRecover, &aTest[i]);
          368  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          369  +        doDataTest1(zSystem, bRecover, &aTest[i], pRc);
          370  +      }
          371  +      testFree(zName);
          372  +    }
          373  +  }
          374  +}
          375  +
          376  +void testCompareDb(
          377  +  Datasource *pData,
          378  +  int nData,
          379  +  int iSeed,
          380  +  TestDb *pControl,
          381  +  TestDb *pDb,
          382  +  int *pRc
          383  +){
          384  +  int i;
          385  +
          386  +  static int nCall = 0;
          387  +  nCall++;
          388  +
          389  +  testScanCompare(pControl, pDb, 0, 0, 0,         0, 0,         pRc);
          390  +  testScanCompare(pControl, pDb, 1, 0, 0,         0, 0,         pRc);
          391  +
          392  +  if( *pRc==0 ){
          393  +    int iKey1;
          394  +    int iKey2;
          395  +    void *pKey1; int nKey1;       /* Start key */
          396  +    void *pKey2; int nKey2;       /* Final key */
          397  +
          398  +    iKey1 = testPrngValue(iSeed) % nData;
          399  +    iKey2 = testPrngValue(iSeed+1) % nData;
          400  +    testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
          401  +    pKey1 = testMalloc(nKey1+1);
          402  +    memcpy(pKey1, pKey2, nKey1+1);
          403  +    testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
          404  +
          405  +    testScanCompare(pControl, pDb, 0, 0, 0,         pKey2, nKey2, pRc);
          406  +    testScanCompare(pControl, pDb, 0, pKey1, nKey1, 0, 0,         pRc);
          407  +    testScanCompare(pControl, pDb, 0, pKey1, nKey1, pKey2, nKey2, pRc);
          408  +    testScanCompare(pControl, pDb, 1, 0, 0,         pKey2, nKey2, pRc);
          409  +    testScanCompare(pControl, pDb, 1, pKey1, nKey1, 0, 0,         pRc);
          410  +    testScanCompare(pControl, pDb, 1, pKey1, nKey1, pKey2, nKey2, pRc);
          411  +    testFree(pKey1);
          412  +  }
          413  +
          414  +  for(i=0; i<nData && *pRc==0; i++){
          415  +    void *pKey; int nKey;
          416  +    testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
          417  +    testFetchCompare(pControl, pDb, pKey, nKey, pRc);
          418  +  }
          419  +}
          420  +
          421  +static void doDataTest2(
          422  +  const char *zSystem,            /* Database system to test */
          423  +  int bRecover,
          424  +  Datatest2 *p,                   /* Structure containing test parameters */
          425  +  int *pRc                        /* OUT: Error code */
          426  +){
          427  +  TestDb *pDb;
          428  +  TestDb *pControl;
          429  +  Datasource *pData;
          430  +  int i;
          431  +  int rc = LSM_OK;
          432  +  int iDot = 0;
          433  +
          434  +  /* Start the test case, open a database and allocate the datasource. */
          435  +  pDb = testOpen(zSystem, 1, &rc);
          436  +  pData = testDatasourceNew(&p->defn);
          437  +  rc = testControlDb(&pControl);
          438  +
          439  +  if( tdb_lsm(pDb) ){
          440  +    int nBuf = 32 * 1024 * 1024;
          441  +    lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf);
          442  +  }
          443  +
          444  +  for(i=0; rc==0 && i<p->nIter; i++){
          445  +    void *pKey1; int nKey1;
          446  +    void *pKey2; int nKey2;
          447  +    int ii;
          448  +    int nRange = MIN(p->nIter*p->nWrite, p->nRange);
          449  +
          450  +    for(ii=0; rc==0 && ii<p->nWrite; ii++){
          451  +      int iKey = (i*p->nWrite + ii) % p->nRange;
          452  +      testWriteDatasource(pControl, pData, iKey, &rc);
          453  +      testWriteDatasource(pDb, pData, iKey, &rc);
          454  +    }
          455  +
          456  +    testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0);
          457  +    pKey1 = testMallocCopy(pKey1, nKey1);
          458  +    testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0);
          459  +
          460  +    testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc);
          461  +    testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc);
          462  +    testFree(pKey1);
          463  +
          464  +    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
          465  +    if( bRecover ){
          466  +      testReopenRecover(&pDb, &rc);
          467  +    }else{
          468  +      testReopen(&pDb, &rc);
          469  +    }
          470  +    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
          471  +
          472  +    /* Update the progress dots... */
          473  +    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
          474  +  }
          475  +
          476  +  testClose(&pDb);
          477  +  testClose(&pControl);
          478  +  testDatasourceFree(pData);
          479  +  testCaseFinish(rc);
          480  +  *pRc = rc;
          481  +}
          482  +
          483  +static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){
          484  +  char *zRet;
          485  +  char *zData;
          486  +  zData = testDatasourceName(&pTest->defn);
          487  +  zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", 
          488  +      zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter
          489  +  );
          490  +  testFree(zData);
          491  +  return zRet;
          492  +}
          493  +
          494  +void test_data_2(
          495  +  const char *zSystem,            /* Database system name */
          496  +  const char *zPattern,           /* Run test cases that match this pattern */
          497  +  int *pRc                        /* IN/OUT: Error code */
          498  +){
          499  +  Datatest2 aTest[] = {
          500  +      /* defn,                                 nRange, nWrite, nIter */
          501  +    { {DATA_RANDOM,     20,25,     100,200},   10000,  10,     50   },
          502  +    { {DATA_RANDOM,     20,25,     100,200},   10000,  200,    50   },
          503  +    { {DATA_RANDOM,     20,25,     100,200},   100,    10,     1000 },
          504  +    { {DATA_RANDOM,     20,25,     100,200},   100,    200,    50   },
          505  +  };
          506  +
          507  +  int i;
          508  +  int bRecover;
          509  +
          510  +  for(bRecover=0; bRecover<2; bRecover++){
          511  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          512  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          513  +      char *zName = getName2(zSystem, bRecover, &aTest[i]);
          514  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          515  +        doDataTest2(zSystem, bRecover, &aTest[i], pRc);
          516  +      }
          517  +      testFree(zName);
          518  +    }
          519  +  }
          520  +}
          521  +
          522  +/*************************************************************************
          523  +** Test case data3.*
          524  +*/
          525  +
          526  +typedef struct Datatest3 Datatest3;
          527  +struct Datatest3 {
          528  +  int nRange;                     /* Keys are between 1 and this value, incl. */
          529  +  int nIter;                      /* Number of iterations */
          530  +  int nWrite;                     /* Number of writes per iteration */
          531  +  int nDelete;                    /* Number of deletes per iteration */
          532  +
          533  +  int nValMin;                    /* Minimum value size for writes */
          534  +  int nValMax;                    /* Maximum value size for writes */
          535  +};
          536  +
          537  +void testPutU32(u8 *aBuf, u32 iVal){
          538  +  aBuf[0] = (iVal >> 24) & 0xFF;
          539  +  aBuf[1] = (iVal >> 16) & 0xFF;
          540  +  aBuf[2] = (iVal >>  8) & 0xFF;
          541  +  aBuf[3] = (iVal >>  0) & 0xFF;
          542  +}
          543  +
          544  +void dt3PutKey(u8 *aBuf, int iKey){
          545  +  assert( iKey<100000 && iKey>=0 );
          546  +  sprintf((char *)aBuf, "%.5d", iKey);
          547  +}
          548  +
          549  +static void doDataTest3(
          550  +  const char *zSystem,            /* Database system to test */
          551  +  Datatest3 *p,                   /* Structure containing test parameters */
          552  +  int *pRc                        /* OUT: Error code */
          553  +){
          554  +  int iDot = 0;
          555  +  int rc = *pRc;
          556  +  TestDb *pDb;
          557  +  u8 *abPresent;                  /* Array of boolean */
          558  +  char *aVal;                     /* Buffer to hold values */
          559  +  int i;
          560  +  u32 iSeq = 10;                  /* prng counter */
          561  +
          562  +  abPresent = (u8 *)testMalloc(p->nRange+1);
          563  +  aVal = (char *)testMalloc(p->nValMax+1);
          564  +  pDb = testOpen(zSystem, 1, &rc);
          565  +
          566  +  for(i=0; i<p->nIter && rc==0; i++){
          567  +    int ii;
          568  +
          569  +    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
          570  +
          571  +    /* Perform nWrite inserts */
          572  +    for(ii=0; ii<p->nWrite; ii++){
          573  +      u8 aKey[6];
          574  +      u32 iKey;
          575  +      int nVal;
          576  +
          577  +      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
          578  +      nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin;
          579  +      testPrngString(testPrngValue(iSeq++), aVal, nVal);
          580  +      dt3PutKey(aKey, iKey);
          581  +
          582  +      testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc);
          583  +      abPresent[iKey] = 1;
          584  +    }
          585  +
          586  +    /* Perform nDelete deletes */
          587  +    for(ii=0; ii<p->nDelete; ii++){
          588  +      u8 aKey1[6];
          589  +      u8 aKey2[6];
          590  +      u32 iKey;
          591  +
          592  +      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
          593  +      dt3PutKey(aKey1, iKey-1);
          594  +      dt3PutKey(aKey2, iKey+1);
          595  +
          596  +      testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc);
          597  +      abPresent[iKey] = 0;
          598  +    }
          599  +
          600  +    testReopen(&pDb, &rc);
          601  +
          602  +    for(ii=1; rc==0 && ii<=p->nRange; ii++){
          603  +      int nDbVal;
          604  +      void *pDbVal;
          605  +      u8 aKey[6];
          606  +      int dbrc;
          607  +
          608  +      dt3PutKey(aKey, ii);
          609  +      dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal);
          610  +      testCompareInt(0, dbrc, &rc);
          611  +
          612  +      if( abPresent[ii] ){
          613  +        testCompareInt(1, (nDbVal>0), &rc);
          614  +      }else{
          615  +        testCompareInt(1, (nDbVal<0), &rc);
          616  +      }
          617  +    }
          618  +  }
          619  +
          620  +  testClose(&pDb);
          621  +  testCaseFinish(rc);
          622  +  *pRc = rc;
          623  +}
          624  +
          625  +static char *getName3(const char *zSystem, Datatest3 *p){
          626  +  return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)",
          627  +      zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, 
          628  +      p->nValMin, p->nValMax
          629  +  );
          630  +}
          631  +
          632  +void test_data_3(
          633  +  const char *zSystem,            /* Database system name */
          634  +  const char *zPattern,           /* Run test cases that match this pattern */
          635  +  int *pRc                        /* IN/OUT: Error code */
          636  +){
          637  +  Datatest3 aTest[] = {
          638  +    /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */
          639  +    {  100,    1000,  5,      5,       50,      100 },
          640  +    {  100,    1000,  2,      2,        5,       10 },
          641  +  };
          642  +
          643  +  int i;
          644  +
          645  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          646  +    char *zName = getName3(zSystem, &aTest[i]);
          647  +    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          648  +      doDataTest3(zSystem, &aTest[i], pRc);
          649  +    }
          650  +    testFree(zName);
          651  +  }
          652  +}
          653  +
          654  +

Added ext/lsm1/lsm-test/lsmtest2.c.

            1  +
            2  +/*
            3  +** This file contains tests related to recovery following application 
            4  +** and system crashes (power failures) while writing to the database.
            5  +*/
            6  +
            7  +#include "lsmtest.h"
            8  +
            9  +/*
           10  +** Structure used by testCksumDatabase() to accumulate checksum values in.
           11  +*/
           12  +typedef struct Cksum Cksum;
           13  +struct Cksum {
           14  +  int nRow;
           15  +  int cksum1;
           16  +  int cksum2;
           17  +};
           18  +
           19  +/*
           20  +** tdb_scan() callback used by testCksumDatabase()
           21  +*/
           22  +static void scanCksumDb(
           23  +  void *pCtx, 
           24  +  void *pKey, int nKey,
           25  +  void *pVal, int nVal
           26  +){
           27  +  Cksum *p = (Cksum *)pCtx;
           28  +  int i;
           29  +
           30  +  p->nRow++;
           31  +  for(i=0; i<nKey; i++){
           32  +    p->cksum1 += ((u8 *)pKey)[i];
           33  +    p->cksum2 += p->cksum1;
           34  +  }
           35  +  for(i=0; i<nVal; i++){
           36  +    p->cksum1 += ((u8 *)pVal)[i];
           37  +    p->cksum2 += p->cksum1;
           38  +  }
           39  +}
           40  +
           41  +/*
           42  +** tdb_scan() callback used by testCountDatabase()
           43  +*/
           44  +static void scanCountDb(
           45  +  void *pCtx, 
           46  +  void *pKey, int nKey,
           47  +  void *pVal, int nVal
           48  +){
           49  +  Cksum *p = (Cksum *)pCtx;
           50  +  p->nRow++;
           51  +
           52  +  unused_parameter(pKey);
           53  +  unused_parameter(nKey);
           54  +  unused_parameter(pVal);
           55  +  unused_parameter(nVal);
           56  +}
           57  +
           58  +
           59  +/*
           60  +** Iterate through the entire contents of database pDb. Write a checksum
           61  +** string based on the db contents into buffer zOut before returning. A
           62  +** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size:
           63  +**
           64  +**    * 32-bit integer (10 bytes)
           65  +**    * 1 space        (1 byte)
           66  +**    * 32-bit hex     (8 bytes)
           67  +**    * 1 space        (1 byte)
           68  +**    * 32-bit hex     (8 bytes)
           69  +**    * nul-terminator (1 byte)
           70  +**
           71  +** The number of entries in the database is returned.
           72  +*/
           73  +int testCksumDatabase(
           74  +  TestDb *pDb,                    /* Database handle */
           75  +  char *zOut                      /* Buffer to write checksum to */
           76  +){
           77  +  Cksum cksum;
           78  +  memset(&cksum, 0, sizeof(Cksum));
           79  +  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb);
           80  +  sprintf(zOut, "%d %x %x", 
           81  +      cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2
           82  +  );
           83  +  assert( strlen(zOut)<TEST_CKSUM_BYTES );
           84  +  return cksum.nRow;
           85  +}
           86  +
           87  +int testCountDatabase(TestDb *pDb){
           88  +  Cksum cksum;
           89  +  memset(&cksum, 0, sizeof(Cksum));
           90  +  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCountDb);
           91  +  return cksum.nRow;
           92  +}
           93  +
           94  +/*
           95  +** This function is a no-op if *pRc is not 0 when it is called.
           96  +**
           97  +** Otherwise, the two nul-terminated strings z1 and z1 are compared. If
           98  +** they are the same, the function returns without doing anything. Otherwise,
           99  +** an error message is printed, *pRc is set to 1 and the test_failed()
          100  +** function called.
          101  +*/
          102  +void testCompareStr(const char *z1, const char *z2, int *pRc){
          103  +  if( *pRc==0 ){
          104  +    if( strcmp(z1, z2) ){
          105  +      testPrintError("testCompareStr: \"%s\" != \"%s\"\n", z1, z2);
          106  +      *pRc = 1;
          107  +      test_failed();
          108  +    }
          109  +  }
          110  +}
          111  +
          112  +/*
          113  +** This function is a no-op if *pRc is not 0 when it is called.
          114  +**
          115  +** Otherwise, the two integers i1 and i2 are compared. If they are equal,
          116  +** the function returns without doing anything. Otherwise, an error message 
          117  +** is printed, *pRc is set to 1 and the test_failed() function called.
          118  +*/
          119  +void testCompareInt(int i1, int i2, int *pRc){
          120  +  if( *pRc==0 && i1!=i2 ){
          121  +    testPrintError("testCompareInt: %d != %d\n", i1, i2);
          122  +    *pRc = 1;
          123  +    test_failed();
          124  +  }
          125  +}
          126  +
          127  +void testCaseStart(int *pRc, char *zFmt, ...){
          128  +  va_list ap;
          129  +  va_start(ap, zFmt);
          130  +  vprintf(zFmt, ap);
          131  +  printf(" ...");
          132  +  va_end(ap);
          133  +  *pRc = 0;
          134  +  fflush(stdout);
          135  +}
          136  +
          137  +/*
          138  +** This function is a no-op if *pRc is non-zero when it is called. Zero
          139  +** is returned in this case.
          140  +**
          141  +** Otherwise, the zFmt (a printf style format string) and following arguments 
          142  +** are used to create a test case name. If zPattern is NULL or a glob pattern
          143  +** that matches the test case name, 1 is returned and the test case started.
          144  +** Otherwise, zero is returned and the test case does not start.
          145  +*/
          146  +int testCaseBegin(int *pRc, const char *zPattern, const char *zFmt, ...){
          147  +  int res = 0;
          148  +  if( *pRc==0 ){
          149  +    char *zTest;
          150  +    va_list ap;
          151  +
          152  +    va_start(ap, zFmt);
          153  +    zTest = testMallocVPrintf(zFmt, ap);
          154  +    va_end(ap);
          155  +    if( zPattern==0 || testGlobMatch(zPattern, zTest) ){
          156  +      printf("%-50s ...", zTest);
          157  +      res = 1;
          158  +    }
          159  +    testFree(zTest);
          160  +    fflush(stdout);
          161  +  }
          162  +
          163  +  return res;
          164  +}
          165  +
          166  +void testCaseFinish(int rc){
          167  +  if( rc==0 ){
          168  +    printf("Ok\n");
          169  +  }else{
          170  +    printf("FAILED\n");
          171  +  }
          172  +  fflush(stdout);
          173  +}
          174  +
          175  +void testCaseSkip(){
          176  +  printf("Skipped\n");
          177  +}
          178  +
          179  +void testSetupSavedLsmdb(
          180  +  const char *zCfg,
          181  +  const char *zFile,
          182  +  Datasource *pData,
          183  +  int nRow,
          184  +  int *pRc
          185  +){
          186  +  if( *pRc==0 ){
          187  +    int rc;
          188  +    TestDb *pDb;
          189  +    rc = tdb_lsm_open(zCfg, zFile, 1, &pDb);
          190  +    if( rc==0 ){
          191  +      testWriteDatasourceRange(pDb, pData, 0, nRow, &rc);
          192  +      testClose(&pDb);
          193  +      if( rc==0 ) testSaveDb(zFile, "log");
          194  +    }
          195  +    *pRc = rc;
          196  +  }
          197  +}
          198  +
          199  +/*
          200  +** This function is a no-op if *pRc is non-zero when it is called.
          201  +**
          202  +** Open the LSM database identified by zFile and compute its checksum
          203  +** (a string, as returned by testCksumDatabase()). If the checksum is
          204  +** identical to zExpect1 or, if it is not NULL, zExpect2, the test passes.
          205  +** Otherwise, print an error message and set *pRc to 1.
          206  +*/
          207  +static void testCompareCksumLsmdb(
          208  +  const char *zFile,              /* Path to LSM database */
          209  +  int bCompress,                  /* True if db is compressed */
          210  +  const char *zExpect1,           /* Expected checksum 1 */
          211  +  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
          212  +  int *pRc                        /* IN/OUT: Test case error code */
          213  +){
          214  +  if( *pRc==0 ){
          215  +    char zCksum[TEST_CKSUM_BYTES];
          216  +    TestDb *pDb;
          217  +
          218  +    *pRc = tdb_lsm_open((bCompress?"compression=1 mmap=0":""), zFile, 0, &pDb);
          219  +    testCksumDatabase(pDb, zCksum);
          220  +    testClose(&pDb);
          221  +
          222  +    if( *pRc==0 ){
          223  +      int r1 = 0;
          224  +      int r2 = -1;
          225  +
          226  +      r1 = strcmp(zCksum, zExpect1);
          227  +      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
          228  +      if( r1 && r2 ){
          229  +        if( zExpect2 ){
          230  +          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
          231  +              zCksum, zExpect1, zExpect2
          232  +          );
          233  +        }else{
          234  +          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
          235  +              zCksum, zExpect1
          236  +          );
          237  +        }
          238  +        *pRc = 1;
          239  +        test_failed();
          240  +      }
          241  +    }
          242  +  }
          243  +}
          244  +
          245  +#if 0 /* not used */
          246  +static void testCompareCksumBtdb(
          247  +  const char *zFile,              /* Path to LSM database */
          248  +  const char *zExpect1,           /* Expected checksum 1 */
          249  +  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
          250  +  int *pRc                        /* IN/OUT: Test case error code */
          251  +){
          252  +  if( *pRc==0 ){
          253  +    char zCksum[TEST_CKSUM_BYTES];
          254  +    TestDb *pDb;
          255  +
          256  +    *pRc = tdb_open("bt", zFile, 0, &pDb);
          257  +    testCksumDatabase(pDb, zCksum);
          258  +    testClose(&pDb);
          259  +
          260  +    if( *pRc==0 ){
          261  +      int r1 = 0;
          262  +      int r2 = -1;
          263  +
          264  +      r1 = strcmp(zCksum, zExpect1);
          265  +      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
          266  +      if( r1 && r2 ){
          267  +        if( zExpect2 ){
          268  +          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
          269  +              zCksum, zExpect1, zExpect2
          270  +          );
          271  +        }else{
          272  +          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
          273  +              zCksum, zExpect1
          274  +          );
          275  +        }
          276  +        *pRc = 1;
          277  +        test_failed();
          278  +      }
          279  +    }
          280  +  }
          281  +}
          282  +#endif /* not used */
          283  +
          284  +/* Above this point are reusable test routines. Not clear that they
          285  +** should really be in this file.
          286  +*************************************************************************/
          287  +
          288  +/*
          289  +** This test verifies that if a system crash occurs while doing merge work
          290  +** on the db, no data is lost.
          291  +*/
          292  +static void crash_test1(int bCompress, int *pRc){
          293  +  const char *DBNAME = "testdb.lsm";
          294  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 200, 200};
          295  +
          296  +  const int nRow = 5000;          /* Database size */
          297  +  const int nIter = 200;          /* Number of test iterations */
          298  +  const int nWork = 20;           /* Maximum lsm_work() calls per iteration */
          299  +  const int nPage = 15;           /* Pages per lsm_work call */
          300  +
          301  +  int i;
          302  +  int iDot = 0;
          303  +  Datasource *pData;
          304  +  CksumDb *pCksumDb;
          305  +  TestDb *pDb;
          306  +  char *zCfg;
          307  +
          308  +  const char *azConfig[2] = {
          309  +    "page_size=1024 block_size=65536 autoflush=16384 safety=2 mmap=0", 
          310  +    "page_size=1024 block_size=65536 autoflush=16384 safety=2 "
          311  +    " compression=1 mmap=0"
          312  +  };
          313  +  assert( bCompress==0 || bCompress==1 );
          314  +
          315  +  /* Allocate datasource. And calculate the expected checksums. */
          316  +  pData = testDatasourceNew(&defn);
          317  +  pCksumDb = testCksumArrayNew(pData, nRow, nRow, 1);
          318  +
          319  +  /* Setup and save the initial database. */
          320  +
          321  +  zCfg = testMallocPrintf("%s automerge=7", azConfig[bCompress]);
          322  +  testSetupSavedLsmdb(zCfg, DBNAME, pData, 5000, pRc);
          323  +  testFree(zCfg);
          324  +
          325  +  for(i=0; i<nIter && *pRc==0; i++){
          326  +    int iWork;
          327  +    int testrc = 0;
          328  +
          329  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          330  +
          331  +    /* Restore and open the database. */
          332  +    testRestoreDb(DBNAME, "log");
          333  +    testrc = tdb_lsm_open(azConfig[bCompress], DBNAME, 0, &pDb);
          334  +    assert( testrc==0 );
          335  +
          336  +    /* Call lsm_work() on the db */
          337  +    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nWork*2)));
          338  +    for(iWork=0; testrc==0 && iWork<nWork; iWork++){
          339  +      int nWrite = 0;
          340  +      lsm_db *db = tdb_lsm(pDb);
          341  +      testrc = lsm_work(db, 0, nPage, &nWrite);
          342  +      /* assert( testrc!=0 || nWrite>0 ); */
          343  +      if( testrc==0 ) testrc = lsm_checkpoint(db, 0);
          344  +    }
          345  +    tdb_close(pDb);
          346  +
          347  +    /* Check that the database content is still correct */
          348  +    testCompareCksumLsmdb(DBNAME, 
          349  +        bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc);
          350  +  }
          351  +
          352  +  testCksumArrayFree(pCksumDb);
          353  +  testDatasourceFree(pData);
          354  +}
          355  +
          356  +/*
          357  +** This test verifies that if a system crash occurs while committing a
          358  +** transaction to the log file, no earlier transactions are lost or damaged.
          359  +*/
          360  +static void crash_test2(int bCompress, int *pRc){
          361  +  const char *DBNAME = "testdb.lsm";
          362  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
          363  +
          364  +  const int nIter = 200;
          365  +  const int nInsert = 20;
          366  +
          367  +  int i;
          368  +  int iDot = 0;
          369  +  Datasource *pData;
          370  +  CksumDb *pCksumDb;
          371  +  TestDb *pDb;
          372  +
          373  +  /* Allocate datasource. And calculate the expected checksums. */
          374  +  pData = testDatasourceNew(&defn);
          375  +  pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1);
          376  +
          377  +  /* Setup and save the initial database. */
          378  +  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
          379  +
          380  +  for(i=0; i<nIter && *pRc==0; i++){
          381  +    int iIns;
          382  +    int testrc = 0;
          383  +
          384  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          385  +
          386  +    /* Restore and open the database. */
          387  +    testRestoreDb(DBNAME, "log");
          388  +    testrc = tdb_lsm_open("safety=2", DBNAME, 0, &pDb);
          389  +    assert( testrc==0 );
          390  +
          391  +    /* Insert nInsert records into the database. Crash midway through. */
          392  +    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nInsert+2)));
          393  +    for(iIns=0; iIns<nInsert; iIns++){
          394  +      void *pKey; int nKey;
          395  +      void *pVal; int nVal;
          396  +
          397  +      testDatasourceEntry(pData, 100+iIns, &pKey, &nKey, &pVal, &nVal);
          398  +      testrc = tdb_write(pDb, pKey, nKey, pVal, nVal);
          399  +      if( testrc ) break;
          400  +    }
          401  +    tdb_close(pDb);
          402  +
          403  +    /* Check that no data was lost when the system crashed. */
          404  +    testCompareCksumLsmdb(DBNAME, bCompress,
          405  +      testCksumArrayGet(pCksumDb, 100 + iIns),
          406  +      testCksumArrayGet(pCksumDb, 100 + iIns + 1),
          407  +      pRc
          408  +    );
          409  +  }
          410  +
          411  +  testDatasourceFree(pData);
          412  +  testCksumArrayFree(pCksumDb);
          413  +}
          414  +
          415  +
          416  +/*
          417  +** This test verifies that if a system crash occurs when checkpointing
          418  +** the database, data is not lost (assuming that any writes not synced
          419  +** to the db have been synced into the log file).
          420  +*/
          421  +static void crash_test3(int bCompress, int *pRc){
          422  +  const char *DBNAME = "testdb.lsm";
          423  +  const int nIter = 100;
          424  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
          425  +
          426  +  int i;
          427  +  int iDot = 0;
          428  +  Datasource *pData;
          429  +  CksumDb *pCksumDb;
          430  +  TestDb *pDb;
          431  +
          432  +  /* Allocate datasource. And calculate the expected checksums. */
          433  +  pData = testDatasourceNew(&defn);
          434  +  pCksumDb = testCksumArrayNew(pData, 110, 150, 10);
          435  +
          436  +  /* Setup and save the initial database. */
          437  +  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
          438  +
          439  +  for(i=0; i<nIter && *pRc==0; i++){
          440  +    int iOpen;
          441  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          442  +    testRestoreDb(DBNAME, "log");
          443  +
          444  +    for(iOpen=0; iOpen<5; iOpen++){
          445  +      /* Open the database. Insert 10 more records. */
          446  +      pDb = testOpen("lsm", 0, pRc);
          447  +      testWriteDatasourceRange(pDb, pData, 100+iOpen*10, 10, pRc);
          448  +
          449  +      /* Schedule a crash simulation then close the db. */
          450  +      tdb_lsm_prepare_sync_crash(pDb, 1 + (i%2));
          451  +      tdb_close(pDb);
          452  +
          453  +      /* Open the database and check that the crash did not cause any
          454  +      ** data loss.  */
          455  +      testCompareCksumLsmdb(DBNAME, bCompress,
          456  +        testCksumArrayGet(pCksumDb, 110 + iOpen*10), 0,
          457  +        pRc
          458  +      );
          459  +    }
          460  +  }
          461  +
          462  +  testDatasourceFree(pData);
          463  +  testCksumArrayFree(pCksumDb);
          464  +}
          465  +
          466  +void do_crash_test(const char *zPattern, int *pRc){
          467  +  struct Test {
          468  +    const char *zTest;
          469  +    void (*x)(int, int *);
          470  +    int bCompress;
          471  +  } aTest [] = {
          472  +    { "crash.lsm.1",     crash_test1, 0 },
          473  +#ifdef HAVE_ZLIB
          474  +    { "crash.lsm_zip.1", crash_test1, 1 },
          475  +#endif
          476  +    { "crash.lsm.2",     crash_test2, 0 },
          477  +    { "crash.lsm.3",     crash_test3, 0 },
          478  +  };
          479  +  int i;
          480  +
          481  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          482  +    struct Test *p = &aTest[i];
          483  +    if( testCaseBegin(pRc, zPattern, "%s", p->zTest) ){
          484  +      p->x(p->bCompress, pRc);
          485  +      testCaseFinish(*pRc);
          486  +    }
          487  +  }
          488  +}

Added ext/lsm1/lsm-test/lsmtest3.c.

            1  +
            2  +
            3  +/*
            4  +** This file contains tests related to the explicit rollback of database
            5  +** transactions and sub-transactions.
            6  +*/
            7  +
            8  +
            9  +/*
           10  +** Repeat 2000 times (until the db contains 100,000 entries):
           11  +**
           12  +**   1. Open a transaction and insert 500 rows, opening a nested 
           13  +**      sub-transaction each 100 rows.
           14  +**
           15  +**   2. Roll back to each sub-transaction savepoint. Check the database
           16  +**      checksum looks Ok.
           17  +**
           18  +**   3. Every second iteration, roll back the main transaction. Check the
           19  +**      db checksum is correct. Every other iteration, commit the main
           20  +**      transaction (increasing the size of the db by 100 rows).
           21  +*/
           22  +
           23  +
           24  +#include "lsmtest.h"
           25  +
           26  +struct CksumDb {
           27  +  int nFirst;
           28  +  int nLast;
           29  +  int nStep;
           30  +  char **azCksum;
           31  +};
           32  +
           33  +CksumDb *testCksumArrayNew(
           34  +  Datasource *pData, 
           35  +  int nFirst, 
           36  +  int nLast, 
           37  +  int nStep
           38  +){
           39  +  TestDb *pDb;
           40  +  CksumDb *pRet;
           41  +  int i;
           42  +  int nEntry;
           43  +  int rc = 0;
           44  +
           45  +  assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 );
           46  + 
           47  +  pRet = malloc(sizeof(CksumDb));
           48  +  memset(pRet, 0, sizeof(CksumDb));
           49  +  pRet->nFirst = nFirst;
           50  +  pRet->nLast = nLast;
           51  +  pRet->nStep = nStep;
           52  +  nEntry = 1 + ((nLast - nFirst) / nStep);
           53  +
           54  +  /* Allocate space so that azCksum is an array of nEntry pointers to
           55  +  ** buffers each TEST_CKSUM_BYTES in size.  */
           56  +  pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES));
           57  +  for(i=0; i<nEntry; i++){
           58  +    char *pStart = (char *)(&pRet->azCksum[nEntry]);
           59  +    pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES];
           60  +  }
           61  +
           62  +  tdb_open("lsm", "tempdb.lsm", 1, &pDb);
           63  +  testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc);
           64  +  for(i=0; i<nEntry; i++){
           65  +    testCksumDatabase(pDb, pRet->azCksum[i]);
           66  +    if( i==nEntry ) break;
           67  +    testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc);
           68  +  }
           69  +
           70  +  tdb_close(pDb);
           71  +
           72  +  return pRet;
           73  +}
           74  +
           75  +char *testCksumArrayGet(CksumDb *p, int nRow){
           76  +  int i;
           77  +  assert( nRow>=p->nFirst );
           78  +  assert( nRow<=p->nLast );
           79  +  assert( ((nRow-p->nFirst) % p->nStep)==0 );
           80  +
           81  +  i = (nRow - p->nFirst) / p->nStep;
           82  +  return p->azCksum[i];
           83  +}
           84  +
           85  +void testCksumArrayFree(CksumDb *p){
           86  +  free(p->azCksum);
           87  +  memset(p, 0x55, sizeof(*p));
           88  +  free(p);
           89  +}
           90  +
           91  +/* End of CksumDb code.
           92  +**************************************************************************/
           93  +
           94  +/*
           95  +** Test utility function. Write key-value pair $i from datasource pData 
           96  +** into database pDb.
           97  +*/
           98  +void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
           99  +  void *pKey; int nKey;
          100  +  void *pVal; int nVal;
          101  +  testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
          102  +  testWrite(pDb, pKey, nKey, pVal, nVal, pRc);
          103  +}
          104  +
          105  +/*
          106  +** Test utility function. Delete datasource pData key $i from database pDb.
          107  +*/
          108  +void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
          109  +  void *pKey; int nKey;
          110  +  testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
          111  +  testDelete(pDb, pKey, nKey, pRc);
          112  +}
          113  +
          114  +/*
          115  +** This function inserts nWrite key/value pairs into database pDb - the
          116  +** nWrite key value pairs starting at iFirst from data source pData.
          117  +*/
          118  +void testWriteDatasourceRange(
          119  +  TestDb *pDb,                    /* Database to write to */
          120  +  Datasource *pData,              /* Data source to read values from */
          121  +  int iFirst,                     /* Index of first key/value pair */
          122  +  int nWrite,                     /* Number of key/value pairs to write */
          123  +  int *pRc                        /* IN/OUT: Error code */
          124  +){
          125  +  int i;
          126  +  for(i=0; i<nWrite; i++){
          127  +    testWriteDatasource(pDb, pData, iFirst+i, pRc);
          128  +  }
          129  +}
          130  +
          131  +void testDeleteDatasourceRange(
          132  +  TestDb *pDb,                    /* Database to write to */
          133  +  Datasource *pData,              /* Data source to read keys from */
          134  +  int iFirst,                     /* Index of first key */
          135  +  int nWrite,                     /* Number of keys to delete */
          136  +  int *pRc                        /* IN/OUT: Error code */
          137  +){
          138  +  int i;
          139  +  for(i=0; i<nWrite; i++){
          140  +    testDeleteDatasource(pDb, pData, iFirst+i, pRc);
          141  +  }
          142  +}
          143  +
          144  +static char *getName(const char *zSystem){ 
          145  +  char *zRet; 
          146  +  zRet = testMallocPrintf("rollback.%s", zSystem);
          147  +  return zRet;
          148  +}
          149  +
          150  +static int rollback_test_1(
          151  +  const char *zSystem,
          152  +  Datasource *pData
          153  +){
          154  +  const int nRepeat = 100;
          155  +
          156  +  TestDb *pDb;
          157  +  int rc;
          158  +  int i;
          159  +  CksumDb *pCksum;
          160  +  char *zName;
          161  +
          162  +  zName = getName(zSystem);
          163  +  testCaseStart(&rc, zName);
          164  +  testFree(zName);
          165  +
          166  +  pCksum = testCksumArrayNew(pData, 0, nRepeat*100, 100);
          167  +  pDb = 0;
          168  +  rc = tdb_open(zSystem, 0, 1, &pDb);
          169  +  if( pDb && tdb_transaction_support(pDb)==0 ){
          170  +    testCaseSkip();
          171  +    goto skip_rollback_test;
          172  +  }
          173  +
          174  +  for(i=0; i<nRepeat && rc==0; i++){
          175  +    char zCksum[TEST_CKSUM_BYTES];
          176  +    int nCurrent = (((i+1)/2) * 100);
          177  +    int nDbRow;
          178  +    int iTrans;
          179  +
          180  +    /* Check that the database is the expected size. */
          181  +    nDbRow = testCountDatabase(pDb);
          182  +    testCompareInt(nCurrent, nDbRow, &rc);
          183  +
          184  +    for(iTrans=2; iTrans<=6 && rc==0; iTrans++){
          185  +      tdb_begin(pDb, iTrans);
          186  +      testWriteDatasourceRange(pDb, pData, nCurrent, 100, &rc);
          187  +      nCurrent += 100;
          188  +    }
          189  +
          190  +    testCksumDatabase(pDb, zCksum);
          191  +    testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          192  +
          193  +    for(iTrans=6; iTrans>2 && rc==0; iTrans--){
          194  +      tdb_rollback(pDb, iTrans);
          195  +      nCurrent -= 100;
          196  +      testCksumDatabase(pDb, zCksum);
          197  +      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          198  +    }
          199  +
          200  +    if( i%2 ){
          201  +      tdb_rollback(pDb, 0);
          202  +      nCurrent -= 100;
          203  +      testCksumDatabase(pDb, zCksum);
          204  +      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          205  +    }else{
          206  +      tdb_commit(pDb, 0);
          207  +    }
          208  +  }
          209  +  testCaseFinish(rc);
          210  +
          211  + skip_rollback_test:
          212  +  tdb_close(pDb);
          213  +  testCksumArrayFree(pCksum);
          214  +  return rc;
          215  +}
          216  +
          217  +void test_rollback(
          218  +  const char *zSystem, 
          219  +  const char *zPattern, 
          220  +  int *pRc
          221  +){
          222  +  if( *pRc==0 ){
          223  +    int bRun = 1;
          224  +
          225  +    if( zPattern ){
          226  +      char *zName = getName(zSystem);
          227  +      bRun = testGlobMatch(zPattern, zName);
          228  +      testFree(zName);
          229  +    }
          230  +
          231  +    if( bRun ){
          232  +      DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 };
          233  +      Datasource *pData = testDatasourceNew(&defn);
          234  +      *pRc = rollback_test_1(zSystem, pData);
          235  +      testDatasourceFree(pData);
          236  +    }
          237  +  }
          238  +}

Added ext/lsm1/lsm-test/lsmtest4.c.

            1  +
            2  +/*
            3  +** This file contains test cases involving multiple database clients.
            4  +*/
            5  +
            6  +#include "lsmtest.h"
            7  +
            8  +/*
            9  +** The following code implements test cases "mc1.*".
           10  +**
           11  +** This test case uses one writer and $nReader readers. All connections
           12  +** are driven by a single thread. All connections are opened at the start
           13  +** of the test and remain open until the test is finished.
           14  +**
           15  +** The test consists of $nStep steps. Each step the following is performed:
           16  +**
           17  +**   1. The writer inserts $nWriteStep records into the db.
           18  +**
           19  +**   2. The writer checks that the contents of the db are as expected.
           20  +**
           21  +**   3. Each reader that currently has an open read transaction also checks
           22  +**      that the contents of the db are as expected (according to the snapshot
           23  +**      the read transaction is reading - see below).
           24  +**
           25  +** After step 1, reader 1 opens a read transaction. After step 2, reader
           26  +** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1
           27  +** closes the current read transaction and opens a new one. And so on.
           28  +** The result is that at step N (for N > $nReader), there exists a reader
           29  +** with an open read transaction reading the snapshot committed following
           30  +** steps (N-$nReader-1) to N. 
           31  +*/
           32  +typedef struct Mctest Mctest;
           33  +struct Mctest {
           34  +  DatasourceDefn defn;            /* Datasource to use */
           35  +  int nStep;                      /* Total number of steps in test */
           36  +  int nWriteStep;                 /* Number of rows to insert each step */
           37  +  int nReader;                    /* Number of read connections */
           38  +};
           39  +static void do_mc_test(
           40  +  const char *zSystem,            /* Database system to test */
           41  +  Mctest *pTest,
           42  +  int *pRc                        /* IN/OUT: return code */
           43  +){
           44  +  const int nDomain = pTest->nStep * pTest->nWriteStep;
           45  +  Datasource *pData;              /* Source of data */
           46  +  TestDb *pDb;                    /* First database connection (writer) */
           47  +  int iReader;                    /* Used to iterate through aReader */
           48  +  int iStep;                      /* Current step in test */
           49  +  int iDot = 0;                   /* Current step in test */
           50  +
           51  +  /* Array of reader connections */
           52  +  struct Reader {
           53  +    TestDb *pDb;                  /* Connection handle */
           54  +    int iLast;                    /* Current snapshot contains keys 0..iLast */
           55  +  } *aReader;
           56  +
           57  +  /* Create a data source */
           58  +  pData = testDatasourceNew(&pTest->defn);
           59  +
           60  +  /* Open the writer connection */
           61  +  pDb = testOpen(zSystem, 1, pRc);
           62  +
           63  +  /* Allocate aReader */
           64  +  aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader);
           65  +  for(iReader=0; iReader<pTest->nReader; iReader++){
           66  +    aReader[iReader].pDb = testOpen(zSystem, 0, pRc);
           67  +  }
           68  +
           69  +  for(iStep=0; iStep<pTest->nStep; iStep++){
           70  +    int iLast;
           71  +    int iBegin;                   /* Start read trans using aReader[iBegin] */
           72  +
           73  +    /* Insert nWriteStep more records into the database */
           74  +    int iFirst = iStep*pTest->nWriteStep;
           75  +    testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc);
           76  +
           77  +    /* Check that the db is Ok according to the writer */
           78  +    iLast = (iStep+1) * pTest->nWriteStep - 1;
           79  +    testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc);
           80  +
           81  +    /* Have reader (iStep % nReader) open a read transaction here. */
           82  +    iBegin = (iStep % pTest->nReader);
           83  +    if( iBegin<iStep ) tdb_commit(aReader[iBegin].pDb, 0);
           84  +    tdb_begin(aReader[iBegin].pDb, 1);
           85  +    aReader[iBegin].iLast = iLast;
           86  +
           87  +    /* Check that the db is Ok for each open reader */
           88  +    for(iReader=0; iReader<pTest->nReader && aReader[iReader].iLast; iReader++){
           89  +      iLast = aReader[iReader].iLast;
           90  +      testDbContents(
           91  +          aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc
           92  +      );
           93  +    }
           94  +
           95  +    /* Report progress */
           96  +    testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot);
           97  +  }
           98  +
           99  +  /* Close all readers */
          100  +  for(iReader=0; iReader<pTest->nReader; iReader++){
          101  +    testClose(&aReader[iReader].pDb);
          102  +  }
          103  +  testFree(aReader);
          104  +
          105  +  /* Close the writer-connection and free the datasource */
          106  +  testClose(&pDb);
          107  +  testDatasourceFree(pData);
          108  +}
          109  +
          110  +
          111  +void test_mc(
          112  +  const char *zSystem,            /* Database system name */
          113  +  const char *zPattern,           /* Run test cases that match this pattern */
          114  +  int *pRc                        /* IN/OUT: Error code */
          115  +){
          116  +  int i;
          117  +  Mctest aTest[] = {
          118  +    { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 },
          119  +  };
          120  +
          121  +  for(i=0; i<ArraySize(aTest); i++){
          122  +    if( testCaseBegin(pRc, zPattern, "mc1.%s.%d", zSystem, i) ){
          123  +      do_mc_test(zSystem, &aTest[i], pRc);
          124  +      testCaseFinish(*pRc);
          125  +    }
          126  +  }
          127  +}

Added ext/lsm1/lsm-test/lsmtest5.c.

            1  +
            2  +/*
            3  +** This file is broken into three semi-autonomous parts:
            4  +**
            5  +**   1. The database functions.
            6  +**   2. The thread wrappers.
            7  +**   3. The implementation of the mt1.* tests.
            8  +*/
            9  +
           10  +/*************************************************************************
           11  +** DATABASE CONTENTS:
           12  +**
           13  +**   The database contains up to N key/value pairs, where N is some large 
           14  +**   number (say 10,000,000). Keys are integer values between 0 and (N-1).
           15  +**   The value associated with each key is a pseudo-random blob of data.
           16  +**
           17  +**   Key/value pair keys are encoded as the two bytes "k." followed by a 
           18  +**   10-digit decimal number. i.e. key 45 -> "k.0000000045".
           19  +**
           20  +**   As well as the key/value pairs, the database also contains checksum 
           21  +**   entries. The checksums form a hierarchy - for every F key/value
           22  +**   entries there is one level 1 checksum. And for each F level 1 checksums
           23  +**   there is one level 2 checksum. And so on.
           24  +**
           25  +**   Checksum keys are encoded as the two byte "c." followed by the 
           26  +**   checksum level, followed by a 10 digit decimal number containing
           27  +**   the value of the first key that contributes to the checksum value.
           28  +**   For example, assuming F==10, the level 1 checksum that spans keys
           29  +**   10 to 19 is "c.1.0000000010".
           30  +**
           31  +**   Clients may perform one of two operations on the database: a read
           32  +**   or a write.
           33  +** 
           34  +** READ OPERATIONS:
           35  +**
           36  +**   A read operation scans a range of F key/value pairs. It computes
           37  +**   the expected checksum and then compares the computed value to the
           38  +**   actual value stored in the level 1 checksum entry. It then scans 
           39  +**   the group of F level 1 checksums, and compares the computed checksum 
           40  +**   to the associated level 2 checksum value, and so on until the 
           41  +**   highest level checksum value has been verified.
           42  +**
           43  +**   If a checksum ever fails to match the expected value, the test 
           44  +**   has failed.
           45  +**
           46  +** WRITE OPERATIONS:
           47  +**
           48  +**   A write operation involves writing (possibly clobbering) a single
           49  +**   key/value pair. The associated level 1 checksum is then recalculated
           50  +**   updated. Then the level 2 checksum, and so on until the highest
           51  +**   level checksum has been modified.
           52  +**
           53  +**   All updates occur inside a single transaction.
           54  +**
           55  +** INTERFACE:
           56  +**
           57  +**   The interface used by test cases to read and write the db consists
           58  +**   of type DbParameters and the following functions:
           59  +**
           60  +**       dbReadOperation()
           61  +**       dbWriteOperation()
           62  +*/
           63  +
           64  +#include "lsmtest.h"
           65  +
           66  +typedef struct DbParameters DbParameters;
           67  +struct DbParameters {
           68  +  int nFanout;                    /* Checksum fanout (F) */
           69  +  int nKey;                       /* Size of key space (N) */
           70  +};
           71  +
           72  +#define DB_KEY_BYTES          (2+5+10+1)
           73  +
           74  +/*
           75  +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
           76  +** This function populates the buffer with a nul-terminated key string 
           77  +** corresponding to key iKey.
           78  +*/
           79  +static void dbFormatKey(
           80  +  DbParameters *pParam,
           81  +  int iLevel,
           82  +  int iKey,                       /* Key value */
           83  +  char *aBuf                      /* Write key string here */
           84  +){
           85  +  if( iLevel==0 ){
           86  +    snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey);
           87  +  }else{
           88  +    int f = 1;
           89  +    int i;
           90  +    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
           91  +    snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f));
           92  +  }
           93  +}
           94  +
           95  +/*
           96  +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
           97  +** This function populates the buffer with the string representation of
           98  +** checksum value iVal.
           99  +*/
          100  +static void dbFormatCksumValue(u32 iVal, char *aBuf){
          101  +  snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal);
          102  +}
          103  +
          104  +/*
          105  +** Return the highest level of checksum in the database described
          106  +** by *pParam.
          107  +*/
          108  +static int dbMaxLevel(DbParameters *pParam){
          109  +  int iMax;
          110  +  int n = 1;
          111  +  for(iMax=0; n<pParam->nKey; iMax++){
          112  +    n = n * pParam->nFanout;
          113  +  }
          114  +  return iMax;
          115  +}
          116  +
          117  +static void dbCksum(
          118  +  void *pCtx,                     /* IN/OUT: Pointer to u32 containing cksum */
          119  +  void *pKey, int nKey,           /* Database key. Unused. */
          120  +  void *pVal, int nVal            /* Database value. Checksum this. */
          121  +){
          122  +  u8 *aVal = (u8 *)pVal;
          123  +  u32 *pCksum = (u32 *)pCtx;
          124  +  u32 cksum = *pCksum;
          125  +  int i;
          126  +
          127  +  unused_parameter(pKey);
          128  +  unused_parameter(nKey);
          129  +
          130  +  for(i=0; i<nVal; i++){
          131  +    cksum += (cksum<<3) + (int)aVal[i];
          132  +  }
          133  +
          134  +  *pCksum = cksum;
          135  +}
          136  +
          137  +/*
          138  +** Compute the value of the checksum stored on level iLevel that contains
          139  +** data from key iKey by scanning the pParam->nFanout entries at level 
          140  +** iLevel-1.
          141  +*/
          142  +static u32 dbComputeCksum(
          143  +  DbParameters *pParam,           /* Database parameters */
          144  +  TestDb *pDb,                    /* Database connection handle */
          145  +  int iLevel,                     /* Level of checksum to compute */
          146  +  int iKey,                       /* Compute checksum for this key */
          147  +  int *pRc                        /* IN/OUT: Error code */
          148  +){
          149  +  u32 cksum = 0;
          150  +  if( *pRc==0 ){
          151  +    int nFirst;
          152  +    int nLast;
          153  +    int iFirst = 0;
          154  +    int iLast = 0;
          155  +    int i;
          156  +    int f = 1;
          157  +    char zFirst[DB_KEY_BYTES];
          158  +    char zLast[DB_KEY_BYTES];
          159  +
          160  +    assert( iLevel>=1 );
          161  +    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
          162  +
          163  +    iFirst = f*(iKey/f);
          164  +    iLast = iFirst + f - 1;
          165  +    dbFormatKey(pParam, iLevel-1, iFirst, zFirst);
          166  +    dbFormatKey(pParam, iLevel-1, iLast, zLast);
          167  +    nFirst = strlen(zFirst);
          168  +    nLast = strlen(zLast);
          169  +
          170  +    *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum);
          171  +  }
          172  +
          173  +  return cksum;
          174  +}
          175  +
          176  +static void dbReadOperation(
          177  +  DbParameters *pParam,           /* Database parameters */
          178  +  TestDb *pDb,                    /* Database connection handle */
          179  +  void (*xDelay)(void *),
          180  +  void *pDelayCtx,
          181  +  int iKey,                       /* Key to read */
          182  +  int *pRc                        /* IN/OUT: Error code */
          183  +){
          184  +  const int iMax = dbMaxLevel(pParam);
          185  +  int i;
          186  +
          187  +  if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc);
          188  +  for(i=1; *pRc==0 && i<=iMax; i++){
          189  +    char zCksum[DB_KEY_BYTES];
          190  +    char zKey[DB_KEY_BYTES];
          191  +    u32 iCksum = 0;
          192  +
          193  +    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
          194  +    if( iCksum ){
          195  +      if( xDelay && i==1 ) xDelay(pDelayCtx);
          196  +      dbFormatCksumValue(iCksum, zCksum);
          197  +      dbFormatKey(pParam, i, iKey, zKey);
          198  +      testFetchStr(pDb, zKey, zCksum, pRc);
          199  +    }
          200  +  }
          201  +  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
          202  +}
          203  +
          204  +static int dbWriteOperation(
          205  +  DbParameters *pParam,           /* Database parameters */
          206  +  TestDb *pDb,                    /* Database connection handle */
          207  +  int iKey,                       /* Key to write to */
          208  +  const char *zValue,             /* Nul-terminated value to write */
          209  +  int *pRc                        /* IN/OUT: Error code */
          210  +){
          211  +  const int iMax = dbMaxLevel(pParam);
          212  +  char zKey[DB_KEY_BYTES];
          213  +  int i;
          214  +  int rc;
          215  +
          216  +  assert( iKey>=0 && iKey<pParam->nKey );
          217  +  dbFormatKey(pParam, 0, iKey, zKey);
          218  +
          219  +  /* Open a write transaction. This may fail - SQLITE4_BUSY */
          220  +  if( *pRc==0 && tdb_transaction_support(pDb) ){
          221  +    rc = tdb_begin(pDb, 2);
          222  +    if( rc==5 ) return 0;
          223  +    *pRc = rc;
          224  +  }
          225  +
          226  +  testWriteStr(pDb, zKey, zValue, pRc);
          227  +  for(i=1; i<=iMax; i++){
          228  +    char zCksum[DB_KEY_BYTES];
          229  +    u32 iCksum = 0;
          230  +
          231  +    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
          232  +    dbFormatCksumValue(iCksum, zCksum);
          233  +    dbFormatKey(pParam, i, iKey, zKey);
          234  +    testWriteStr(pDb, zKey, zCksum, pRc);
          235  +  }
          236  +  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
          237  +  return 1;
          238  +}
          239  +
          240  +/*************************************************************************
          241  +** The following block contains testXXX() functions that implement a
          242  +** wrapper around the systems native multi-thread support. There are no
          243  +** synchronization primitives - just functions to launch and join 
          244  +** threads. Wrapper functions are:
          245  +**
          246  +**    testThreadSupport()
          247  +**
          248  +**    testThreadInit()
          249  +**    testThreadShutdown()
          250  +**    testThreadLaunch()
          251  +**    testThreadWait()
          252  +**
          253  +**    testThreadSetHalt()
          254  +**    testThreadGetHalt()
          255  +**    testThreadSetResult()
          256  +**    testThreadGetResult()
          257  +**
          258  +**    testThreadEnterMutex()
          259  +**    testThreadLeaveMutex()
          260  +*/
          261  +typedef struct ThreadSet ThreadSet;
          262  +#ifdef LSM_MUTEX_PTHREADS
          263  +
          264  +#include <pthread.h>
          265  +#include <unistd.h>
          266  +
          267  +typedef struct Thread Thread;
          268  +struct Thread {
          269  +  int rc;
          270  +  char *zMsg;
          271  +  pthread_t id;
          272  +  void (*xMain)(ThreadSet *, int, void *);
          273  +  void *pCtx;
          274  +  ThreadSet *pThreadSet;
          275  +};
          276  +
          277  +struct ThreadSet {
          278  +  int bHalt;                      /* Halt flag */
          279  +  int nThread;                    /* Number of threads */
          280  +  Thread *aThread;                /* Array of Thread structures */
          281  +  pthread_mutex_t mutex;          /* Mutex used for cheating */
          282  +};
          283  +
          284  +/*
          285  +** Return true if this build supports threads, or false otherwise. If
          286  +** this function returns false, no other testThreadXXX() functions should
          287  +** be called.
          288  +*/
          289  +static int testThreadSupport(){ return 1; }
          290  +
          291  +/*
          292  +** Allocate and return a thread-set handle with enough space allocated
          293  +** to handle up to nMax threads. Each call to this function should be
          294  +** matched by a call to testThreadShutdown() to delete the object.
          295  +*/
          296  +static ThreadSet *testThreadInit(int nMax){
          297  +  int nByte;                      /* Total space to allocate */
          298  +  ThreadSet *p;                   /* Return value */
          299  +
          300  +  nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax;
          301  +  p = (ThreadSet *)testMalloc(nByte);
          302  +  p->nThread = nMax;
          303  +  p->aThread = (Thread *)&p[1];
          304  +  pthread_mutex_init(&p->mutex, 0);
          305  +
          306  +  return p;
          307  +}
          308  +
          309  +/*
          310  +** Delete a thread-set object and release all resources held by it.
          311  +*/
          312  +static void testThreadShutdown(ThreadSet *p){
          313  +  int i;
          314  +  for(i=0; i<p->nThread; i++){
          315  +    testFree(p->aThread[i].zMsg);
          316  +  }
          317  +  pthread_mutex_destroy(&p->mutex);
          318  +  testFree(p);
          319  +}
          320  +
          321  +static void *ttMain(void *pArg){
          322  +  Thread *pThread = (Thread *)pArg;
          323  +  int iThread;
          324  +  iThread = (pThread - pThread->pThreadSet->aThread);
          325  +  pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx);
          326  +  return 0;
          327  +}
          328  +
          329  +/*
          330  +** Launch a new thread.
          331  +*/
          332  +static int testThreadLaunch(
          333  +  ThreadSet *p,
          334  +  int iThread,
          335  +  void (*xMain)(ThreadSet *, int, void *),
          336  +  void *pCtx
          337  +){
          338  +  int rc;
          339  +  Thread *pThread;
          340  +
          341  +  assert( iThread>=0 && iThread<p->nThread );
          342  +
          343  +  pThread = &p->aThread[iThread];
          344  +  assert( pThread->pThreadSet==0 );
          345  +  pThread->xMain = xMain;
          346  +  pThread->pCtx = pCtx;
          347  +  pThread->pThreadSet = p;
          348  +  rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread);
          349  +
          350  +  return rc;
          351  +}
          352  +
          353  +/*
          354  +** Set the thread-set "halt" flag.
          355  +*/
          356  +static void testThreadSetHalt(ThreadSet *pThreadSet){
          357  +  pThreadSet->bHalt = 1;
          358  +}
          359  +
          360  +/*
          361  +** Return the current value of the thread-set "halt" flag.
          362  +*/
          363  +static int testThreadGetHalt(ThreadSet *pThreadSet){
          364  +  return pThreadSet->bHalt;
          365  +}
          366  +
          367  +static void testThreadSleep(ThreadSet *pThreadSet, int nMs){
          368  +  int nRem = nMs;
          369  +  while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){
          370  +    usleep(50000);
          371  +    nRem -= 50;
          372  +  }
          373  +}
          374  +
          375  +/*
          376  +** Wait for all threads launched to finish before returning. If nMs
          377  +** is greater than zero, set the "halt" flag to tell all threads
          378  +** to halt after waiting nMs milliseconds.
          379  +*/
          380  +static void testThreadWait(ThreadSet *pThreadSet, int nMs){
          381  +  int i;
          382  +
          383  +  testThreadSleep(pThreadSet, nMs);
          384  +  testThreadSetHalt(pThreadSet);
          385  +  for(i=0; i<pThreadSet->nThread; i++){
          386  +    Thread *pThread = &pThreadSet->aThread[i];
          387  +    if( pThread->xMain ){
          388  +      pthread_join(pThread->id, 0);
          389  +    }
          390  +  }
          391  +}
          392  +
          393  +/*
          394  +** Set the result for thread iThread. 
          395  +*/
          396  +static void testThreadSetResult(
          397  +  ThreadSet *pThreadSet,          /* Thread-set handle */
          398  +  int iThread,                    /* Set result for this thread */
          399  +  int rc,                         /* Result error code */
          400  +  char *zFmt,                     /* Result string format */
          401  +  ...                             /* Result string formatting args... */
          402  +){
          403  +  va_list ap;
          404  +
          405  +  testFree(pThreadSet->aThread[iThread].zMsg);
          406  +  pThreadSet->aThread[iThread].rc = rc;
          407  +  pThreadSet->aThread[iThread].zMsg = 0;
          408  +  if( zFmt ){
          409  +    va_start(ap, zFmt);
          410  +    pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap);
          411  +    va_end(ap);
          412  +  }
          413  +}
          414  +
          415  +/*
          416  +** Retrieve the result for thread iThread. 
          417  +*/
          418  +static int testThreadGetResult(
          419  +  ThreadSet *pThreadSet,          /* Thread-set handle */
          420  +  int iThread,                    /* Get result for this thread */
          421  +  const char **pzRes              /* OUT: Pointer to result string */
          422  +){
          423  +  if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg;
          424  +  return pThreadSet->aThread[iThread].rc;
          425  +}
          426  +
          427  +/*
          428  +** Enter and leave the test case mutex.
          429  +*/
          430  +#if 0
          431  +static void testThreadEnterMutex(ThreadSet *p){
          432  +  pthread_mutex_lock(&p->mutex);
          433  +}
          434  +static void testThreadLeaveMutex(ThreadSet *p){
          435  +  pthread_mutex_unlock(&p->mutex);
          436  +}
          437  +#endif
          438  +#endif
          439  +
          440  +#if !defined(LSM_MUTEX_PTHREADS)
          441  +static int testThreadSupport(){ return 0; }
          442  +
          443  +#define testThreadInit(a) 0
          444  +#define testThreadShutdown(a)
          445  +#define testThreadLaunch(a,b,c,d) 0
          446  +#define testThreadWait(a,b)
          447  +#define testThreadSetHalt(a)
          448  +#define testThreadGetHalt(a) 0
          449  +#define testThreadGetResult(a,b,c) 0
          450  +#define testThreadSleep(a,b) 0
          451  +
          452  +static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){
          453  +  unused_parameter(a);
          454  +  unused_parameter(b);
          455  +  unused_parameter(c);
          456  +  unused_parameter(d);
          457  +}
          458  +#endif
          459  +/* End of threads wrapper.
          460  +*************************************************************************/
          461  +
          462  +/*************************************************************************
          463  +** Below this point is the third part of this file - the implementation
          464  +** of the mt1.* tests.
          465  +*/
          466  +typedef struct Mt1Test Mt1Test;
          467  +struct Mt1Test {
          468  +  DbParameters param;             /* Description of database to read/write */
          469  +  int nReadwrite;                 /* Number of read/write threads */
          470  +  int nFastReader;                /* Number of fast reader threads */
          471  +  int nSlowReader;                /* Number of slow reader threads */
          472  +  int nMs;                        /* How long to run for */
          473  +  const char *zSystem;            /* Database system to test */
          474  +};
          475  +
          476  +typedef struct Mt1DelayCtx Mt1DelayCtx;
          477  +struct Mt1DelayCtx {
          478  +  ThreadSet *pSet;                /* Threadset to sleep within */
          479  +  int nMs;                        /* Sleep in ms */
          480  +};
          481  +
          482  +static void xMt1Delay(void *pCtx){
          483  +  Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx;
          484  +  testThreadSleep(p->pSet, p->nMs);
          485  +}
          486  +
          487  +#define MT1_THREAD_RDWR 0
          488  +#define MT1_THREAD_SLOW 1
          489  +#define MT1_THREAD_FAST 2
          490  +
          491  +static void xMt1Work(lsm_db *pDb, void *pCtx){
          492  +#if 0
          493  +  char *z = 0;
          494  +  lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z);
          495  +  printf("%s\n", z);
          496  +  fflush(stdout);
          497  +#endif
          498  +}
          499  +
          500  +/*
          501  +** This is the main() proc for all threads in test case "mt1".
          502  +*/
          503  +static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){
          504  +  Mt1Test *p = (Mt1Test *)pCtx;   /* Test parameters */
          505  +  Mt1DelayCtx delay;
          506  +  int nRead = 0;                  /* Number of calls to dbReadOperation() */
          507  +  int nWrite = 0;                 /* Number of completed database writes */
          508  +  int rc = 0;                     /* Error code */
          509  +  int iPrng;                      /* Prng argument variable */
          510  +  TestDb *pDb;                    /* Database handle */
          511  +  int eType;
          512  +
          513  +  delay.pSet = pThreadSet;
          514  +  delay.nMs = 0;
          515  +  if( iThread<p->nReadwrite ){
          516  +    eType = MT1_THREAD_RDWR;
          517  +  }else if( iThread<(p->nReadwrite+p->nFastReader) ){
          518  +    eType = MT1_THREAD_FAST;
          519  +  }else{
          520  +    eType = MT1_THREAD_SLOW;
          521  +    delay.nMs = (p->nMs / 20);
          522  +  }
          523  +
          524  +  /* Open a new database connection. Initialize the pseudo-random number
          525  +  ** argument based on the thread number.  */
          526  +  iPrng = testPrngValue(iThread);
          527  +  pDb = testOpen(p->zSystem, 0, &rc);
          528  +
          529  +  if( rc==0 ){
          530  +    tdb_lsm_config_work_hook(pDb, xMt1Work, 0);
          531  +  }
          532  +
          533  +  /* Loop until either an error occurs or some other thread sets the
          534  +  ** halt flag.  */
          535  +  while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){
          536  +    int iKey;
          537  +
          538  +    /* Perform a read operation on an arbitrarily selected key. */
          539  +    iKey = (testPrngValue(iPrng++) % p->param.nKey);
          540  +    dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc);
          541  +    if( rc ) continue;
          542  +    nRead++;
          543  +
          544  +    /* Attempt to write an arbitrary key value pair (and update the associated
          545  +    ** checksum entries). dbWriteOperation() returns 1 if the write is
          546  +    ** successful, or 0 if it failed with an LSM_BUSY error.  */
          547  +    if( eType==MT1_THREAD_RDWR ){
          548  +      char aValue[50];
          549  +      char aRnd[25];
          550  +
          551  +      iKey = (testPrngValue(iPrng++) % p->param.nKey);
          552  +      testPrngString(iPrng, aRnd, sizeof(aRnd));
          553  +      iPrng += sizeof(aRnd);
          554  +      snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd);
          555  +      nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc);
          556  +    }
          557  +  }
          558  +  testClose(&pDb);
          559  +
          560  +  /* If an error has occured, set the thread error code and the threadset 
          561  +  ** halt flag to tell the other test threads to halt. Otherwise, set the
          562  +  ** thread error code to 0 and post a message with the number of read
          563  +  ** and write operations completed.  */
          564  +  if( rc ){
          565  +    testThreadSetResult(pThreadSet, iThread, rc, 0);
          566  +    testThreadSetHalt(pThreadSet);
          567  +  }else{
          568  +    testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite);
          569  +  }
          570  +}
          571  +
          572  +static void do_test_mt1(
          573  +  const char *zSystem,            /* Database system name */
          574  +  const char *zPattern,           /* Run test cases that match this pattern */
          575  +  int *pRc                        /* IN/OUT: Error code */
          576  +){
          577  +  Mt1Test aTest[] = {
          578  +    /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */
          579  +    { {10, 1000},     4, 0, 0,   10000,   0 },
          580  +    { {10, 1000},     4, 4, 2,   100000,  0 },
          581  +    { {10, 100000},   4, 0, 0,   10000,   0 },
          582  +    { {10, 100000},   4, 4, 2,   100000,  0 },
          583  +  };
          584  +  int i;
          585  +
          586  +  for(i=0; *pRc==0 && i<ArraySize(aTest); i++){
          587  +    Mt1Test *p = &aTest[i];
          588  +    int bRun = testCaseBegin(pRc, zPattern, 
          589  +        "mt1.%s.db=%d,%d.ms=%d.rdwr=%d.fast=%d.slow=%d", 
          590  +        zSystem, p->param.nFanout, p->param.nKey, 
          591  +        p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader
          592  +    );
          593  +    if( bRun ){
          594  +      TestDb *pDb;
          595  +      ThreadSet *pSet;
          596  +      int iThread;
          597  +      int nThread;
          598  +
          599  +      p->zSystem = zSystem;
          600  +      pDb = testOpen(zSystem, 1, pRc);
          601  +
          602  +      nThread = p->nReadwrite + p->nFastReader + p->nSlowReader;
          603  +      pSet = testThreadInit(nThread);
          604  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          605  +        testThreadLaunch(pSet, iThread, mt1Main, (void *)p);
          606  +      }
          607  +
          608  +      testThreadWait(pSet, p->nMs);
          609  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          610  +        *pRc = testThreadGetResult(pSet, iThread, 0);
          611  +      }
          612  +      testCaseFinish(*pRc);
          613  +
          614  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          615  +        const char *zMsg = 0;
          616  +        *pRc = testThreadGetResult(pSet, iThread, &zMsg);
          617  +        printf("  Info: thread %d (%d): %s\n", iThread, *pRc, zMsg);
          618  +      }
          619  +
          620  +      testThreadShutdown(pSet);
          621  +      testClose(&pDb);
          622  +    }
          623  +  }
          624  +}
          625  +
          626  +void test_mt(
          627  +  const char *zSystem,            /* Database system name */
          628  +  const char *zPattern,           /* Run test cases that match this pattern */
          629  +  int *pRc                        /* IN/OUT: Error code */
          630  +){
          631  +  if( testThreadSupport()==0 ) return;
          632  +  do_test_mt1(zSystem, zPattern, pRc);
          633  +}

Added ext/lsm1/lsm-test/lsmtest6.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +typedef struct OomTest OomTest;
            5  +struct OomTest {
            6  +  lsm_env *pEnv;
            7  +  int iNext;                      /* Next value to pass to testMallocOom() */
            8  +  int nFail;                      /* Number of OOM events injected */
            9  +  int bEnable;
           10  +  int rc;                         /* Test case error code */
           11  +};
           12  +
           13  +static void testOomStart(OomTest *p){
           14  +  memset(p, 0, sizeof(OomTest));
           15  +  p->iNext = 1;
           16  +  p->bEnable = 1;
           17  +  p->nFail = 1;
           18  +  p->pEnv = tdb_lsm_env();
           19  +}
           20  +
           21  +static void xOomHook(OomTest *p){
           22  +  p->nFail++;
           23  +}
           24  +
           25  +static int testOomContinue(OomTest *p){
           26  +  if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){
           27  +    return 0;
           28  +  }
           29  +  p->nFail = 0;
           30  +  testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p);
           31  +  return 1;
           32  +}
           33  +
           34  +static void testOomEnable(OomTest *p, int bEnable){
           35  +  p->bEnable = bEnable;
           36  +  testMallocOomEnable(p->pEnv, bEnable);
           37  +}
           38  +
           39  +static void testOomNext(OomTest *p){
           40  +  p->iNext++;
           41  +}
           42  +
           43  +static int testOomHit(OomTest *p){
           44  +  return (p->nFail>0);
           45  +}
           46  +
           47  +static int testOomFinish(OomTest *p){
           48  +  return p->rc;
           49  +}
           50  +
           51  +static void testOomAssert(OomTest *p, int bVal){
           52  +  if( bVal==0 ){
           53  +    test_failed();
           54  +    p->rc = 1;
           55  +  }
           56  +}
           57  +
           58  +/*
           59  +** Test that the error code matches the state of the OomTest object passed
           60  +** as the first argument. Specifically, check that rc is LSM_NOMEM if an 
           61  +** OOM error has already been injected, or LSM_OK if not.
           62  +*/
           63  +static void testOomAssertRc(OomTest *p, int rc){
           64  +  testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM);
           65  +  testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 );
           66  +}
           67  +
           68  +static void testOomOpen(
           69  +  OomTest *pOom,
           70  +  const char *zName,
           71  +  lsm_db **ppDb,
           72  +  int *pRc
           73  +){
           74  +  if( *pRc==LSM_OK ){
           75  +    int rc;
           76  +    rc = lsm_new(tdb_lsm_env(), ppDb);
           77  +    if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName);
           78  +    testOomAssertRc(pOom, rc);
           79  +    *pRc = rc;
           80  +  }
           81  +}
           82  +
           83  +static void testOomFetch(
           84  +  OomTest *pOom,
           85  +  lsm_db *pDb,
           86  +  void *pKey, int nKey,
           87  +  void *pVal, int nVal,
           88  +  int *pRc
           89  +){
           90  +  testOomAssertRc(pOom, *pRc);
           91  +  if( *pRc==LSM_OK ){
           92  +    lsm_cursor *pCsr;
           93  +    int rc;
           94  +
           95  +    rc = lsm_csr_open(pDb, &pCsr);
           96  +    if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0);
           97  +    testOomAssertRc(pOom, rc);
           98  +
           99  +    if( rc==LSM_OK ){
          100  +      const void *p; int n;
          101  +      testOomAssert(pOom, lsm_csr_valid(pCsr));
          102  +
          103  +      rc = lsm_csr_key(pCsr, &p, &n);
          104  +      testOomAssertRc(pOom, rc);
          105  +      testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) );
          106  +    }
          107  +
          108  +    if( rc==LSM_OK ){
          109  +      const void *p; int n;
          110  +      testOomAssert(pOom, lsm_csr_valid(pCsr));
          111  +
          112  +      rc = lsm_csr_value(pCsr, &p, &n);
          113  +      testOomAssertRc(pOom, rc);
          114  +      testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) );
          115  +    }
          116  +
          117  +    lsm_csr_close(pCsr);
          118  +    *pRc = rc;
          119  +  }
          120  +}
          121  +
          122  +static void testOomWrite(
          123  +  OomTest *pOom,
          124  +  lsm_db *pDb,
          125  +  void *pKey, int nKey,
          126  +  void *pVal, int nVal,
          127  +  int *pRc
          128  +){
          129  +  testOomAssertRc(pOom, *pRc);
          130  +  if( *pRc==LSM_OK ){
          131  +    int rc;
          132  +
          133  +    rc = lsm_insert(pDb, pKey, nKey, pVal, nVal);
          134  +    testOomAssertRc(pOom, rc);
          135  +
          136  +    *pRc = rc;
          137  +  }
          138  +}
          139  +
          140  +
          141  +static void testOomFetchStr(
          142  +  OomTest *pOom,
          143  +  lsm_db *pDb,
          144  +  const char *zKey,
          145  +  const char *zVal,
          146  +  int *pRc
          147  +){
          148  +  int nKey = strlen(zKey);
          149  +  int nVal = strlen(zVal);
          150  +  testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
          151  +}
          152  +
          153  +static void testOomFetchData(
          154  +  OomTest *pOom,
          155  +  lsm_db *pDb,
          156  +  Datasource *pData,
          157  +  int iKey,
          158  +  int *pRc
          159  +){
          160  +  void *pKey; int nKey;
          161  +  void *pVal; int nVal;
          162  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          163  +  testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
          164  +}
          165  +
          166  +static void testOomWriteStr(
          167  +  OomTest *pOom,
          168  +  lsm_db *pDb,
          169  +  const char *zKey,
          170  +  const char *zVal,
          171  +  int *pRc
          172  +){
          173  +  int nKey = strlen(zKey);
          174  +  int nVal = strlen(zVal);
          175  +  testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
          176  +}
          177  +
          178  +static void testOomWriteData(
          179  +  OomTest *pOom,
          180  +  lsm_db *pDb,
          181  +  Datasource *pData,
          182  +  int iKey,
          183  +  int *pRc
          184  +){
          185  +  void *pKey; int nKey;
          186  +  void *pVal; int nVal;
          187  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          188  +  testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
          189  +}
          190  +
          191  +static void testOomScan(
          192  +  OomTest *pOom, 
          193  +  lsm_db *pDb, 
          194  +  int bReverse,
          195  +  const void *pKey, int nKey,
          196  +  int nScan,
          197  +  int *pRc
          198  +){
          199  +  if( *pRc==0 ){
          200  +    int rc;
          201  +    int iScan = 0;
          202  +    lsm_cursor *pCsr;
          203  +    int (*xAdvance)(lsm_cursor *) = 0;
          204  +    
          205  +
          206  +    rc = lsm_csr_open(pDb, &pCsr);
          207  +    testOomAssertRc(pOom, rc);
          208  +
          209  +    if( rc==LSM_OK ){
          210  +      if( bReverse ){
          211  +        rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE);
          212  +        xAdvance = lsm_csr_prev;
          213  +      }else{
          214  +        rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE);
          215  +        xAdvance = lsm_csr_next;
          216  +      }
          217  +    }
          218  +    testOomAssertRc(pOom, rc);
          219  +
          220  +    while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan<nScan ){
          221  +      const void *p; int n;
          222  +
          223  +      rc = lsm_csr_key(pCsr, &p, &n);
          224  +      testOomAssertRc(pOom, rc);
          225  +      if( rc==LSM_OK ){
          226  +        rc = lsm_csr_value(pCsr, &p, &n);
          227  +        testOomAssertRc(pOom, rc);
          228  +      }
          229  +      if( rc==LSM_OK ){
          230  +        rc = xAdvance(pCsr);
          231  +        testOomAssertRc(pOom, rc);
          232  +      }
          233  +      iScan++;
          234  +    }
          235  +
          236  +    lsm_csr_close(pCsr);
          237  +    *pRc = rc;
          238  +  }
          239  +}
          240  +
          241  +#define LSMTEST6_TESTDB "testdb.lsm" 
          242  +
          243  +void testDeleteLsmdb(const char *zFile){
          244  +  char *zLog = testMallocPrintf("%s-log", zFile);
          245  +  char *zShm = testMallocPrintf("%s-shm", zFile);
          246  +  unlink(zFile);
          247  +  unlink(zLog);
          248  +  unlink(zShm);
          249  +  testFree(zLog);
          250  +  testFree(zShm);
          251  +}
          252  +
          253  +static void copy_file(const char *zFrom, const char *zTo, int isDatabase){
          254  +
          255  +  if( access(zFrom, F_OK) ){
          256  +    unlink(zTo);
          257  +  }else{
          258  +    int fd1;
          259  +    int fd2;
          260  +    off_t sz;
          261  +    off_t i;
          262  +    struct stat buf;
          263  +    u8 *aBuf;
          264  +
          265  +    fd1 = open(zFrom, O_RDONLY | _O_BINARY, 0644);
          266  +    fd2 = open(zTo, O_RDWR | O_CREAT | _O_BINARY, 0644);
          267  +
          268  +    fstat(fd1, &buf);
          269  +    sz = buf.st_size;
          270  +    ftruncate(fd2, sz);
          271  +
          272  +    aBuf = testMalloc(4096);
          273  +    for(i=0; i<sz; i+=4096){
          274  +      int bLockPage = isDatabase && i == 0;
          275  +      int nByte = MIN((bLockPage ? 4066 : 4096), sz - i);
          276  +      memset(aBuf, 0, 4096);
          277  +      read(fd1, aBuf, nByte);
          278  +      write(fd2, aBuf, nByte);
          279  +      if( bLockPage ){
          280  +        lseek(fd1, 4096, SEEK_SET);
          281  +        lseek(fd2, 4096, SEEK_SET);
          282  +      }
          283  +    }
          284  +    testFree(aBuf);
          285  +
          286  +    close(fd1);
          287  +    close(fd2);
          288  +  }
          289  +}
          290  +
          291  +void testCopyLsmdb(const char *zFrom, const char *zTo){
          292  +  char *zLog1 = testMallocPrintf("%s-log", zFrom);
          293  +  char *zLog2 = testMallocPrintf("%s-log", zTo);
          294  +  char *zShm1 = testMallocPrintf("%s-shm", zFrom);
          295  +  char *zShm2 = testMallocPrintf("%s-shm", zTo);
          296  +
          297  +  unlink(zShm2);
          298  +  unlink(zLog2);
          299  +  unlink(zTo);
          300  +  copy_file(zFrom, zTo, 1);
          301  +  copy_file(zLog1, zLog2, 0);
          302  +  copy_file(zShm1, zShm2, 0);
          303  +
          304  +  testFree(zLog1); testFree(zLog2); testFree(zShm1); testFree(zShm2);
          305  +}
          306  +
          307  +/*
          308  +** File zFile is the path to a database. This function makes backups
          309  +** of the database file and its log as follows:
          310  +**
          311  +**     cp $(zFile)         $(zFile)-save
          312  +**     cp $(zFile)-$(zAux) $(zFile)-save-$(zAux)
          313  +**
          314  +** Function testRestoreDb() can be used to copy the files back in the
          315  +** other direction.
          316  +*/
          317  +void testSaveDb(const char *zFile, const char *zAux){
          318  +  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
          319  +  char *zFileSave = testMallocPrintf("%s-save", zFile);
          320  +  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
          321  +
          322  +  unlink(zFileSave);
          323  +  unlink(zLogSave);
          324  +  copy_file(zFile, zFileSave, 1);
          325  +  copy_file(zLog, zLogSave, 0);
          326  +
          327  +  testFree(zLog); testFree(zFileSave); testFree(zLogSave);
          328  +}
          329  +
          330  +/*
          331  +** File zFile is the path to a database. This function restores
          332  +** a backup of the database made by a previous call to testSaveDb().
          333  +** Specifically, it does the equivalent of:
          334  +**
          335  +**     cp $(zFile)-save         $(zFile)
          336  +**     cp $(zFile)-save-$(zAux) $(zFile)-$(zAux)
          337  +*/
          338  +void testRestoreDb(const char *zFile, const char *zAux){
          339  +  char *zLog = testMallocPrintf("%s-%s", zFile, zAux);
          340  +  char *zFileSave = testMallocPrintf("%s-save", zFile);
          341  +  char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux);
          342  +
          343  +  copy_file(zFileSave, zFile, 1);
          344  +  copy_file(zLogSave, zLog, 0);
          345  +
          346  +  testFree(zLog); testFree(zFileSave); testFree(zLogSave);
          347  +}
          348  +
          349  +
          350  +static int lsmWriteStr(lsm_db *pDb, const char *zKey, const char *zVal){
          351  +  int nKey = strlen(zKey);
          352  +  int nVal = strlen(zVal);
          353  +  return lsm_insert(pDb, (void *)zKey, nKey, (void *)zVal, nVal);
          354  +}
          355  +
          356  +static void setup_delete_db(void){
          357  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          358  +}
          359  +
          360  +/*
          361  +** Create a small database. With the following content:
          362  +**
          363  +**    "one"   -> "one"
          364  +**    "two"   -> "four"
          365  +**    "three" -> "nine"
          366  +**    "four"  -> "sixteen"
          367  +**    "five"  -> "twentyfive"
          368  +**    "six"   -> "thirtysix"
          369  +**    "seven" -> "fourtynine"
          370  +**    "eight" -> "sixtyfour"
          371  +*/
          372  +static void setup_populate_db(void){
          373  +  const char *azStr[] = {
          374  +    "one",   "one",
          375  +    "two",   "four",
          376  +    "three", "nine",
          377  +    "four",  "sixteen",
          378  +    "five",  "twentyfive",
          379  +    "six",   "thirtysix",
          380  +    "seven", "fourtynine",
          381  +    "eight", "sixtyfour",
          382  +  };
          383  +  int rc;
          384  +  int ii;
          385  +  lsm_db *pDb;
          386  +
          387  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          388  +
          389  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          390  +  if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB);
          391  +
          392  +  for(ii=0; rc==LSM_OK && ii<ArraySize(azStr); ii+=2){
          393  +    rc = lsmWriteStr(pDb, azStr[ii], azStr[ii+1]);
          394  +  }
          395  +  lsm_close(pDb);
          396  +
          397  +  testSaveDb(LSMTEST6_TESTDB, "log");
          398  +  assert( rc==LSM_OK );
          399  +}
          400  +
          401  +static Datasource *getDatasource(void){
          402  +  const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 };
          403  +  return testDatasourceNew(&defn);
          404  +}
          405  +
          406  +/*
          407  +** Set up a database file with the following properties:
          408  +**
          409  +**   * Page size is 1024 bytes.
          410  +**   * Block size is 64 KB.
          411  +**   * Contains 5000 key-value pairs starting at 0 from the
          412  +**     datasource returned getDatasource().
          413  +*/
          414  +static void setup_populate_db2(void){
          415  +  Datasource *pData;
          416  +  int ii;
          417  +  int rc;
          418  +  int nBlocksize = 64*1024;
          419  +  int nPagesize = 1024;
          420  +  int nWritebuffer = 4*1024;
          421  +  lsm_db *pDb;
          422  +
          423  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          424  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          425  +  if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB);
          426  +
          427  +  lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &nBlocksize); 
          428  +  lsm_config(pDb, LSM_CONFIG_PAGE_SIZE, &nPagesize); 
          429  +  lsm_config(pDb, LSM_CONFIG_AUTOFLUSH, &nWritebuffer); 
          430  +
          431  +  pData = getDatasource();
          432  +  for(ii=0; rc==LSM_OK && ii<5000; ii++){
          433  +    void *pKey; int nKey;
          434  +    void *pVal; int nVal;
          435  +    testDatasourceEntry(pData, ii, &pKey, &nKey, &pVal, &nVal);
          436  +    lsm_insert(pDb, pKey, nKey, pVal, nVal);
          437  +  }
          438  +  testDatasourceFree(pData);
          439  +  lsm_close(pDb);
          440  +
          441  +  testSaveDb(LSMTEST6_TESTDB, "log");
          442  +  assert( rc==LSM_OK );
          443  +}
          444  +
          445  +/*
          446  +** Test the results of OOM conditions in lsm_new().
          447  +*/
          448  +static void simple_oom_1(OomTest *pOom){
          449  +  int rc;
          450  +  lsm_db *pDb;
          451  +
          452  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          453  +  testOomAssertRc(pOom, rc);
          454  +
          455  +  lsm_close(pDb);
          456  +}
          457  +
          458  +/*
          459  +** Test the results of OOM conditions in lsm_open().
          460  +*/
          461  +static void simple_oom_2(OomTest *pOom){
          462  +  int rc;
          463  +  lsm_db *pDb;
          464  +
          465  +  rc = lsm_new(tdb_lsm_env(), &pDb);
          466  +  if( rc==LSM_OK ){
          467  +    rc = lsm_open(pDb, "testdb.lsm");
          468  +  }
          469  +  testOomAssertRc(pOom, rc);
          470  +
          471  +  lsm_close(pDb);
          472  +}
          473  +
          474  +/*
          475  +** Test the results of OOM conditions in simple fetch operations.
          476  +*/
          477  +static void simple_oom_3(OomTest *pOom){
          478  +  int rc = LSM_OK;
          479  +  lsm_db *pDb;
          480  +
          481  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          482  +
          483  +  testOomFetchStr(pOom, pDb, "four",  "sixteen",    &rc);
          484  +  testOomFetchStr(pOom, pDb, "seven", "fourtynine", &rc);
          485  +  testOomFetchStr(pOom, pDb, "one",   "one",        &rc);
          486  +  testOomFetchStr(pOom, pDb, "eight", "sixtyfour",  &rc);
          487  +
          488  +  lsm_close(pDb);
          489  +}
          490  +
          491  +/*
          492  +** Test the results of OOM conditions in simple write operations.
          493  +*/
          494  +static void simple_oom_4(OomTest *pOom){
          495  +  int rc = LSM_OK;
          496  +  lsm_db *pDb;
          497  +
          498  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          499  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          500  +
          501  +  testOomWriteStr(pOom, pDb, "123", "onetwothree", &rc);
          502  +  testOomWriteStr(pOom, pDb, "456", "fourfivesix", &rc);
          503  +  testOomWriteStr(pOom, pDb, "789", "seveneightnine", &rc);
          504  +  testOomWriteStr(pOom, pDb, "123", "teneleventwelve", &rc);
          505  +  testOomWriteStr(pOom, pDb, "456", "fourteenfifteensixteen", &rc);
          506  +
          507  +  lsm_close(pDb);
          508  +}
          509  +
          510  +static void simple_oom_5(OomTest *pOom){
          511  +  Datasource *pData = getDatasource();
          512  +  int rc = LSM_OK;
          513  +  lsm_db *pDb;
          514  +
          515  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          516  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          517  +
          518  +  testOomFetchData(pOom, pDb, pData, 3333, &rc);
          519  +  testOomFetchData(pOom, pDb, pData, 0, &rc);
          520  +  testOomFetchData(pOom, pDb, pData, 4999, &rc);
          521  +
          522  +  lsm_close(pDb);
          523  +  testDatasourceFree(pData);
          524  +}
          525  +
          526  +static void simple_oom_6(OomTest *pOom){
          527  +  Datasource *pData = getDatasource();
          528  +  int rc = LSM_OK;
          529  +  lsm_db *pDb;
          530  +
          531  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          532  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          533  +
          534  +  testOomWriteData(pOom, pDb, pData, 5000, &rc);
          535  +  testOomWriteData(pOom, pDb, pData, 5001, &rc);
          536  +  testOomWriteData(pOom, pDb, pData, 5002, &rc);
          537  +  testOomFetchData(pOom, pDb, pData, 5001, &rc);
          538  +  testOomFetchData(pOom, pDb, pData, 1234, &rc);
          539  +
          540  +  lsm_close(pDb);
          541  +  testDatasourceFree(pData);
          542  +}
          543  +
          544  +static void simple_oom_7(OomTest *pOom){
          545  +  Datasource *pData = getDatasource();
          546  +  int rc = LSM_OK;
          547  +  lsm_db *pDb;
          548  +
          549  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          550  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          551  +  testOomScan(pOom, pDb, 0, "abc", 3, 20, &rc);
          552  +  lsm_close(pDb);
          553  +  testDatasourceFree(pData);
          554  +}
          555  +
          556  +static void simple_oom_8(OomTest *pOom){
          557  +  Datasource *pData = getDatasource();
          558  +  int rc = LSM_OK;
          559  +  lsm_db *pDb;
          560  +  testRestoreDb(LSMTEST6_TESTDB, "log");
          561  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc);
          562  +  testOomScan(pOom, pDb, 1, "xyz", 3, 20, &rc);
          563  +  lsm_close(pDb);
          564  +  testDatasourceFree(pData);
          565  +}
          566  +
          567  +/*
          568  +** This test case has two clients connected to a database. The first client
          569  +** hits an OOM while writing to the database. Check that the second 
          570  +** connection is still able to query the db following the OOM.
          571  +*/
          572  +static void simple_oom2_1(OomTest *pOom){
          573  +  const int nRecord = 100;        /* Number of records initially in db */
          574  +  const int nIns = 10;            /* Number of records inserted with OOM */
          575  +
          576  +  Datasource *pData = getDatasource();
          577  +  int rc = LSM_OK;
          578  +  lsm_db *pDb1;
          579  +  lsm_db *pDb2;
          580  +  int i;
          581  +
          582  +  testDeleteLsmdb(LSMTEST6_TESTDB);
          583  +
          584  +  /* Open the two connections. Initialize the in-memory tree so that it
          585  +  ** contains 100 records. Do all this with OOM injection disabled. */
          586  +  testOomEnable(pOom, 0);
          587  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb1, &rc);
          588  +  testOomOpen(pOom, LSMTEST6_TESTDB, &pDb2, &rc);
          589  +  for(i=0; i<nRecord; i++){
          590  +    testOomWriteData(pOom, pDb1, pData, i, &rc);
          591  +  }
          592  +  testOomEnable(pOom, 1);
          593  +  assert( rc==0 );
          594  +
          595  +  /* Insert 10 more records using pDb1. Stop when an OOM is encountered. */
          596  +  for(i=nRecord; i<nRecord+nIns; i++){
          597  +    testOomWriteData(pOom, pDb1, pData, i, &rc);
          598  +    if( rc ) break;
          599  +  }
          600  +  testOomAssertRc(pOom, rc);
          601  +
          602  +  /* Switch off OOM injection. Write a few rows using pDb2. Then check
          603  +  ** that the database may be successfully queried.  */
          604  +  testOomEnable(pOom, 0);
          605  +  rc = 0;
          606  +  for(; i<nRecord+nIns && rc==0; i++){
          607  +    testOomWriteData(pOom, pDb2, pData, i, &rc);
          608  +  }
          609  +  for(i=0; i<nRecord+nIns; i++) testOomFetchData(pOom, pDb2, pData, i, &rc);
          610  +  testOomEnable(pOom, 1);
          611  +
          612  +  lsm_close(pDb1);
          613  +  lsm_close(pDb2);
          614  +  testDatasourceFree(pData);
          615  +}
          616  +
          617  +
          618  +static void do_test_oom1(const char *zPattern, int *pRc){
          619  +  struct SimpleOom {
          620  +    const char *zName;
          621  +    void (*xSetup)(void);
          622  +    void (*xFunc)(OomTest *);
          623  +  } aSimple[] = {
          624  +    { "oom1.lsm.1", setup_delete_db,    simple_oom_1 },
          625  +    { "oom1.lsm.2", setup_delete_db,    simple_oom_2 },
          626  +    { "oom1.lsm.3", setup_populate_db,  simple_oom_3 },
          627  +    { "oom1.lsm.4", setup_delete_db,    simple_oom_4 },
          628  +    { "oom1.lsm.5", setup_populate_db2, simple_oom_5 },
          629  +    { "oom1.lsm.6", setup_populate_db2, simple_oom_6 },
          630  +    { "oom1.lsm.7", setup_populate_db2, simple_oom_7 },
          631  +    { "oom1.lsm.8", setup_populate_db2, simple_oom_8 },
          632  +
          633  +    { "oom2.lsm.1", setup_delete_db,    simple_oom2_1 },
          634  +  };
          635  +  int i;
          636  +
          637  +  for(i=0; i<ArraySize(aSimple); i++){
          638  +    if( *pRc==0 && testCaseBegin(pRc, zPattern, "%s", aSimple[i].zName) ){
          639  +      OomTest t;
          640  +
          641  +      if( aSimple[i].xSetup ){
          642  +        aSimple[i].xSetup();
          643  +      }
          644  +
          645  +      for(testOomStart(&t); testOomContinue(&t); testOomNext(&t)){
          646  +        aSimple[i].xFunc(&t);
          647  +      }
          648  +
          649  +      printf("(%d injections).", t.iNext-2);
          650  +      testCaseFinish( (*pRc = testOomFinish(&t)) );
          651  +      testMallocOom(tdb_lsm_env(), 0, 0, 0, 0);
          652  +    }
          653  +  }
          654  +}
          655  +
          656  +void test_oom(
          657  +  const char *zPattern,           /* Run test cases that match this pattern */
          658  +  int *pRc                        /* IN/OUT: Error code */
          659  +){
          660  +  do_test_oom1(zPattern, pRc);
          661  +}

Added ext/lsm1/lsm-test/lsmtest7.c.

            1  +
            2  +
            3  +#include "lsmtest.h"
            4  +
            5  +
            6  +/*
            7  +** Test that the rules for when lsm_csr_next() and lsm_csr_prev() are
            8  +** enforced. Specifically:
            9  +**
           10  +**   * Both functions always return LSM_MISUSE if the cursor is at EOF
           11  +**     when they are called.
           12  +**
           13  +**   * lsm_csr_next() may only be used after lsm_csr_seek(LSM_SEEK_GE) or 
           14  +**     lsm_csr_first(). 
           15  +**
           16  +**   * lsm_csr_prev() may only be used after lsm_csr_seek(LSM_SEEK_LE) or 
           17  +**     lsm_csr_last().
           18  +*/
           19  +static void do_test_api1_lsm(lsm_db *pDb, int *pRc){
           20  +  int ret;
           21  +  lsm_cursor *pCsr;
           22  +  lsm_cursor *pCsr2;
           23  +  int nKey;
           24  +  const void *pKey;
           25  +
           26  +  ret = lsm_csr_open(pDb, &pCsr);
           27  +  testCompareInt(LSM_OK, ret, pRc);
           28  +
           29  +  ret = lsm_csr_next(pCsr);
           30  +  testCompareInt(LSM_MISUSE, ret, pRc);
           31  +  ret = lsm_csr_prev(pCsr);
           32  +  testCompareInt(LSM_MISUSE, ret, pRc);
           33  +
           34  +  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_GE);
           35  +  testCompareInt(LSM_OK, ret, pRc);
           36  +  ret = lsm_csr_next(pCsr);
           37  +  testCompareInt(LSM_OK, ret, pRc);
           38  +  ret = lsm_csr_prev(pCsr);
           39  +  testCompareInt(LSM_MISUSE, ret, pRc);
           40  +
           41  +  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LE);
           42  +  testCompareInt(LSM_OK, ret, pRc);
           43  +  ret = lsm_csr_next(pCsr);
           44  +  testCompareInt(LSM_MISUSE, ret, pRc);
           45  +  ret = lsm_csr_prev(pCsr);
           46  +  testCompareInt(LSM_OK, ret, pRc);
           47  +
           48  +  ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LEFAST);
           49  +  testCompareInt(LSM_OK, ret, pRc);
           50  +  ret = lsm_csr_next(pCsr);
           51  +  testCompareInt(LSM_MISUSE, ret, pRc);
           52  +  ret = lsm_csr_prev(pCsr);
           53  +  testCompareInt(LSM_MISUSE, ret, pRc);
           54  +
           55  +  ret = lsm_csr_key(pCsr, &pKey, &nKey);
           56  +  testCompareInt(LSM_OK, ret, pRc);
           57  +
           58  +  ret = lsm_csr_open(pDb, &pCsr2);
           59  +  testCompareInt(LSM_OK, ret, pRc);
           60  +
           61  +  ret = lsm_csr_seek(pCsr2, pKey, nKey, LSM_SEEK_EQ);
           62  +  testCompareInt(LSM_OK, ret, pRc);
           63  +  testCompareInt(1, lsm_csr_valid(pCsr2), pRc);
           64  +  ret = lsm_csr_next(pCsr2);
           65  +  testCompareInt(LSM_MISUSE, ret, pRc);
           66  +  ret = lsm_csr_prev(pCsr2);
           67  +  testCompareInt(LSM_MISUSE, ret, pRc);
           68  +
           69  +  lsm_csr_close(pCsr2);
           70  +
           71  +  ret = lsm_csr_first(pCsr);
           72  +  testCompareInt(LSM_OK, ret, pRc);
           73  +  ret = lsm_csr_next(pCsr);
           74  +  testCompareInt(LSM_OK, ret, pRc);
           75  +  ret = lsm_csr_prev(pCsr);
           76  +  testCompareInt(LSM_MISUSE, ret, pRc);
           77  +
           78  +  ret = lsm_csr_last(pCsr);
           79  +  testCompareInt(LSM_OK, ret, pRc);
           80  +  ret = lsm_csr_prev(pCsr);
           81  +  testCompareInt(LSM_OK, ret, pRc);
           82  +  ret = lsm_csr_next(pCsr);
           83  +  testCompareInt(LSM_MISUSE, ret, pRc);
           84  +
           85  +  ret = lsm_csr_first(pCsr);
           86  +  while( lsm_csr_valid(pCsr) ){
           87  +    ret = lsm_csr_next(pCsr);
           88  +    testCompareInt(LSM_OK, ret, pRc);
           89  +  }
           90  +  ret = lsm_csr_next(pCsr);
           91  +  testCompareInt(LSM_OK, ret, pRc);
           92  +  ret = lsm_csr_prev(pCsr);
           93  +  testCompareInt(LSM_MISUSE, ret, pRc);
           94  +
           95  +  ret = lsm_csr_last(pCsr);
           96  +  while( lsm_csr_valid(pCsr) ){
           97  +    ret = lsm_csr_prev(pCsr);
           98  +    testCompareInt(LSM_OK, ret, pRc);
           99  +  }
          100  +  ret = lsm_csr_prev(pCsr);
          101  +  testCompareInt(LSM_OK, ret, pRc);
          102  +  ret = lsm_csr_next(pCsr);
          103  +  testCompareInt(LSM_MISUSE, ret, pRc);
          104  +
          105  +  lsm_csr_close(pCsr);
          106  +}
          107  +
          108  +static void do_test_api1(const char *zPattern, int *pRc){
          109  +  if( testCaseBegin(pRc, zPattern, "api1.lsm") ){
          110  +    const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 };
          111  +    Datasource *pData;
          112  +    TestDb *pDb;
          113  +    int rc = 0;
          114  +
          115  +    pDb = testOpen("lsm_lomem", 1, &rc);
          116  +    pData = testDatasourceNew(&defn);
          117  +    testWriteDatasourceRange(pDb, pData, 0, 1000, pRc);
          118  +
          119  +    do_test_api1_lsm(tdb_lsm(pDb), pRc);
          120  +
          121  +    testDatasourceFree(pData);
          122  +    testClose(&pDb);
          123  +
          124  +    testCaseFinish(*pRc);
          125  +  }
          126  +}
          127  +
          128  +static lsm_db *newLsmConnection(
          129  +  const char *zDb, 
          130  +  int nPgsz, 
          131  +  int nBlksz,
          132  +  int *pRc
          133  +){
          134  +  lsm_db *db = 0;
          135  +  if( *pRc==0 ){
          136  +    int n1 = nPgsz;
          137  +    int n2 = nBlksz;
          138  +    *pRc = lsm_new(tdb_lsm_env(), &db);
          139  +    if( *pRc==0 ){
          140  +      if( n1 ) lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1);
          141  +      if( n2 ) lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2);
          142  +      *pRc = lsm_open(db, "testdb.lsm");
          143  +    }
          144  +  }
          145  +  return db;
          146  +}
          147  +
          148  +static void testPagesize(lsm_db *db, int nPgsz, int nBlksz, int *pRc){
          149  +  if( *pRc==0 ){
          150  +    int n1 = 0;
          151  +    int n2 = 0;
          152  +
          153  +    lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1);
          154  +    lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2);
          155  +
          156  +    testCompareInt(n1, nPgsz, pRc);
          157  +    testCompareInt(n2, nBlksz, pRc);
          158  +  }
          159  +}
          160  +
          161  +/*
          162  +** Test case "api2" tests that the default page and block sizes of a 
          163  +** database may only be modified before lsm_open() is called. And that
          164  +** after lsm_open() is called lsm_config() may be used to read the 
          165  +** actual page and block size of the db.
          166  +*/
          167  +static void do_test_api2(const char *zPattern, int *pRc){
          168  +  if( *pRc==0 && testCaseBegin(pRc, zPattern, "api2.lsm") ){
          169  +    lsm_db *db1 = 0;
          170  +    lsm_db *db2 = 0;
          171  +
          172  +    testDeleteLsmdb("testdb.lsm");
          173  +    db1 = newLsmConnection("testdb.lsm", 0, 0, pRc);
          174  +    testPagesize(db1, 4096, 1024, pRc);
          175  +    db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
          176  +    testPagesize(db2, 4096, 1024, pRc);
          177  +    lsm_close(db1);
          178  +    lsm_close(db2);
          179  +
          180  +    testDeleteLsmdb("testdb.lsm");
          181  +    db1 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
          182  +    testPagesize(db1, 1024, 64*1024, pRc);
          183  +    db2 = newLsmConnection("testdb.lsm", 0, 0, pRc);
          184  +    testPagesize(db2, 1024, 64*1024, pRc);
          185  +    lsm_close(db1);
          186  +    lsm_close(db2);
          187  +
          188  +    testDeleteLsmdb("testdb.lsm");
          189  +    db1 = newLsmConnection("testdb.lsm", 8192, 2*1024, pRc);
          190  +    testPagesize(db1, 8192, 2*1024, pRc);
          191  +    db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc);
          192  +    testPagesize(db2, 8192, 2*1024, pRc);
          193  +    lsm_close(db1);
          194  +    lsm_close(db2);
          195  +
          196  +    testCaseFinish(*pRc);
          197  +  }
          198  +}
          199  +
          200  +void test_api(
          201  +  const char *zPattern,           /* Run test cases that match this pattern */
          202  +  int *pRc                        /* IN/OUT: Error code */
          203  +){
          204  +  do_test_api1(zPattern, pRc);
          205  +  do_test_api2(zPattern, pRc);
          206  +}

Added ext/lsm1/lsm-test/lsmtest8.c.

            1  +
            2  +/*
            3  +** This file contains test cases to verify that "live-recovery" following
            4  +** a mid-transaction failure of a writer process.
            5  +*/
            6  +
            7  +
            8  +/* 
            9  +** This test file includes lsmInt.h to get access to the definition of the
           10  +** ShmHeader structure. This is required to cause strategic damage to the
           11  +** shared memory header as part of recovery testing.
           12  +*/
           13  +#include "lsmInt.h"
           14  +
           15  +#include "lsmtest.h"
           16  +
           17  +typedef struct SetupStep SetupStep;
           18  +struct SetupStep {
           19  +  int bFlush;                     /* Flush to disk and checkpoint */
           20  +  int iInsStart;                  /* First key-value from ds to insert */
           21  +  int nIns;                       /* Number of rows to insert */
           22  +  int iDelStart;                  /* First key from ds to delete */
           23  +  int nDel;                       /* Number of rows to delete */
           24  +};
           25  +
           26  +static void doSetupStep(
           27  +  TestDb *pDb, 
           28  +  Datasource *pData, 
           29  +  const SetupStep *pStep, 
           30  +  int *pRc
           31  +){
           32  +  testWriteDatasourceRange(pDb, pData, pStep->iInsStart, pStep->nIns, pRc);
           33  +  testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc);
           34  +  if( *pRc==0 ){
           35  +    int nSave = -1;
           36  +    int nBuf = 64;
           37  +    lsm_db *db = tdb_lsm(pDb);
           38  +
           39  +    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave);
           40  +    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf);
           41  +    lsm_begin(db, 1);
           42  +    lsm_commit(db, 0);
           43  +    lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave);
           44  +
           45  +    *pRc = lsm_work(db, 0, 0, 0);
           46  +    if( *pRc==0 ){
           47  +      *pRc = lsm_checkpoint(db, 0);
           48  +    }
           49  +  }
           50  +}
           51  +
           52  +static void doSetupStepArray(
           53  +  TestDb *pDb, 
           54  +  Datasource *pData, 
           55  +  const SetupStep *aStep, 
           56  +  int nStep
           57  +){
           58  +  int i;
           59  +  for(i=0; i<nStep; i++){
           60  +    int rc = 0;
           61  +    doSetupStep(pDb, pData, &aStep[i], &rc);
           62  +    assert( rc==0 );
           63  +  }
           64  +}
           65  +
           66  +static void setupDatabase1(TestDb *pDb, Datasource **ppData){
           67  +  const SetupStep aStep[] = {
           68  +    { 0,                                  1,     2000, 0, 0 },
           69  +    { 1,                                  0,     0, 0, 0 },
           70  +    { 0,                                  10001, 1000, 0, 0 },
           71  +  };
           72  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 100, 500};
           73  +  Datasource *pData;
           74  +
           75  +  pData = testDatasourceNew(&defn);
           76  +  doSetupStepArray(pDb, pData, aStep, ArraySize(aStep));
           77  +  if( ppData ){
           78  +    *ppData = pData;
           79  +  }else{
           80  +    testDatasourceFree(pData);
           81  +  }
           82  +}
           83  +
           84  +#include <stdio.h>
           85  +void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){
           86  +  if( *pRc==0 ){
           87  +    FILE *fd;
           88  +    fd = fopen(zFile, "rb");
           89  +    if( fd==0 ){
           90  +      *pRc = 1;
           91  +    }else{
           92  +      if( 0!=fseek(fd, iOff, SEEK_SET) ){
           93  +        *pRc = 1;
           94  +      }else{
           95  +        assert( nByte>=0 );
           96  +        if( (size_t)nByte!=fread(pOut, 1, nByte, fd) ){
           97  +          *pRc = 1;
           98  +        }
           99  +      }
          100  +      fclose(fd);
          101  +    }
          102  +  }
          103  +}
          104  +
          105  +void testWriteFile(
          106  +  const char *zFile, 
          107  +  int iOff, 
          108  +  void *pOut, 
          109  +  int nByte, 
          110  +  int *pRc
          111  +){
          112  +  if( *pRc==0 ){
          113  +    FILE *fd;
          114  +    fd = fopen(zFile, "r+b");
          115  +    if( fd==0 ){
          116  +      *pRc = 1;
          117  +    }else{
          118  +      if( 0!=fseek(fd, iOff, SEEK_SET) ){
          119  +        *pRc = 1;
          120  +      }else{
          121  +        assert( nByte>=0 );
          122  +        if( (size_t)nByte!=fwrite(pOut, 1, nByte, fd) ){
          123  +          *pRc = 1;
          124  +        }
          125  +      }
          126  +      fclose(fd);
          127  +    }
          128  +  }
          129  +}
          130  +
          131  +static ShmHeader *getShmHeader(const char *zDb){
          132  +  int rc = 0;
          133  +  char *zShm = testMallocPrintf("%s-shm", zDb);
          134  +  ShmHeader *pHdr;
          135  +
          136  +  pHdr = testMalloc(sizeof(ShmHeader));
          137  +  testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc);
          138  +  assert( rc==0 );
          139  +
          140  +  return pHdr;
          141  +}
          142  +
          143  +/*
          144  +** This function makes a copy of the three files associated with LSM 
          145  +** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db",
          146  +** "test.db-log" and "test.db-shm").
          147  +**
          148  +** It then opens a new database connection to the copy with the xLock() call
          149  +** instrumented so that it appears that some other process already connected
          150  +** to the db (holding a shared lock on DMS2). This prevents recovery from
          151  +** running. Then:
          152  +**
          153  +**    1) Check that the checksum of the database is zCksum. 
          154  +**    2) Write a few keys to the database. Then delete the same keys. 
          155  +**    3) Check that the checksum is zCksum.
          156  +**    4) Flush the db to disk and run a checkpoint. 
          157  +**    5) Check once more that the checksum is still zCksum.
          158  +*/
          159  +static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){
          160  +  if( *pRc==LSM_OK ){
          161  +    const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500};
          162  +    Datasource *pData;
          163  +    const char *zCopy = "testcopy.lsm";
          164  +    char zCksum2[TEST_CKSUM_BYTES];
          165  +    TestDb *pDb = 0;
          166  +    int rc;
          167  +
          168  +    pData = testDatasourceNew(&defn);
          169  +
          170  +    testCopyLsmdb(zDb, zCopy);
          171  +    rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb);
          172  +    if( rc==0 ){
          173  +      ShmHeader *pHdr;
          174  +      lsm_db *db;
          175  +      testCksumDatabase(pDb, zCksum2);
          176  +      testCompareStr(zCksum, zCksum2, &rc);
          177  +
          178  +      testWriteDatasourceRange(pDb, pData, 1, 10, &rc);
          179  +      testDeleteDatasourceRange(pDb, pData, 1, 10, &rc);
          180  +
          181  +      /* Test that the two tree-headers are now consistent. */
          182  +      pHdr = getShmHeader(zCopy);
          183  +      if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){
          184  +        rc = 1;
          185  +      }
          186  +      testFree(pHdr);
          187  +
          188  +      if( rc==0 ){
          189  +        int nBuf = 64;
          190  +        db = tdb_lsm(pDb);
          191  +        lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf);
          192  +        lsm_begin(db, 1);
          193  +        lsm_commit(db, 0);
          194  +        rc = lsm_work(db, 0, 0, 0);
          195  +      }
          196  +
          197  +      testCksumDatabase(pDb, zCksum2);
          198  +      testCompareStr(zCksum, zCksum2, &rc);
          199  +    }
          200  +
          201  +    testDatasourceFree(pData);
          202  +    testClose(&pDb);
          203  +    testDeleteLsmdb(zCopy);
          204  +    *pRc = rc;
          205  +  }
          206  +}
          207  +
          208  +static void doWriterCrash1(int *pRc){
          209  +  const int nWrite = 2000;
          210  +  const int nStep = 10;
          211  +  const int iWriteStart = 20000;
          212  +  int rc = 0;
          213  +  TestDb *pDb = 0;
          214  +  Datasource *pData = 0;
          215  +
          216  +  rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb);
          217  +  if( rc==0 ){
          218  +    int iDot = 0;
          219  +    char zCksum[TEST_CKSUM_BYTES];
          220  +    int i;
          221  +    setupDatabase1(pDb, &pData);
          222  +    testCksumDatabase(pDb, zCksum);
          223  +    testBegin(pDb, 2, &rc);
          224  +    for(i=0; rc==0 && i<nWrite; i+=nStep){
          225  +      testCaseProgress(i, nWrite, testCaseNDot(), &iDot);
          226  +      testWriteDatasourceRange(pDb, pData, iWriteStart+i, nStep, &rc);
          227  +      doLiveRecovery("testdb.lsm", zCksum, &rc);
          228  +    }
          229  +  }
          230  +  testCommit(pDb, 0, &rc);
          231  +  testClose(&pDb);
          232  +  testDatasourceFree(pData);
          233  +  *pRc = rc;
          234  +}
          235  +
          236  +/*
          237  +** This test case verifies that inconsistent tree-headers in shared-memory
          238  +** are resolved correctly. 
          239  +*/
          240  +static void doWriterCrash2(int *pRc){
          241  +  int rc = 0;
          242  +  TestDb *pDb = 0;
          243  +  Datasource *pData = 0;
          244  +
          245  +  rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb);
          246  +  if( rc==0 ){
          247  +    ShmHeader *pHdr1;
          248  +    ShmHeader *pHdr2;
          249  +    char zCksum1[TEST_CKSUM_BYTES];
          250  +    char zCksum2[TEST_CKSUM_BYTES];
          251  +
          252  +    pHdr1 = testMalloc(sizeof(ShmHeader));
          253  +    pHdr2 = testMalloc(sizeof(ShmHeader));
          254  +    setupDatabase1(pDb, &pData);
          255  +
          256  +    /* Grab a copy of the shared-memory header. And the db checksum */
          257  +    testReadFile("testdb.lsm-shm", 0, (void *)pHdr1, sizeof(ShmHeader), &rc);
          258  +    testCksumDatabase(pDb, zCksum1);
          259  +
          260  +    /* Modify the database */
          261  +    testBegin(pDb, 2, &rc);
          262  +    testWriteDatasourceRange(pDb, pData, 30000, 200, &rc);
          263  +    testCommit(pDb, 0, &rc);
          264  +
          265  +    /* Grab a second copy of the shared-memory header. And the db checksum */
          266  +    testReadFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          267  +    testCksumDatabase(pDb, zCksum2);
          268  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          269  +
          270  +    /* If both tree-headers are valid, tree-header-1 is used. */
          271  +    memcpy(&pHdr2->hdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1));
          272  +    pHdr2->bWriter = 1;
          273  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          274  +    doLiveRecovery("testdb.lsm", zCksum1, &rc);
          275  +
          276  +    /* If both tree-headers are valid, tree-header-1 is used. */
          277  +    memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1));
          278  +    memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1));
          279  +    pHdr2->bWriter = 1;
          280  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          281  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          282  +
          283  +    /* If tree-header 1 is invalid, tree-header-2 is used */
          284  +    memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1));
          285  +    pHdr2->hdr1.aCksum[0] = 5;
          286  +    pHdr2->hdr1.aCksum[0] = 6;
          287  +    pHdr2->bWriter = 1;
          288  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          289  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          290  +
          291  +    /* If tree-header 2 is invalid, tree-header-1 is used */
          292  +    memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1));
          293  +    pHdr2->hdr2.aCksum[0] = 5;
          294  +    pHdr2->hdr2.aCksum[0] = 6;
          295  +    pHdr2->bWriter = 1;
          296  +    testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc);
          297  +    doLiveRecovery("testdb.lsm", zCksum2, &rc);
          298  +
          299  +    testFree(pHdr1);
          300  +    testFree(pHdr2);
          301  +    testClose(&pDb);
          302  +  }
          303  +
          304  +  *pRc = rc;
          305  +}
          306  +
          307  +void do_writer_crash_test(const char *zPattern, int *pRc){
          308  +  struct Test {
          309  +    const char *zName;
          310  +    void (*xFunc)(int *);
          311  +  } aTest[] = {
          312  +    { "writercrash1.lsm", doWriterCrash1 },
          313  +    { "writercrash2.lsm", doWriterCrash2 },
          314  +  };
          315  +  int i;
          316  +  for(i=0; i<ArraySize(aTest); i++){
          317  +    struct Test *p = &aTest[i];
          318  +    if( testCaseBegin(pRc, zPattern, p->zName) ){
          319  +      p->xFunc(pRc);
          320  +      testCaseFinish(*pRc);
          321  +    }
          322  +  }
          323  +
          324  +}
          325  +
          326  +

Added ext/lsm1/lsm-test/lsmtest9.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE
            5  +#define DATA_RANDOM     TEST_DATASOURCE_RANDOM
            6  +
            7  +typedef struct Datatest4 Datatest4;
            8  +
            9  +/*
           10  +** Test overview:
           11  +**
           12  +**   1. Insert (Datatest4.nRec) records into a database.
           13  +**
           14  +**   2. Repeat (Datatest4.nRepeat) times:
           15  +**
           16  +**      2a. Delete 2/3 of the records in the database.
           17  +**
           18  +**      2b. Run lsm_work(nMerge=1).
           19  +**
           20  +**      2c. Insert as many records as were deleted in 2a.
           21  +**
           22  +**      2d. Check database content is as expected.
           23  +**
           24  +**      2e. If (Datatest4.bReopen) is true, close and reopen the database.
           25  +*/
           26  +struct Datatest4 {
           27  +  /* Datasource definition */
           28  +  DatasourceDefn defn;
           29  +
           30  +  int nRec;
           31  +  int nRepeat;
           32  +  int bReopen;
           33  +};
           34  +
           35  +static void doDataTest4(
           36  +  const char *zSystem,            /* Database system to test */
           37  +  Datatest4 *p,                   /* Structure containing test parameters */
           38  +  int *pRc                        /* OUT: Error code */
           39  +){
           40  +  lsm_db *db = 0;
           41  +  TestDb *pDb;
           42  +  TestDb *pControl;
           43  +  Datasource *pData;
           44  +  int i;
           45  +  int rc = 0;
           46  +  int iDot = 0;
           47  +  int bMultiThreaded = 0;         /* True for MT LSM database */
           48  +
           49  +  int nRecOn3 = (p->nRec / 3);
           50  +  int iData = 0;
           51  +
           52  +  /* Start the test case, open a database and allocate the datasource. */
           53  +  rc = testControlDb(&pControl);
           54  +  pDb = testOpen(zSystem, 1, &rc);
           55  +  pData = testDatasourceNew(&p->defn);
           56  +  if( rc==0 ){
           57  +    db = tdb_lsm(pDb);
           58  +    bMultiThreaded = tdb_lsm_multithread(pDb);
           59  +  }
           60  +
           61  +  testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc);
           62  +  testWriteDatasourceRange(pDb,      pData, iData, nRecOn3*3, &rc);
           63  +
           64  +  for(i=0; rc==0 && i<p->nRepeat; i++){
           65  +
           66  +    testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc);
           67  +    testDeleteDatasourceRange(pDb,      pData, iData, nRecOn3*2, &rc);
           68  +
           69  +    if( db ){
           70  +      int nDone;
           71  +#if 0
           72  +      fprintf(stderr, "lsm_work() start...\n"); fflush(stderr);
           73  +#endif
           74  +      do {
           75  +        nDone = 0;
           76  +        rc = lsm_work(db, 1, (1<<30), &nDone);
           77  +      }while( rc==0 && nDone>0 );
           78  +      if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK;
           79  +#if 0 
           80  +      fprintf(stderr, "lsm_work() done...\n"); fflush(stderr);
           81  +#endif
           82  +    }
           83  +
           84  +if( i+1<p->nRepeat ){
           85  +    iData += (nRecOn3*2);
           86  +    testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc);
           87  +    testWriteDatasourceRange(pDb,      pData, iData+nRecOn3, nRecOn3*2, &rc);
           88  +
           89  +    testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc);
           90  +
           91  +    /* If Datatest4.bReopen is true, close and reopen the database */
           92  +    if( p->bReopen ){
           93  +      testReopen(&pDb, &rc);
           94  +      if( rc==0 ) db = tdb_lsm(pDb);
           95  +    }
           96  +}
           97  +
           98  +    /* Update the progress dots... */
           99  +    testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot);
          100  +  }
          101  +
          102  +  testClose(&pDb);
          103  +  testClose(&pControl);
          104  +  testDatasourceFree(pData);
          105  +  testCaseFinish(rc);
          106  +  *pRc = rc;
          107  +}
          108  +
          109  +static char *getName4(const char *zSystem, Datatest4 *pTest){
          110  +  char *zRet;
          111  +  char *zData;
          112  +  zData = testDatasourceName(&pTest->defn);
          113  +  zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", 
          114  +      zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen
          115  +  );
          116  +  testFree(zData);
          117  +  return zRet;
          118  +}
          119  +
          120  +void test_data_4(
          121  +  const char *zSystem,            /* Database system name */
          122  +  const char *zPattern,           /* Run test cases that match this pattern */
          123  +  int *pRc                        /* IN/OUT: Error code */
          124  +){
          125  +  Datatest4 aTest[] = {
          126  +      /* defn,                                 nRec, nRepeat, bReopen */
          127  +    { {DATA_RANDOM,     20,25,     500,600}, 10000,      10,       0   },
          128  +    { {DATA_RANDOM,     20,25,     500,600}, 10000,      10,       1   },
          129  +  };
          130  +
          131  +  int i;
          132  +
          133  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          134  +    char *zName = getName4(zSystem, &aTest[i]);
          135  +    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          136  +      doDataTest4(zSystem, &aTest[i], pRc);
          137  +    }
          138  +    testFree(zName);
          139  +  }
          140  +}
          141  +
          142  +
          143  +

Added ext/lsm1/lsm-test/lsmtest_bt.c.

            1  +
            2  +#include "lsmtest.h"
            3  +#include "bt.h"
            4  +
            5  +int do_bt(int nArg, char **azArg){
            6  +  struct Option {
            7  +    const char *zName;
            8  +    int bPgno;
            9  +    int eOpt;
           10  +  } aOpt [] = { 
           11  +    { "dbhdr",          0, BT_INFO_HDRDUMP },
           12  +    { "filename",       0, BT_INFO_FILENAME },
           13  +    { "block_freelist", 0, BT_INFO_BLOCK_FREELIST },
           14  +    { "page_freelist",  0, BT_INFO_PAGE_FREELIST },
           15  +    { "filename",       0, BT_INFO_FILENAME },
           16  +    { "page",           1, BT_INFO_PAGEDUMP },
           17  +    { "page_ascii",     1, BT_INFO_PAGEDUMP_ASCII },
           18  +    { "leaks",          0, BT_INFO_PAGE_LEAKS },
           19  +    { 0, 0 } 
           20  +  };
           21  +  int iOpt;
           22  +  int rc;
           23  +  bt_info buf;
           24  +  char *zOpt;
           25  +  char *zFile;
           26  +
           27  +  bt_db *db = 0;
           28  +
           29  +  if( nArg<2 ){
           30  +    testPrintUsage("FILENAME OPTION ...");
           31  +    return -1;
           32  +  }
           33  +  zFile = azArg[0];
           34  +  zOpt = azArg[1];
           35  +
           36  +  rc = testArgSelect(aOpt, "option", zOpt, &iOpt);
           37  +  if( rc!=0 ) return rc;
           38  +  if( nArg!=2+aOpt[iOpt].bPgno ){
           39  +    testPrintFUsage("FILENAME %s %s", zOpt, aOpt[iOpt].bPgno ? "PGNO" : "");
           40  +    return -4;
           41  +  }
           42  +
           43  +  rc = sqlite4BtNew(sqlite4_env_default(), 0, &db);
           44  +  if( rc!=SQLITE4_OK ){
           45  +    testPrintError("sqlite4BtNew() failed: %d", rc);
           46  +    return -2;
           47  +  }
           48  +  rc = sqlite4BtOpen(db, zFile);
           49  +  if( rc!=SQLITE4_OK ){
           50  +    testPrintError("sqlite4BtOpen() failed: %d", rc);
           51  +    return -3;
           52  +  }
           53  +
           54  +  buf.eType = aOpt[iOpt].eOpt;
           55  +  buf.pgno = 0;
           56  +  sqlite4_buffer_init(&buf.output, 0);
           57  +
           58  +  if( aOpt[iOpt].bPgno ){
           59  +    buf.pgno = (u32)atoi(azArg[2]);
           60  +  }
           61  +
           62  +  rc = sqlite4BtControl(db, BT_CONTROL_INFO, &buf);
           63  +  if( rc!=SQLITE4_OK ){
           64  +    testPrintError("sqlite4BtControl() failed: %d\n", rc);
           65  +    return -4;
           66  +  }
           67  +
           68  +  printf("%s\n", (char*)buf.output.p);
           69  +  sqlite4_buffer_clear(&buf.output);
           70  +  return 0;
           71  +}
           72  +
           73  +
           74  +
           75  +

Added ext/lsm1/lsm-test/lsmtest_datasource.c.

            1  +
            2  +
            3  +#include "lsmtest.h"
            4  +
            5  +struct Datasource {
            6  +  int eType;
            7  +
            8  +  int nMinKey;
            9  +  int nMaxKey;
           10  +  int nMinVal;
           11  +  int nMaxVal;
           12  +
           13  +  char *aKey;
           14  +  char *aVal;
           15  +};
           16  +
           17  +void testDatasourceEntry(
           18  +  Datasource *p, 
           19  +  int iData, 
           20  +  void **ppKey, int *pnKey,
           21  +  void **ppVal, int *pnVal
           22  +){
           23  +  assert( (ppKey==0)==(pnKey==0) );
           24  +  assert( (ppVal==0)==(pnVal==0) );
           25  +
           26  +  if( ppKey ){
           27  +    int nKey = 0;
           28  +    switch( p->eType ){
           29  +      case TEST_DATASOURCE_RANDOM: {
           30  +        int nRange = (1 + p->nMaxKey - p->nMinKey);
           31  +        nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; 
           32  +        testPrngString((u32)iData, p->aKey, nKey);
           33  +        break;
           34  +      }
           35  +      case TEST_DATASOURCE_SEQUENCE:
           36  +        nKey = sprintf(p->aKey, "%012d", iData);
           37  +        break;
           38  +    }
           39  +    *ppKey = p->aKey;
           40  +    *pnKey = nKey;
           41  +  }
           42  +  if( ppVal ){
           43  +    u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal;
           44  +    testPrngString((u32)~iData, p->aVal, (int)nVal);
           45  +    *ppVal = p->aVal;
           46  +    *pnVal = (int)nVal;
           47  +  }
           48  +}
           49  +
           50  +void testDatasourceFree(Datasource *p){
           51  +  testFree(p);
           52  +}
           53  +
           54  +/*
           55  +** Return a pointer to a nul-terminated string that corresponds to the
           56  +** contents of the datasource-definition passed as the first argument.
           57  +** The caller should eventually free the returned pointer using testFree().
           58  +*/
           59  +char *testDatasourceName(const DatasourceDefn *p){
           60  +  char *zRet;
           61  +  zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)",
           62  +      (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"),
           63  +      p->nMinKey, p->nMaxKey,
           64  +      p->nMinVal, p->nMaxVal
           65  +  );
           66  +  return zRet;
           67  +}
           68  +
           69  +Datasource *testDatasourceNew(const DatasourceDefn *pDefn){
           70  +  Datasource *p;
           71  +  int nMinKey; 
           72  +  int nMaxKey;
           73  +  int nMinVal;
           74  +  int nMaxVal; 
           75  +
           76  +  if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){
           77  +    nMinKey = 128;
           78  +    nMaxKey = 128;
           79  +  }else{
           80  +    nMinKey = MAX(0, pDefn->nMinKey);
           81  +    nMaxKey = MAX(nMinKey, pDefn->nMaxKey);
           82  +  }
           83  +  nMinVal = MAX(0, pDefn->nMinVal);
           84  +  nMaxVal = MAX(nMinVal, pDefn->nMaxVal);
           85  +
           86  +  p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1);
           87  +  p->eType = pDefn->eType;
           88  +  p->nMinKey = nMinKey;
           89  +  p->nMinVal = nMinVal;
           90  +  p->nMaxKey = nMaxKey;
           91  +  p->nMaxVal = nMaxVal;
           92  +  
           93  +  p->aKey = (char *)&p[1];
           94  +  p->aVal = &p->aKey[nMaxKey];
           95  +  return p;
           96  +};

Added ext/lsm1/lsm-test/lsmtest_func.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +
            5  +int do_work(int nArg, char **azArg){
            6  +  struct Option {
            7  +    const char *zName;
            8  +  } aOpt [] = {
            9  +    { "-nmerge" },
           10  +    { "-nkb" },
           11  +    { 0 }
           12  +  };
           13  +
           14  +  lsm_db *pDb;
           15  +  int rc;
           16  +  int i;
           17  +  const char *zDb;
           18  +  int nMerge = 1;
           19  +  int nKB = (1<<30);
           20  +
           21  +  if( nArg==0 ) goto usage;
           22  +  zDb = azArg[nArg-1];
           23  +  for(i=0; i<(nArg-1); i++){
           24  +    int iSel;
           25  +    rc = testArgSelect(aOpt, "option", azArg[i], &iSel);
           26  +    if( rc ) return rc;
           27  +    switch( iSel ){
           28  +      case 0:
           29  +        i++;
           30  +        if( i==(nArg-1) ) goto usage;
           31  +        nMerge = atoi(azArg[i]);
           32  +        break;
           33  +      case 1:
           34  +        i++;
           35  +        if( i==(nArg-1) ) goto usage;
           36  +        nKB = atoi(azArg[i]);
           37  +        break;
           38  +    }
           39  +  }
           40  +
           41  +  rc = lsm_new(0, &pDb);
           42  +  if( rc!=LSM_OK ){
           43  +    testPrintError("lsm_open(): rc=%d\n", rc);
           44  +  }else{
           45  +    rc = lsm_open(pDb, zDb);
           46  +    if( rc!=LSM_OK ){
           47  +      testPrintError("lsm_open(): rc=%d\n", rc);
           48  +    }else{
           49  +      int n = -1;
           50  +      lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n);
           51  +      n = n*2;
           52  +      lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n);
           53  +
           54  +      rc = lsm_work(pDb, nMerge, nKB, 0);
           55  +      if( rc!=LSM_OK ){
           56  +        testPrintError("lsm_work(): rc=%d\n", rc);
           57  +      }
           58  +    }
           59  +  }
           60  +  if( rc==LSM_OK ){
           61  +    rc = lsm_checkpoint(pDb, 0);
           62  +  }
           63  +
           64  +  lsm_close(pDb);
           65  +  return rc;
           66  +
           67  + usage:
           68  +  testPrintUsage("?-optimize? ?-n N? DATABASE");
           69  +  return -1;
           70  +}
           71  +
           72  +
           73  +/*
           74  +**   lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO??
           75  +*/
           76  +int do_show(int nArg, char **azArg){
           77  +  lsm_db *pDb;
           78  +  int rc;
           79  +  const char *zDb;
           80  +
           81  +  int eOpt = LSM_INFO_DB_STRUCTURE;
           82  +  unsigned int iPg = 0;
           83  +  int bConfig = 0;
           84  +  const char *zConfig = "";
           85  +
           86  +  struct Option {
           87  +    const char *zName;
           88  +    int bConfig;
           89  +    int eOpt;
           90  +  } aOpt [] = { 
           91  +    { "array",       0, LSM_INFO_ARRAY_STRUCTURE },
           92  +    { "array-pages", 0, LSM_INFO_ARRAY_PAGES },
           93  +    { "blocksize",   1, LSM_CONFIG_BLOCK_SIZE },
           94  +    { "pagesize",    1, LSM_CONFIG_PAGE_SIZE },
           95  +    { "freelist",    0, LSM_INFO_FREELIST },
           96  +    { "page-ascii",  0, LSM_INFO_PAGE_ASCII_DUMP },
           97  +    { "page-hex",    0, LSM_INFO_PAGE_HEX_DUMP },
           98  +    { 0, 0 } 
           99  +  };
          100  +
          101  +  char *z = 0; 
          102  +  int iDb = 0;                    /* Index of DATABASE in azArg[] */
          103  +
          104  +  /* Check if there is a "-config" option: */
          105  +  if( nArg>2 && strlen(azArg[0])>1 
          106  +   && memcmp(azArg[0], "-config", strlen(azArg[0]))==0
          107  +  ){
          108  +    zConfig = azArg[1];
          109  +    iDb = 2;
          110  +  }
          111  +  if( nArg<(iDb+1) ) goto usage;
          112  +
          113  +  if( nArg>(iDb+1) ){
          114  +    rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt);
          115  +    if( rc!=0 ) return rc;
          116  +    bConfig = aOpt[eOpt].bConfig;
          117  +    eOpt = aOpt[eOpt].eOpt;
          118  +    if( (bConfig==0 && eOpt==LSM_INFO_FREELIST)
          119  +     || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE)
          120  +     || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE)
          121  +    ){
          122  +      if( nArg!=(iDb+2) ) goto usage;
          123  +    }else{
          124  +      if( nArg!=(iDb+3) ) goto usage;
          125  +      iPg = atoi(azArg[iDb+2]);
          126  +    }
          127  +  }
          128  +  zDb = azArg[iDb];
          129  +
          130  +  rc = lsm_new(0, &pDb);
          131  +  tdb_lsm_configure(pDb, zConfig);
          132  +  if( rc!=LSM_OK ){
          133  +    testPrintError("lsm_new(): rc=%d\n", rc);
          134  +  }else{
          135  +    rc = lsm_open(pDb, zDb);
          136  +    if( rc!=LSM_OK ){
          137  +      testPrintError("lsm_open(): rc=%d\n", rc);
          138  +    }
          139  +  }
          140  +
          141  +  if( rc==LSM_OK ){
          142  +    if( bConfig==0 ){
          143  +      switch( eOpt ){
          144  +        case LSM_INFO_DB_STRUCTURE:
          145  +        case LSM_INFO_FREELIST:
          146  +          rc = lsm_info(pDb, eOpt, &z);
          147  +          break;
          148  +        case LSM_INFO_ARRAY_STRUCTURE:
          149  +        case LSM_INFO_ARRAY_PAGES:
          150  +        case LSM_INFO_PAGE_ASCII_DUMP:
          151  +        case LSM_INFO_PAGE_HEX_DUMP:
          152  +          rc = lsm_info(pDb, eOpt, iPg, &z);
          153  +          break;
          154  +        default:
          155  +          assert( !"no chance" );
          156  +      }
          157  +
          158  +      if( rc==LSM_OK ){
          159  +        printf("%s\n", z ? z : "");
          160  +        fflush(stdout);
          161  +      }
          162  +      lsm_free(lsm_get_env(pDb), z);
          163  +    }else{
          164  +      int iRes = -1;
          165  +      lsm_config(pDb, eOpt, &iRes);
          166  +      printf("%d\n", iRes);
          167  +      fflush(stdout);
          168  +    }
          169  +  }
          170  +
          171  +  lsm_close(pDb);
          172  +  return rc;
          173  +
          174  + usage:
          175  +  testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?");
          176  +  return -1;
          177  +}

Added ext/lsm1/lsm-test/lsmtest_io.c.

            1  +
            2  +/*
            3  +** SUMMARY
            4  +**
            5  +**   This file implements the 'io' subcommand of the test program. It is used
            6  +**   for testing the performance of various combinations of write() and fsync()
            7  +**   system calls. All operations occur on a single file, which may or may not
            8  +**   exist when a test is started.
            9  +**
           10  +**   A test consists of a series of commands. Each command is either a write
           11  +**   or an fsync. A write is specified as "<amount>@<offset>", where <amount>
           12  +**   is the amount of data written, and <offset> is the offset of the file
           13  +**   to write to. An <amount> or an <offset> is specified as an integer number
           14  +**   of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of
           15  +**   KB, MB or GB, respectively. An fsync is simply "S". All commands are
           16  +**   case-insensitive.
           17  +**
           18  +**   Example test program:
           19  +**
           20  +**        2M@6M 1492K@4M S 4096@4K S
           21  +**
           22  +**   This program writes 2 MB of data starting at the offset 6MB offset of
           23  +**   the file, followed by 1492 KB of data written at the 4MB offset of the
           24  +**   file, followed by a call to fsync(), a write of 4KB of data at byte
           25  +**   offset 4096, and finally another call to fsync().
           26  +**
           27  +**   Commands may either be specified on the command line (one command per
           28  +**   command line argument) or read from stdin. Commands read from stdin
           29  +**   must be separated by white-space.
           30  +**
           31  +** COMMAND LINE INVOCATION
           32  +**
           33  +**   The sub-command implemented in this file must be invoked with at least
           34  +**   two arguments - the path to the file to write to and the page-size to
           35  +**   use for writing. If there are more than two arguments, then each
           36  +**   subsequent argument is assumed to be a test command. If there are exactly
           37  +**   two arguments, the test commands are read from stdin.
           38  +**
           39  +**   A write command does not result in a single call to system call write().
           40  +**   Instead, the specified region is written sequentially using one or
           41  +**   more calls to write(), each of which writes not more than one page of
           42  +**   data. For example, if the page-size is 4KB, the command "2M@6M" results
           43  +**   in 512 calls to write(), each of which writes 4KB of data.
           44  +**
           45  +** EXAMPLES
           46  +**
           47  +**   Two equivalent examples:
           48  +**
           49  +**     $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S
           50  +**     3544K written in 129 ms
           51  +**     $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 
           52  +**     3544K written in 127 ms
           53  +**
           54  +*/
           55  +
           56  +#include "lsmtest.h"
           57  +
           58  +typedef struct IoContext IoContext;
           59  +
           60  +struct IoContext {
           61  +  int fd;
           62  +  int nWrite;
           63  +};
           64  +
           65  +/*
           66  +** As isspace(3)
           67  +*/
           68  +static int safe_isspace(char c){
           69  +  if( c&0x80) return 0;
           70  +  return isspace(c);
           71  +}
           72  +
           73  +/*
           74  +** As isdigit(3)
           75  +*/
           76  +static int safe_isdigit(char c){
           77  +  if( c&0x80) return 0;
           78  +  return isdigit(c);
           79  +}
           80  +
           81  +static i64 getNextSize(char *zIn, char **pzOut, int *pRc){
           82  +  i64 iRet = 0;
           83  +  if( *pRc==0 ){
           84  +    char *z = zIn;
           85  +
           86  +    if( !safe_isdigit(*z) ){
           87  +      *pRc = 1;
           88  +      return 0;
           89  +    }
           90  +
           91  +    /* Process digits */
           92  +    while( safe_isdigit(*z) ){
           93  +      iRet = iRet*10 + (*z - '0');
           94  +      z++;
           95  +    }
           96  +
           97  +    /* Process suffix */
           98  +    switch( *z ){
           99  +      case 'k': case 'K':
          100  +        iRet = iRet * 1024;
          101  +        z++;
          102  +        break;
          103  +
          104  +      case 'm': case 'M':
          105  +        iRet = iRet * 1024 * 1024;
          106  +        z++;
          107  +        break;
          108  +
          109  +      case 'g': case 'G':
          110  +        iRet = iRet * 1024 * 1024 * 1024;
          111  +        z++;
          112  +        break;
          113  +    }
          114  +
          115  +    if( pzOut ) *pzOut = z;
          116  +  }
          117  +  return iRet;
          118  +}
          119  +
          120  +static int doOneCmd(
          121  +  IoContext *pCtx,
          122  +  u8 *aData,
          123  +  int pgsz,
          124  +  char *zCmd,
          125  +  char **pzOut
          126  +){
          127  +  char c;
          128  +  char *z = zCmd;
          129  +
          130  +  while( safe_isspace(*z) ) z++;
          131  +  c = *z;
          132  +
          133  +  if( c==0 ){
          134  +    if( pzOut ) *pzOut = z;
          135  +    return 0;
          136  +  }
          137  +
          138  +  if( c=='s' || c=='S' ){
          139  +    if( pzOut ) *pzOut = &z[1];
          140  +    return fdatasync(pCtx->fd);
          141  +  }
          142  +
          143  +  if( safe_isdigit(c) ){
          144  +    i64 iOff = 0;
          145  +    int nByte = 0;
          146  +    int rc = 0;
          147  +    int nPg;
          148  +    int iPg;
          149  +
          150  +    nByte = (int)getNextSize(z, &z, &rc);
          151  +    if( rc || *z!='@' ) goto bad_command;
          152  +    z++;
          153  +    iOff = getNextSize(z, &z, &rc);
          154  +    if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command;
          155  +    if( pzOut ) *pzOut = z;
          156  +
          157  +    nPg = (nByte+pgsz-1) / pgsz;
          158  +    lseek(pCtx->fd, (off_t)iOff, SEEK_SET);
          159  +    for(iPg=0; iPg<nPg; iPg++){
          160  +      write(pCtx->fd, aData, pgsz);
          161  +    }
          162  +    pCtx->nWrite += nByte/1024;
          163  +
          164  +    return 0;
          165  +  }
          166  +
          167  + bad_command:
          168  +  testPrintError("unrecognized command: %s", zCmd);
          169  +  return 1;
          170  +}
          171  +
          172  +static int readStdin(char **pzOut){
          173  +  int nAlloc = 128;
          174  +  char *zOut = 0;
          175  +  int nOut = 0;
          176  +
          177  +  while( !feof(stdin) ){
          178  +    int nRead;
          179  +
          180  +    nAlloc = nAlloc*2;
          181  +    zOut = realloc(zOut, nAlloc);
          182  +    nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin);
          183  +
          184  +    if( nRead==0 ) break;
          185  +    nOut += nRead;
          186  +    zOut[nOut] = '\0';
          187  +  }
          188  +
          189  +  *pzOut = zOut;
          190  +  return 0;
          191  +}
          192  +
          193  +int do_io(int nArg, char **azArg){
          194  +  IoContext ctx;
          195  +  int pgsz;
          196  +  char *zFile;
          197  +  char *zPgsz;
          198  +  int i;
          199  +  int rc = 0;
          200  +
          201  +  char *zStdin = 0;
          202  +  char *z;
          203  +
          204  +  u8 *aData;
          205  +
          206  +  memset(&ctx, 0, sizeof(IoContext));
          207  +  if( nArg<2 ){
          208  +    testPrintUsage("FILE PGSZ ?CMD-1 ...?");
          209  +    return -1;
          210  +  }
          211  +  zFile = azArg[0];
          212  +  zPgsz = azArg[1];
          213  +
          214  +  pgsz = (int)getNextSize(zPgsz, 0, &rc);
          215  +  if( pgsz<=0 ){
          216  +    testPrintError("Ridiculous page size: %d", pgsz);
          217  +    return -1;
          218  +  }
          219  +  aData = malloc(pgsz);
          220  +  memset(aData, 0x77, pgsz);
          221  +
          222  +  ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644);
          223  +  if( ctx.fd<0 ){
          224  +    perror("open: ");
          225  +    return -1;
          226  +  }
          227  +
          228  +  if( nArg==2 ){
          229  +    readStdin(&zStdin);
          230  +    testTimeInit();
          231  +    z = zStdin;
          232  +    while( *z && rc==0 ){
          233  +      rc = doOneCmd(&ctx, aData, pgsz, z, &z);
          234  +    }
          235  +  }else{
          236  +    testTimeInit();
          237  +    for(i=2; i<nArg; i++){
          238  +      rc = doOneCmd(&ctx, aData, pgsz, azArg[i], 0);
          239  +    }
          240  +  }
          241  +
          242  +  printf("%dK written in %d ms\n", ctx.nWrite, testTimeGet());
          243  +
          244  +  free(zStdin);
          245  +  close(ctx.fd);
          246  +
          247  +  return 0;
          248  +}

Added ext/lsm1/lsm-test/lsmtest_main.c.

            1  +
            2  +#include "lsmtest.h"
            3  +#include <sqlite3.h>
            4  +
            5  +void test_failed(){ 
            6  +  assert( 0 );
            7  +  return; 
            8  +}
            9  +
           10  +#define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__)
           11  +static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){
           12  +  if( rc ){
           13  +    *pRc = rc;
           14  +    fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
           15  +    test_failed();
           16  +  }
           17  +}
           18  +
           19  +static int lsm_memcmp(u8 *a, u8 *b, int c){
           20  +  int i;
           21  +  for(i=0; i<c; i++){
           22  +    if( a[i]!=b[i] ) return a[i] - b[i];
           23  +  }
           24  +  return 0;
           25  +}
           26  +
           27  +/*
           28  +** A test utility function.
           29  +*/
           30  +void testFetch(
           31  +  TestDb *pDb,                    /* Database handle */
           32  +  void *pKey, int nKey,           /* Key to query database for */
           33  +  void *pVal, int nVal,           /* Expected value */
           34  +  int *pRc                        /* IN/OUT: Error code */
           35  +){
           36  +  if( *pRc==0 ){
           37  +    void *pDbVal;
           38  +    int nDbVal;
           39  +    int rc;
           40  +
           41  +    static int nCall = 0; nCall++;
           42  +
           43  +    rc = tdb_fetch(pDb, pKey, nKey, &pDbVal, &nDbVal);
           44  +    testSetError(rc);
           45  +    if( rc==0 && (nVal!=nDbVal || (nVal>0 && lsm_memcmp(pVal, pDbVal, nVal))) ){
           46  +      testSetError(1);
           47  +    }
           48  +  }
           49  +}
           50  +
           51  +void testWrite(
           52  +  TestDb *pDb,                    /* Database handle */
           53  +  void *pKey, int nKey,           /* Key to query database for */
           54  +  void *pVal, int nVal,           /* Value to write */
           55  +  int *pRc                        /* IN/OUT: Error code */
           56  +){
           57  +  if( *pRc==0 ){
           58  +    int rc;
           59  +static int nCall = 0;
           60  +nCall++;
           61  +    rc = tdb_write(pDb, pKey, nKey, pVal, nVal);
           62  +    testSetError(rc);
           63  +  }
           64  +}
           65  +void testDelete(
           66  +  TestDb *pDb,                    /* Database handle */
           67  +  void *pKey, int nKey,           /* Key to query database for */
           68  +  int *pRc                        /* IN/OUT: Error code */
           69  +){
           70  +  if( *pRc==0 ){
           71  +    int rc;
           72  +    *pRc = rc = tdb_delete(pDb, pKey, nKey);
           73  +    testSetError(rc);
           74  +  }
           75  +}
           76  +void testDeleteRange(
           77  +  TestDb *pDb,                    /* Database handle */
           78  +  void *pKey1, int nKey1,
           79  +  void *pKey2, int nKey2,
           80  +  int *pRc                        /* IN/OUT: Error code */
           81  +){
           82  +  if( *pRc==0 ){
           83  +    int rc;
           84  +    *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2);
           85  +    testSetError(rc);
           86  +  }
           87  +}
           88  +
           89  +void testBegin(TestDb *pDb, int iTrans, int *pRc){
           90  +  if( *pRc==0 ){
           91  +    int rc;
           92  +    rc = tdb_begin(pDb, iTrans);
           93  +    testSetError(rc);
           94  +  }
           95  +}
           96  +void testCommit(TestDb *pDb, int iTrans, int *pRc){
           97  +  if( *pRc==0 ){
           98  +    int rc;
           99  +    rc = tdb_commit(pDb, iTrans);
          100  +    testSetError(rc);
          101  +  }
          102  +}
          103  +#if 0 /* unused */
          104  +static void testRollback(TestDb *pDb, int iTrans, int *pRc){
          105  +  if( *pRc==0 ){
          106  +    int rc;
          107  +    rc = tdb_rollback(pDb, iTrans);
          108  +    testSetError(rc);
          109  +  }
          110  +}
          111  +#endif
          112  +
          113  +void testWriteStr(
          114  +  TestDb *pDb,                    /* Database handle */
          115  +  const char *zKey,               /* Key to query database for */
          116  +  const char *zVal,               /* Value to write */
          117  +  int *pRc                        /* IN/OUT: Error code */
          118  +){
          119  +  int nVal = (zVal ? strlen(zVal) : 0);
          120  +  testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
          121  +}
          122  +
          123  +#if 0 /* unused */
          124  +static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){
          125  +  testDelete(pDb, (void *)zKey, strlen(zKey), pRc);
          126  +}
          127  +#endif
          128  +void testFetchStr(
          129  +  TestDb *pDb,                    /* Database handle */
          130  +  const char *zKey,               /* Key to query database for */
          131  +  const char *zVal,               /* Value to write */
          132  +  int *pRc                        /* IN/OUT: Error code */
          133  +){
          134  +  int nVal = (zVal ? strlen(zVal) : 0);
          135  +  testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
          136  +}
          137  +
          138  +void testFetchCompare(
          139  +  TestDb *pControl, 
          140  +  TestDb *pDb, 
          141  +  void *pKey, int nKey, 
          142  +  int *pRc
          143  +){
          144  +  int rc;
          145  +  void *pDbVal1;
          146  +  void *pDbVal2;
          147  +  int nDbVal1;
          148  +  int nDbVal2;
          149  +
          150  +  static int nCall = 0;
          151  +  nCall++;
          152  +
          153  +  rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1);
          154  +  testSetError(rc);
          155  +
          156  +  rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2);
          157  +  testSetError(rc);
          158  +
          159  +  if( *pRc==0 
          160  +   && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1)))
          161  +  ){
          162  +    testSetError(1);
          163  +  }
          164  +}
          165  +
          166  +typedef struct ScanResult ScanResult;
          167  +struct ScanResult {
          168  +  TestDb *pDb;
          169  +
          170  +  int nRow;
          171  +  u32 cksum1;
          172  +  u32 cksum2;
          173  +  void *pKey1; int nKey1;
          174  +  void *pKey2; int nKey2;
          175  +
          176  +  int bReverse;
          177  +  int nPrevKey;
          178  +  u8 aPrevKey[256];
          179  +};
          180  +
          181  +static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){
          182  +  int res;
          183  +  res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2));
          184  +  if( res==0 ){
          185  +    res = nKey1 - nKey2;
          186  +  }
          187  +  return res;
          188  +}
          189  +
          190  +int test_scan_debug = 0;
          191  +
          192  +static void scanCompareCb(
          193  +  void *pCtx, 
          194  +  void *pKey, int nKey,
          195  +  void *pVal, int nVal
          196  +){
          197  +  ScanResult *p = (ScanResult *)pCtx;
          198  +  u8 *aKey = (u8 *)pKey;
          199  +  u8 *aVal = (u8 *)pVal;
          200  +  int i;
          201  +
          202  +  if( test_scan_debug ){
          203  +    printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey);
          204  +    fflush(stdout);
          205  +  }
          206  +#if 0
          207  +  if( test_scan_debug ) printf("%.20s\n", (char *)pVal);
          208  +#endif
          209  +
          210  +#if 0
          211  +  /* Check tdb_fetch() matches */
          212  +  int rc = 0;
          213  +  testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc);
          214  +  assert( rc==0 );
          215  +#endif
          216  +
          217  +  /* Update the checksum data */
          218  +  p->nRow++;
          219  +  for(i=0; i<nKey; i++){
          220  +    p->cksum1 += ((int)aKey[i] << (i&0x0F));
          221  +    p->cksum2 += p->cksum1;
          222  +  }
          223  +  for(i=0; i<nVal; i++){
          224  +    p->cksum1 += ((int)aVal[i] << (i&0x0F));
          225  +    p->cksum2 += p->cksum1;
          226  +  }
          227  +
          228  +  /* Check that the delivered row is not out of order. */
          229  +  if( nKey<(int)sizeof(p->aPrevKey) ){
          230  +    if( p->nPrevKey ){
          231  +      int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey);
          232  +      if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){
          233  +        testPrintError("Returned key out of order at %s:%d\n", 
          234  +            __FILE__, __LINE__
          235  +        );
          236  +      }
          237  +    }
          238  +
          239  +    p->nPrevKey = nKey;
          240  +    memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey));
          241  +  }
          242  +
          243  +  /* Check that the delivered row is within range. */
          244  +  if( p->pKey1 && (
          245  +      (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0)
          246  +   || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey)
          247  +  )){
          248  +    testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__);
          249  +  }
          250  +  if( p->pKey2 && (
          251  +      (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0)
          252  +   || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2<nKey)
          253  +  )){
          254  +    testPrintError("Returned key too large at %s:%d\n", __FILE__, __LINE__);
          255  +  }
          256  +
          257  +}
          258  +
          259  +/*
          260  +** Scan the contents of the two databases. Check that they match.
          261  +*/
          262  +void testScanCompare(
          263  +  TestDb *pDb1,                   /* Control (trusted) database */
          264  +  TestDb *pDb2,                   /* Database being tested */
          265  +  int bReverse,
          266  +  void *pKey1, int nKey1, 
          267  +  void *pKey2, int nKey2, 
          268  +  int *pRc
          269  +){
          270  +  static int nCall = 0; nCall++;
          271  +  if( *pRc==0 ){
          272  +    ScanResult res1;
          273  +    ScanResult res2;
          274  +    void *pRes1 = (void *)&res1;
          275  +    void *pRes2 = (void *)&res2;
          276  +
          277  +    memset(&res1, 0, sizeof(ScanResult));
          278  +    memset(&res2, 0, sizeof(ScanResult));
          279  +
          280  +    res1.pDb = pDb1;
          281  +    res1.nKey1 = nKey1; res1.pKey1 = pKey1;
          282  +    res1.nKey2 = nKey2; res1.pKey2 = pKey2;
          283  +    res1.bReverse = bReverse;
          284  +    res2.pDb = pDb2;
          285  +    res2.nKey1 = nKey1; res2.pKey1 = pKey1;
          286  +    res2.nKey2 = nKey2; res2.pKey2 = pKey2;
          287  +    res2.bReverse = bReverse;
          288  +
          289  +    tdb_scan(pDb1, pRes1, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
          290  +if( test_scan_debug ) printf("\n\n\n");
          291  +    tdb_scan(pDb2, pRes2, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
          292  +if( test_scan_debug ) printf("\n\n\n");
          293  +
          294  +    if( res1.nRow!=res2.nRow 
          295  +     || res1.cksum1!=res2.cksum1 
          296  +     || res1.cksum2!=res2.cksum2
          297  +    ){
          298  +      printf("expected: %d %X %X\n", res1.nRow, res1.cksum1, res1.cksum2);
          299  +      printf("got:      %d %X %X\n", res2.nRow, res2.cksum1, res2.cksum2);
          300  +      testSetError(1);
          301  +      *pRc = 1;
          302  +    }
          303  +  }
          304  +}
          305  +
          306  +void testClose(TestDb **ppDb){
          307  +  tdb_close(*ppDb);
          308  +  *ppDb = 0;
          309  +}
          310  +
          311  +TestDb *testOpen(const char *zSystem, int bClear, int *pRc){
          312  +  TestDb *pDb = 0;
          313  +  if( *pRc==0 ){
          314  +    int rc;
          315  +    rc = tdb_open(zSystem, 0, bClear, &pDb);
          316  +    if( rc!=0 ){
          317  +      testSetError(rc);
          318  +      *pRc = rc;
          319  +    }
          320  +  }
          321  +  return pDb;
          322  +}
          323  +
          324  +void testReopen(TestDb **ppDb, int *pRc){
          325  +  if( *pRc==0 ){
          326  +    const char *zLib;
          327  +    zLib = tdb_library_name(*ppDb);
          328  +    testClose(ppDb);
          329  +    *pRc = tdb_open(zLib, 0, 0, ppDb);
          330  +  }
          331  +}
          332  +
          333  +
          334  +#if 0 /* unused */
          335  +static void testSystemSelect(const char *zSys, int *piSel, int *pRc){
          336  +  if( *pRc==0 ){
          337  +    struct SysName { const char *zName; } *aName;
          338  +    int nSys;
          339  +    int i;
          340  +
          341  +    for(nSys=0; tdb_system_name(nSys); nSys++);
          342  +    aName = malloc(sizeof(struct SysName) * (nSys+1));
          343  +    for(i=0; i<=nSys; i++){
          344  +      aName[i].zName = tdb_system_name(i);
          345  +    }
          346  +
          347  +    *pRc = testArgSelect(aName, "db", zSys, piSel);
          348  +    free(aName);
          349  +  }
          350  +}
          351  +#endif
          352  +
          353  +char *testMallocVPrintf(const char *zFormat, va_list ap){
          354  +  int nByte;
          355  +  va_list copy;
          356  +  char *zRet;
          357  +
          358  +  __va_copy(copy, ap);
          359  +  nByte = vsnprintf(0, 0, zFormat, copy);
          360  +  va_end(copy);
          361  +
          362  +  assert( nByte>=0 );
          363  +  zRet = (char *)testMalloc(nByte+1);
          364  +  vsnprintf(zRet, nByte+1, zFormat, ap);
          365  +  return zRet;
          366  +}
          367  +
          368  +char *testMallocPrintf(const char *zFormat, ...){
          369  +  va_list ap;
          370  +  char *zRet;
          371  +
          372  +  va_start(ap, zFormat);
          373  +  zRet = testMallocVPrintf(zFormat, ap);
          374  +  va_end(ap);
          375  +
          376  +  return zRet;
          377  +}
          378  +
          379  +
          380  +/*
          381  +** A wrapper around malloc(3).
          382  +**
          383  +** This function should be used for all allocations made by test procedures.
          384  +** It has the following properties:
          385  +**
          386  +**   * Test code may assume that allocations may not fail.
          387  +**   * Returned memory is always zeroed.
          388  +**
          389  +** Allocations made using testMalloc() should be freed using testFree().
          390  +*/
          391  +void *testMalloc(int n){
          392  +  u8 *p = (u8*)malloc(n + 8);
          393  +  memset(p, 0, n+8);
          394  +  *(int*)p = n;
          395  +  return (void*)&p[8];
          396  +}
          397  +
          398  +void *testMallocCopy(void *pCopy, int nByte){
          399  +  void *pRet = testMalloc(nByte);
          400  +  memcpy(pRet, pCopy, nByte);
          401  +  return pRet;
          402  +}
          403  +
          404  +void *testRealloc(void *ptr, int n){
          405  +  if( ptr ){
          406  +    u8 *p = (u8*)ptr - 8;
          407  +    int nOrig =  *(int*)p;
          408  +    p = (u8*)realloc(p, n+8);
          409  +    if( nOrig<n ){
          410  +      memset(&p[8+nOrig], 0, n-nOrig);
          411  +    }
          412  +    *(int*)p = n;
          413  +    return (void*)&p[8];
          414  +  }
          415  +  return testMalloc(n);
          416  +}
          417  +
          418  +/*
          419  +** Free an allocation made by an earlier call to testMalloc().
          420  +*/
          421  +void testFree(void *ptr){
          422  +  if( ptr ){
          423  +    u8 *p = (u8*)ptr - 8;
          424  +    memset(p, 0x55, *(int*)p + 8);
          425  +    free(p);
          426  +  }
          427  +}
          428  +
          429  +/*
          430  +** String zPattern contains a glob pattern. Return true if zStr matches 
          431  +** the pattern, or false if it does not.
          432  +*/
          433  +int testGlobMatch(const char *zPattern, const char *zStr){
          434  +  int i = 0;
          435  +  int j = 0;
          436  +
          437  +  while( zPattern[i] ){
          438  +    char p = zPattern[i];
          439  +
          440  +    if( p=='*' || p=='%' ){
          441  +      do {
          442  +        if( testGlobMatch(&zPattern[i+1], &zStr[j]) ) return 1;
          443  +      }while( zStr[j++] );
          444  +      return 0;
          445  +    }
          446  +
          447  +    if( zStr[j]==0 || (p!='?' && p!=zStr[j]) ){
          448  +      /* Match failed. */
          449  +      return 0;
          450  +    }
          451  +
          452  +    j++;
          453  +    i++;
          454  +  }
          455  +
          456  +  return (zPattern[i]==0 && zStr[j]==0);
          457  +}
          458  +
          459  +/* 
          460  +** End of test utilities 
          461  +**************************************************************************/
          462  +
          463  +int do_test(int nArg, char **azArg){
          464  +  int j;
          465  +  int rc;
          466  +  int nFail = 0;
          467  +  const char *zPattern = 0;
          468  +
          469  +  if( nArg>1 ){
          470  +    testPrintError("Usage: test ?PATTERN?\n");
          471  +    return 1;
          472  +  }
          473  +  if( nArg==1 ){
          474  +    zPattern = azArg[0];
          475  +  }
          476  +
          477  +  for(j=0; tdb_system_name(j); j++){
          478  +    rc = 0;
          479  +
          480  +    test_data_1(tdb_system_name(j), zPattern, &rc);
          481  +    test_data_2(tdb_system_name(j), zPattern, &rc);
          482  +    test_data_3(tdb_system_name(j), zPattern, &rc);
          483  +    test_data_4(tdb_system_name(j), zPattern, &rc);
          484  +    test_rollback(tdb_system_name(j), zPattern, &rc);
          485  +    test_mc(tdb_system_name(j), zPattern, &rc);
          486  +    test_mt(tdb_system_name(j), zPattern, &rc);
          487  +
          488  +    if( rc ) nFail++;
          489  +  }
          490  +
          491  +  rc = 0;
          492  +  test_oom(zPattern, &rc);
          493  +  if( rc ) nFail++;
          494  +
          495  +  rc = 0;
          496  +  test_api(zPattern, &rc);
          497  +  if( rc ) nFail++;
          498  +
          499  +  rc = 0;
          500  +  do_crash_test(zPattern, &rc);
          501  +  if( rc ) nFail++;
          502  +
          503  +  rc = 0;
          504  +  do_writer_crash_test(zPattern, &rc);
          505  +  if( rc ) nFail++;
          506  +
          507  +  return (nFail!=0);
          508  +}
          509  +
          510  +static lsm_db *configure_lsm_db(TestDb *pDb){
          511  +  lsm_db *pLsm;
          512  +  pLsm = tdb_lsm(pDb);
          513  +  if( pLsm ){
          514  +    tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4");
          515  +  }
          516  +  return pLsm;
          517  +}
          518  +
          519  +typedef struct WriteHookEvent WriteHookEvent;
          520  +struct WriteHookEvent {
          521  +  i64 iOff;
          522  +  int nData;
          523  +  int nUs;
          524  +};
          525  +WriteHookEvent prev = {0, 0, 0};
          526  +
          527  +static void flushPrev(FILE *pOut){
          528  +  if( prev.nData ){
          529  +    fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs);
          530  +    prev.nData = 0;
          531  +  }
          532  +}
          533  +
          534  +#if 0 /* unused */
          535  +static void do_speed_write_hook2(
          536  +  void *pCtx,
          537  +  int bLog,
          538  +  i64 iOff,
          539  +  int nData,
          540  +  int nUs
          541  +){
          542  +  FILE *pOut = (FILE *)pCtx;
          543  +  if( bLog ) return;
          544  +
          545  +  if( prev.nData && nData && iOff==prev.iOff+prev.nData ){
          546  +    prev.nData += nData;
          547  +    prev.nUs += nUs;
          548  +  }else{
          549  +    flushPrev(pOut);
          550  +    if( nData==0 ){
          551  +      fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs);
          552  +    }else{
          553  +      prev.iOff = iOff;
          554  +      prev.nData = nData;
          555  +      prev.nUs = nUs;
          556  +    }
          557  +  }
          558  +}
          559  +#endif
          560  +
          561  +#define ST_REPEAT  0
          562  +#define ST_WRITE   1
          563  +#define ST_PAUSE   2
          564  +#define ST_FETCH   3
          565  +#define ST_SCAN    4
          566  +#define ST_NSCAN   5
          567  +#define ST_KEYSIZE 6
          568  +#define ST_VALSIZE 7
          569  +#define ST_TRANS   8
          570  +
          571  +
          572  +static void print_speed_test_help(){
          573  +  printf(
          574  +"\n"
          575  +"Repeat the following $repeat times:\n"
          576  +"  1. Insert $write key-value pairs. One transaction for each write op.\n"
          577  +"  2. Pause for $pause ms.\n"
          578  +"  3. Perform $fetch queries on the database.\n"
          579  +"\n"
          580  +"  Keys are $keysize bytes in size. Values are $valsize bytes in size\n"
          581  +"  Both keys and values are pseudo-randomly generated\n"
          582  +"\n"
          583  +"Options are:\n"
          584  +"  -repeat  $repeat                 (default value 10)\n"
          585  +"  -write   $write                  (default value 10000)\n"
          586  +"  -pause   $pause                  (default value 0)\n"
          587  +"  -fetch   $fetch                  (default value 0)\n"
          588  +"  -keysize $keysize                (default value 12)\n"
          589  +"  -valsize $valsize                (default value 100)\n"
          590  +"  -system  $system                 (default value \"lsm\")\n"
          591  +"  -trans   $trans                  (default value 0)\n"
          592  +"\n"
          593  +);
          594  +}
          595  +
          596  +int do_speed_test2(int nArg, char **azArg){
          597  +  struct Option {
          598  +    const char *zOpt;
          599  +    int eVal;
          600  +    int iDefault;
          601  +  } aOpt[] = {
          602  +    { "-repeat",  ST_REPEAT,    10},
          603  +    { "-write",   ST_WRITE,  10000},
          604  +    { "-pause",   ST_PAUSE,      0},
          605  +    { "-fetch",   ST_FETCH,      0},
          606  +    { "-scan",    ST_SCAN,       0},
          607  +    { "-nscan",   ST_NSCAN,      0},
          608  +    { "-keysize", ST_KEYSIZE,   12},
          609  +    { "-valsize", ST_VALSIZE,  100},
          610  +    { "-trans",   ST_TRANS,      0},
          611  +    { "-system",  -1,            0},
          612  +    { "help",     -2,            0},
          613  +    {0, 0, 0}
          614  +  };
          615  +  int i;
          616  +  int aParam[9];
          617  +  int rc = 0;
          618  +  int bReadonly = 0;
          619  +  int nContent = 0;
          620  +
          621  +  TestDb *pDb;
          622  +  Datasource *pData;
          623  +  DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 };
          624  +  char *zSystem = "";
          625  +  int bLsm = 1;
          626  +  FILE *pLog = 0;
          627  +
          628  +#ifdef NDEBUG
          629  +  /* If NDEBUG is defined, disable the dynamic memory related checks in
          630  +  ** lsmtest_mem.c. They slow things down.  */
          631  +  testMallocUninstall(tdb_lsm_env());
          632  +#endif
          633  +
          634  +  /* Initialize aParam[] with default values. */
          635  +  for(i=0; i<ArraySize(aOpt); i++){
          636  +    if( aOpt[i].zOpt ) aParam[aOpt[i].eVal] = aOpt[i].iDefault;
          637  +  }
          638  +
          639  +  /* Process the command line switches. */
          640  +  for(i=0; i<nArg; i+=2){
          641  +    int iSel;
          642  +    rc = testArgSelect(aOpt, "switch", azArg[i], &iSel);
          643  +    if( rc ){
          644  +      return rc;
          645  +    }
          646  +    if( aOpt[iSel].eVal==-2 ){
          647  +      print_speed_test_help();
          648  +      return 0;
          649  +    }
          650  +    if( i+1==nArg ){
          651  +      testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
          652  +      return 1;
          653  +    }
          654  +    if( aOpt[iSel].eVal>=0 ){
          655  +      aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]);
          656  +    }else{
          657  +      zSystem = azArg[i+1];
          658  +      bLsm = 0;
          659  +#if 0
          660  +      for(j=0; zSystem[j]; j++){
          661  +        if( zSystem[j]=='=' ) bLsm = 1;
          662  +      }
          663  +#endif
          664  +    }
          665  +  }
          666  +  
          667  +  printf("#");
          668  +  for(i=0; i<ArraySize(aOpt); i++){
          669  +    if( aOpt[i].zOpt ){
          670  +      if( aOpt[i].eVal>=0 ){
          671  +        printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]);
          672  +      }else if( aOpt[i].eVal==-1 ){
          673  +        printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem);
          674  +      }
          675  +    }
          676  +  }
          677  +  printf("\n");
          678  +
          679  +  defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE];
          680  +  defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE];
          681  +  pData = testDatasourceNew(&defn);
          682  +
          683  +  if( aParam[ST_WRITE]==0 ){
          684  +    bReadonly = 1;
          685  +  }
          686  +
          687  +  if( bLsm ){
          688  +    rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb);
          689  +  }else{
          690  +    pDb = testOpen(zSystem, !bReadonly, &rc);
          691  +  }
          692  +  if( rc!=0 ) return rc;
          693  +  if( bReadonly ){
          694  +    nContent = testCountDatabase(pDb);
          695  +  }
          696  +
          697  +#if 0
          698  +  pLog = fopen("/tmp/speed.log", "w");
          699  +  tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
          700  +#endif
          701  +
          702  +  for(i=0; i<aParam[ST_REPEAT] && rc==0; i++){
          703  +    int msWrite, msFetch;
          704  +    int iFetch;
          705  +    int nWrite = aParam[ST_WRITE];
          706  +
          707  +    if( bReadonly ){
          708  +      msWrite = 0;
          709  +    }else{
          710  +      testTimeInit();
          711  +
          712  +      if( aParam[ST_TRANS] ) testBegin(pDb, 2, &rc);
          713  +      testWriteDatasourceRange(pDb, pData, i*nWrite, nWrite, &rc);
          714  +      if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
          715  +
          716  +      msWrite = testTimeGet();
          717  +      nContent += nWrite;
          718  +    }
          719  +
          720  +    if( aParam[ST_PAUSE] ){
          721  +      if( aParam[ST_PAUSE]/1000 ) sleep(aParam[ST_PAUSE]/1000);
          722  +      if( aParam[ST_PAUSE]%1000 ) usleep(1000 * (aParam[ST_PAUSE]%1000));
          723  +    }
          724  +
          725  +    if( aParam[ST_FETCH] ){
          726  +      testTimeInit();
          727  +      if( aParam[ST_TRANS] ) testBegin(pDb, 1, &rc);
          728  +      for(iFetch=0; iFetch<aParam[ST_FETCH]; iFetch++){
          729  +        int iKey = testPrngValue(i*nWrite+iFetch) % nContent;
          730  +#ifndef NDEBUG
          731  +        testDatasourceFetch(pDb, pData, iKey, &rc);
          732  +#else
          733  +        void *pKey; int nKey;           /* Database key to query for */
          734  +        void *pVal; int nVal;           /* Result of query */
          735  +
          736  +        testDatasourceEntry(pData, iKey, &pKey, &nKey, 0, 0);
          737  +        rc = tdb_fetch(pDb, pKey, nKey, &pVal, &nVal);
          738  +        if( rc==0 && nVal<0 ) rc = 1;
          739  +        if( rc ) break;
          740  +#endif
          741  +      }
          742  +      if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
          743  +      msFetch = testTimeGet();
          744  +    }else{
          745  +      msFetch = 0;
          746  +    }
          747  +
          748  +    if( i==(aParam[ST_REPEAT]-1) ){
          749  +      testTimeInit();
          750  +      testClose(&pDb);
          751  +      msWrite += testTimeGet();
          752  +    }
          753  +
          754  +    printf("%d %d %d\n", i, msWrite, msFetch);
          755  +    fflush(stdout);
          756  +  }
          757  +
          758  +  testClose(&pDb);
          759  +  testDatasourceFree(pData);
          760  +
          761  +  if( pLog ){
          762  +    flushPrev(pLog);
          763  +    fclose(pLog);
          764  +  }
          765  +  return rc;
          766  +}
          767  +
          768  +int do_speed_tests(int nArg, char **azArg){
          769  +
          770  +  struct DbSystem {
          771  +    const char *zLibrary;
          772  +    const char *zColor;
          773  +  } aSys[] = {
          774  +    { "sqlite3",      "black" },
          775  +    { "leveldb",      "blue" },
          776  +    { "lsm",          "red" },
          777  +    { "lsm_mt2",      "orange" },
          778  +    { "lsm_mt3",      "purple" },
          779  +    { "kyotocabinet", "green" },
          780  +    {0, 0}
          781  +  };
          782  +
          783  +  int i;
          784  +  int j;
          785  +  int rc;
          786  +  int nSleep = 0;                 /* ms of rest allowed between INSERT tests */
          787  +  int nRow = 0;                   /* Number of rows to insert into database */
          788  +  int nStep;                      /* Measure INSERT time after this many rows */
          789  +  int nSelStep;                   /* Measure SELECT time after this many rows */
          790  +  int nSelTest;                   /* Number of SELECTs to run for timing */
          791  +  int doReadTest = 1;
          792  +  int doWriteTest = 1;
          793  +
          794  +  int *aTime;                     /* INSERT timing data */
          795  +  int *aWrite;                    /* Writes per nStep inserts */
          796  +  int *aSelTime;                  /* SELECT timing data */
          797  +  int isFirst = 1;
          798  +  int bSleep = 0;
          799  +
          800  +  /* File to write gnuplot script to. */
          801  +  const char *zOut = "lsmtest_speed.gnuplot";
          802  +
          803  +  u32 sys_mask = 0;
          804  +
          805  +  testMallocUninstall(tdb_lsm_env());
          806  +
          807  +  for(i=0; i<nArg; i++){
          808  +    struct Opt { 
          809  +      const char *zOpt; 
          810  +      int isSwitch;
          811  +    } aOpt[] = {
          812  +      { "sqlite3" , 0},
          813  +      { "leveldb" , 0},
          814  +      { "lsm" , 0},
          815  +      { "lsm_mt2" , 0},
          816  +      { "lsm_mt3" , 0},
          817  +      { "kyotocabinet" , 0},
          818  +      { "-rows"     , 1},
          819  +      { "-sleep"    , 2},
          820  +      { "-testmode" , 3},
          821  +      { "-out"      , 4},
          822  +      { 0, 0}
          823  +    };
          824  +    int iSel;
          825  +
          826  +    rc = testArgSelect(aOpt, "argument", azArg[i], &iSel);
          827  +    if( rc ) return rc;
          828  +
          829  +    if( aOpt[iSel].isSwitch ){
          830  +      i++;
          831  +
          832  +      if( i>=nArg ){
          833  +        testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
          834  +        return 1;
          835  +      }
          836  +      if( aOpt[iSel].isSwitch==1 ){
          837  +        nRow = atoi(azArg[i]);
          838  +      }
          839  +      if( aOpt[iSel].isSwitch==2 ){
          840  +        nSleep = atoi(azArg[i]);
          841  +      }
          842  +      if( aOpt[iSel].isSwitch==3 ){
          843  +        struct Mode {
          844  +          const char *zMode;
          845  +          int doReadTest;
          846  +          int doWriteTest;
          847  +        } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}};
          848  +        int iMode;
          849  +        rc = testArgSelect(aMode, "option", azArg[i], &iMode);
          850  +        if( rc ) return rc;
          851  +        doReadTest = aMode[iMode].doReadTest;
          852  +        doWriteTest = aMode[iMode].doWriteTest;
          853  +      }
          854  +      if( aOpt[iSel].isSwitch==4 ){
          855  +        /* The "-out FILE" switch. This option is used to specify a file to
          856  +        ** write the gnuplot script to. */
          857  +        zOut = azArg[i];
          858  +      }
          859  +    }else{
          860  +      /* A db name */
          861  +      rc = testArgSelect(aOpt, "system", azArg[i], &iSel);
          862  +      if( rc ) return rc;
          863  +      sys_mask |= (1<<iSel);
          864  +    }
          865  +  }
          866  +
          867  +  if( sys_mask==0 ) sys_mask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
          868  +  nRow = MAX(nRow, 100000);
          869  +  nStep = nRow/100;
          870  +  nSelStep = nRow/10;
          871  +  nSelTest = (nSelStep > 100000) ? 100000 : nSelStep;
          872  +
          873  +  aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep);
          874  +  aWrite = malloc(sizeof(int) * nRow/nStep);
          875  +  aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep);
          876  +
          877  +  /* This loop collects the INSERT speed data. */
          878  +  if( doWriteTest ){
          879  +    printf("Writing output to file \"%s\".\n",  zOut);
          880  +
          881  +    for(j=0; aSys[j].zLibrary; j++){
          882  +      FILE *pLog = 0;
          883  +      TestDb *pDb;                  /* Database being tested */
          884  +      lsm_db *pLsm;
          885  +      int iDot = 0;
          886  +  
          887  +      if( ((1<<j)&sys_mask)==0 ) continue;
          888  +      if( bSleep && nSleep ) sqlite3_sleep(nSleep);
          889  +      bSleep = 1;
          890  +
          891  +      testCaseBegin(&rc, 0, "speed.insert.%s", aSys[j].zLibrary);
          892  +
          893  +      rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
          894  +      if( rc ) return rc;
          895  +
          896  +      pLsm = configure_lsm_db(pDb);
          897  +#if 0
          898  +      pLog = fopen("/tmp/speed.log", "w");
          899  +      tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
          900  +#endif
          901  +  
          902  +      testTimeInit();
          903  +      for(i=0; i<nRow; i+=nStep){
          904  +        int iStep;
          905  +        int nWrite1 = 0, nWrite2 = 0;
          906  +        testCaseProgress(i, nRow, testCaseNDot(), &iDot);
          907  +        if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite1);
          908  +        for(iStep=0; iStep<nStep; iStep++){
          909  +          u32 aKey[4];                  /* 16-byte key */
          910  +          u32 aVal[25];                 /* 100 byte value */
          911  +          testPrngArray(i+iStep, aKey, ArraySize(aKey));
          912  +          testPrngArray(i+iStep, aVal, ArraySize(aVal));
          913  +          rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
          914  +        }
          915  +        aTime[(j*nRow+i)/nStep] = testTimeGet();
          916  +        if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite2);
          917  +        aWrite[i/nStep] = nWrite2 - nWrite1;
          918  +      }
          919  +
          920  +      tdb_close(pDb);
          921  +      if( pLog ) fclose(pLog);
          922  +      testCaseFinish(rc);
          923  +    }
          924  +  }
          925  +
          926  +  /* This loop collects the SELECT speed data. */
          927  +  if( doReadTest ){
          928  +    for(j=0; aSys[j].zLibrary; j++){
          929  +      int iDot = 0;
          930  +      TestDb *pDb;                  /* Database being tested */
          931  +
          932  +      if( ((1<<j)&sys_mask)==0 ) continue;
          933  +      if( bSleep && nSleep ) sqlite3_sleep(nSleep);
          934  +      bSleep = 1;
          935  +
          936  +      testCaseBegin(&rc, 0, "speed.select.%s", aSys[j].zLibrary);
          937  +
          938  +      if( doWriteTest ){
          939  +        rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
          940  +        if( rc ) return rc;
          941  +        configure_lsm_db(pDb);
          942  +
          943  +        for(i=0; i<nRow; i+=nSelStep){
          944  +          int iStep;
          945  +          int iSel;
          946  +          testCaseProgress(i, nRow, testCaseNDot(), &iDot);
          947  +          for(iStep=0; iStep<nSelStep; iStep++){
          948  +            u32 aKey[4];                  /* 16-byte key */
          949  +            u32 aVal[25];                 /* 100 byte value */
          950  +            testPrngArray(i+iStep, aKey, ArraySize(aKey));
          951  +            testPrngArray(i+iStep, aVal, ArraySize(aVal));
          952  +            rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
          953  +          }
          954  +    
          955  +          testTimeInit();
          956  +          for(iSel=0; iSel<nSelTest; iSel++){
          957  +            void *pDummy;
          958  +            int nDummy;
          959  +            u32 iKey;
          960  +            u32 aKey[4];                  /* 16-byte key */
          961  +    
          962  +            iKey = testPrngValue(iSel) % (i+nSelStep);
          963  +            testPrngArray(iKey, aKey, ArraySize(aKey));
          964  +            rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
          965  +          }
          966  +          aSelTime[(j*nRow+i)/nSelStep] = testTimeGet();
          967  +          tdb_fetch(pDb, 0, 0, 0, 0);
          968  +        }
          969  +      }else{
          970  +        int t;
          971  +        int iSel;
          972  +
          973  +        rc = tdb_open(aSys[j].zLibrary, 0, 0, &pDb);
          974  +        configure_lsm_db(pDb);
          975  +
          976  +        testTimeInit();
          977  +        for(iSel=0; rc==LSM_OK && iSel<nSelTest; iSel++){
          978  +          void *pDummy;
          979  +          int nDummy;
          980  +          u32 iKey;
          981  +          u32 aKey[4];                  /* 16-byte key */
          982  +#ifndef NDEBUG
          983  +          u32 aVal[25];                 /* 100 byte value */
          984  +#endif
          985  +
          986  +          testCaseProgress(iSel, nSelTest, testCaseNDot(), &iDot);
          987  +    
          988  +          iKey = testPrngValue(iSel) % nRow;
          989  +          testPrngArray(iKey, aKey, ArraySize(aKey));
          990  +          rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
          991  +
          992  +#ifndef NDEBUG
          993  +          testPrngArray(iKey, aVal, ArraySize(aVal));
          994  +          assert( nDummy==100 && memcmp(aVal, pDummy, 100)==0 );
          995  +#endif
          996  +        }
          997  +        if( rc!=LSM_OK ) return rc;
          998  +
          999  +        t = testTimeGet();
         1000  +        tdb_fetch(pDb, 0, 0, 0, 0);
         1001  +
         1002  +        printf("%s: %d selects/second\n", 
         1003  +            aSys[j].zLibrary, (int)((double)nSelTest*1000.0/t)
         1004  +        );
         1005  +      }
         1006  +
         1007  +      tdb_close(pDb);
         1008  +      testCaseFinish(rc);
         1009  +    }
         1010  +  }
         1011  +
         1012  +
         1013  +  if( doWriteTest ){
         1014  +    FILE *pOut = fopen(zOut, "w");
         1015  +    if( !pOut ){
         1016  +      printf("fopen(\"%s\", \"w\"): %s\n", zOut, strerror(errno));
         1017  +      return 1;
         1018  +    }
         1019  +
         1020  +    fprintf(pOut, "set xlabel \"Rows Inserted\"\n");
         1021  +    fprintf(pOut, "set ylabel \"Inserts per second\"\n");
         1022  +    if( doReadTest ){
         1023  +      fprintf(pOut, "set y2label \"Selects per second\"\n");
         1024  +    }else if( sys_mask==(1<<2) ){
         1025  +      fprintf(pOut, "set y2label \"Page writes per insert\"\n");
         1026  +    }
         1027  +    fprintf(pOut, "set yrange [0:*]\n");
         1028  +    fprintf(pOut, "set y2range [0:*]\n");
         1029  +    fprintf(pOut, "set xrange [%d:*]\n", MAX(nStep, nRow/20) );
         1030  +    fprintf(pOut, "set ytics nomirror\n");
         1031  +    fprintf(pOut, "set y2tics nomirror\n");
         1032  +    fprintf(pOut, "set key box lw 0.01\n");
         1033  +    fprintf(pOut, "plot ");
         1034  +  
         1035  +    for(j=0; aSys[j].zLibrary; j++){
         1036  +      if( (1<<j)&sys_mask ){
         1037  +        const char *zLib = aSys[j].zLibrary;
         1038  +        fprintf(pOut, "%s\"-\" ti \"%s INSERT\" with lines lc rgb \"%s\" ", 
         1039  +            (isFirst?"":", "), zLib, aSys[j].zColor
         1040  +        );
         1041  +        if( doReadTest ){
         1042  +          fprintf(pOut, ", \"-\" ti \"%s SELECT\" "
         1043  +                 "axis x1y2 with points lw 3 lc rgb \"%s\""
         1044  +              , zLib, aSys[j].zColor
         1045  +          );
         1046  +        }
         1047  +        isFirst = 0;
         1048  +      }
         1049  +    }
         1050  +
         1051  +    assert( strcmp(aSys[2].zLibrary, "lsm")==0 );
         1052  +    if( sys_mask==(1<<2) && !doReadTest ){
         1053  +      fprintf(pOut, ", \"-\" ti \"lsm pages written\" "
         1054  +        "axis x1y2 with boxes lw 1 lc rgb \"grey\""
         1055  +      );
         1056  +    }
         1057  +  
         1058  +    fprintf(pOut, "\n");
         1059  +  
         1060  +    for(j=0; aSys[j].zLibrary; j++){
         1061  +      if( ((1<<j)&sys_mask)==0 ) continue;
         1062  +      fprintf(pOut, "# Rows    Inserts per second\n");
         1063  +      for(i=0; i<nRow; i+=nStep){
         1064  +        int iTime = aTime[(j*nRow+i)/nStep];
         1065  +        int ips = (int)((i+nStep)*1000.0 / (double)iTime);
         1066  +        fprintf(pOut, "%d %d\n", i+nStep, ips);
         1067  +      }
         1068  +      fprintf(pOut, "end\n");
         1069  +  
         1070  +      if( doReadTest ){
         1071  +        fprintf(pOut, "# Rows    Selects per second\n");
         1072  +        for(i=0; i<nRow; i+=nSelStep){
         1073  +          int sps = (int)(nSelTest*1000.0/(double)aSelTime[(j*nRow+i)/nSelStep]);
         1074  +          fprintf(pOut, "%d %d\n", i+nSelStep, sps);
         1075  +        }
         1076  +        fprintf(pOut, "end\n");
         1077  +      }else if( sys_mask==(1<<2) ){
         1078  +        for(i=0; i<(nRow/nStep); i++){
         1079  +          fprintf(pOut, "%d %f\n", i*nStep, (double)aWrite[i] / (double)nStep);
         1080  +        }
         1081  +        fprintf(pOut, "end\n");
         1082  +      }
         1083  +    }
         1084  +  
         1085  +    fprintf(pOut, "pause -1\n");
         1086  +    fclose(pOut);
         1087  +  }
         1088  +
         1089  +  free(aTime);
         1090  +  free(aSelTime);
         1091  +  free(aWrite);
         1092  +  testMallocInstall(tdb_lsm_env());
         1093  +  return 0;
         1094  +}
         1095  +
         1096  +/*
         1097  +** Usage: lsmtest random ?N?
         1098  +**
         1099  +** This command prints a sequence of zero or more numbers from the PRNG
         1100  +** system to stdout. If the "N" argument is missing, values the first 10
         1101  +** values (i=0, i=1, ... i=9) are printed. Otherwise, the first N.
         1102  +**
         1103  +** This was added to verify that the PRNG values do not change between
         1104  +** runs of the lsmtest program.
         1105  +*/
         1106  +int do_random_tests(int nArg, char **azArg){
         1107  +  int i;
         1108  +  int nRand;
         1109  +  if( nArg==0 ){
         1110  +    nRand = 10;
         1111  +  }else if( nArg==1 ){
         1112  +    nRand = atoi(azArg[0]);
         1113  +  }else{
         1114  +    testPrintError("Usage: random ?N?\n");
         1115  +    return -1;
         1116  +  }
         1117  +  for(i=0; i<nRand; i++){
         1118  +    printf("0x%x\n", testPrngValue(i));
         1119  +  }
         1120  +  return 0;
         1121  +}
         1122  +
         1123  +static int testFormatSize(char *aBuf, int nBuf, i64 nByte){
         1124  +  int res;
         1125  +  if( nByte<(1<<10) ){
         1126  +    res = snprintf(aBuf, nBuf, "%d byte", (int)nByte);
         1127  +  }else if( nByte<(1<<20) ){
         1128  +    res = snprintf(aBuf, nBuf, "%dK", (int)(nByte/(1<<10)));
         1129  +  }else{
         1130  +    res = snprintf(aBuf, nBuf, "%dM", (int)(nByte/(1<<20)));
         1131  +  }
         1132  +  return res;
         1133  +}
         1134  +
         1135  +static i64 testReadSize(char *z){
         1136  +  int n = strlen(z);
         1137  +  char c = z[n-1];
         1138  +  i64 nMul = 1;
         1139  +
         1140  +  switch( c ){
         1141  +    case 'g': case 'G':
         1142  +      nMul = (1<<30);
         1143  +      break;
         1144  +
         1145  +    case 'm': case 'M':
         1146  +      nMul = (1<<20);
         1147  +      break;
         1148  +
         1149  +    case 'k': case 'K':
         1150  +      nMul = (1<<10);
         1151  +      break;
         1152  +
         1153  +    default:
         1154  +      nMul = 1;
         1155  +  }
         1156  +
         1157  +  return nMul * (i64)atoi(z);
         1158  +} 
         1159  +
         1160  +/*
         1161  +** Usage: lsmtest writespeed FILESIZE BLOCKSIZE SYNCSIZE
         1162  +*/
         1163  +static int do_writer_test(int nArg, char **azArg){
         1164  +  int nBlock;
         1165  +  int nSize;
         1166  +  int i;
         1167  +  int fd;
         1168  +  int ms;
         1169  +  char aFilesize[32];
         1170  +  char aBlockSize[32];
         1171  +
         1172  +  char *aPage;
         1173  +  int *aOrder;
         1174  +  int nSync;
         1175  +
         1176  +  i64 filesize;
         1177  +  i64 blocksize;
         1178  +  i64 syncsize;
         1179  +  int nPage = 4096;
         1180  +
         1181  +  /* How long to sleep before running a trial (in ms). */
         1182  +#if 0
         1183  +  const int nSleep = 10000;
         1184  +#endif
         1185  +  const int nSleep = 0;
         1186  +
         1187  +  if( nArg!=3 ){
         1188  +    testPrintUsage("FILESIZE BLOCKSIZE SYNCSIZE");
         1189  +    return -1;
         1190  +  }
         1191  +
         1192  +  filesize = testReadSize(azArg[0]);
         1193  +  blocksize = testReadSize(azArg[1]);
         1194  +  syncsize = testReadSize(azArg[2]);
         1195  +
         1196  +  nBlock = (int)(filesize / blocksize);
         1197  +  nSize = (int)blocksize;
         1198  +  nSync = (int)(syncsize / blocksize);
         1199  +
         1200  +  aPage = (char *)malloc(4096);
         1201  +  aOrder = (int *)malloc(nBlock * sizeof(int));
         1202  +  for(i=0; i<nBlock; i++) aOrder[i] = i;
         1203  +  for(i=0; i<(nBlock*25); i++){
         1204  +    int tmp;
         1205  +    u32 a = testPrngValue(i);
         1206  +    u32 b = testPrngValue(a);
         1207  +    a = a % nBlock;
         1208  +    b = b % nBlock;
         1209  +    tmp = aOrder[a];
         1210  +    aOrder[a] = aOrder[b];
         1211  +    aOrder[b] = tmp;
         1212  +  }
         1213  +
         1214  +  testFormatSize(aFilesize, sizeof(aFilesize), (i64)nBlock * (i64)nSize);
         1215  +  testFormatSize(aBlockSize, sizeof(aFilesize), nSize);
         1216  +
         1217  +  printf("Testing writing a %s file using %s blocks. ", aFilesize, aBlockSize);
         1218  +  if( nSync==1 ){
         1219  +    printf("Sync after each block.\n");
         1220  +  }else{
         1221  +    printf("Sync after each %d blocks.\n", nSync);
         1222  +  }
         1223  +
         1224  +  printf("Preparing file... ");
         1225  +  fflush(stdout);
         1226  +  unlink("writer.out");
         1227  +  fd = open("writer.out", O_RDWR|O_CREAT|_O_BINARY, 0664);
         1228  +  if( fd<0 ){
         1229  +    testPrintError("open(): %d - %s\n", errno, strerror(errno));
         1230  +    return -1;
         1231  +  }
         1232  +  testTimeInit();
         1233  +  for(i=0; i<nBlock; i++){
         1234  +    int iPg;
         1235  +    memset(aPage, i&0xFF, nPage);
         1236  +    for(iPg=0; iPg<(nSize/nPage); iPg++){
         1237  +      write(fd, aPage, nPage);
         1238  +    }
         1239  +  }
         1240  +  fsync(fd);
         1241  +  printf("ok (%d ms)\n", testTimeGet());
         1242  +
         1243  +  for(i=0; i<5; i++){
         1244  +    int j;
         1245  +
         1246  +    sqlite3_sleep(nSleep);
         1247  +    printf("Now writing sequentially...  ");
         1248  +    fflush(stdout);
         1249  +
         1250  +    lseek(fd, 0, SEEK_SET);
         1251  +    testTimeInit();
         1252  +    for(j=0; j<nBlock; j++){
         1253  +      int iPg;
         1254  +      if( ((j+1)%nSync)==0 ) fdatasync(fd);
         1255  +      memset(aPage, j&0xFF, nPage);
         1256  +      for(iPg=0; iPg<(nSize/nPage); iPg++){
         1257  +        write(fd, aPage, nPage);
         1258  +      }
         1259  +    }
         1260  +    fdatasync(fd);
         1261  +    ms = testTimeGet();
         1262  +    printf("%d ms\n", ms);
         1263  +    sqlite3_sleep(nSleep);
         1264  +    printf("Now in an arbitrary order... ");
         1265  +
         1266  +    fflush(stdout);
         1267  +    testTimeInit();
         1268  +    for(j=0; j<nBlock; j++){
         1269  +      int iPg;
         1270  +      if( ((j+1)%nSync)==0 ) fdatasync(fd);
         1271  +      lseek(fd, aOrder[j]*nSize, SEEK_SET);
         1272  +      memset(aPage, j&0xFF, nPage);
         1273  +      for(iPg=0; iPg<(nSize/nPage); iPg++){
         1274  +        write(fd, aPage, nPage);
         1275  +      }
         1276  +    }
         1277  +    fdatasync(fd);
         1278  +    ms = testTimeGet();
         1279  +    printf("%d ms\n", ms);
         1280  +  }
         1281  +
         1282  +  close(fd);
         1283  +  free(aPage);
         1284  +  free(aOrder);
         1285  +
         1286  +  return 0;
         1287  +}
         1288  +
         1289  +static void do_insert_work_hook(lsm_db *db, void *p){
         1290  +  char *z = 0;
         1291  +  lsm_info(db, LSM_INFO_DB_STRUCTURE, &z);
         1292  +  if( z ){
         1293  +    printf("%s\n", z);
         1294  +    fflush(stdout);
         1295  +    lsm_free(lsm_get_env(db), z);
         1296  +  }
         1297  +
         1298  +  unused_parameter(p);
         1299  +}
         1300  +
         1301  +typedef struct InsertWriteHook InsertWriteHook;
         1302  +struct InsertWriteHook {
         1303  +  FILE *pOut;
         1304  +  int bLog;
         1305  +  i64 iOff;
         1306  +  int nData;
         1307  +};
         1308  +
         1309  +static void flushHook(InsertWriteHook *pHook){
         1310  +  if( pHook->nData ){
         1311  +    fprintf(pHook->pOut, "write %s %d %d\n", 
         1312  +        (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData
         1313  +    );
         1314  +    pHook->nData = 0;
         1315  +    fflush(pHook->pOut);
         1316  +  }
         1317  +}
         1318  +
         1319  +static void do_insert_write_hook(
         1320  +  void *pCtx,
         1321  +  int bLog,
         1322  +  i64 iOff,
         1323  +  int nData,
         1324  +  int nUs
         1325  +){
         1326  +  InsertWriteHook *pHook = (InsertWriteHook *)pCtx;
         1327  +  if( bLog ) return;
         1328  +
         1329  +  if( nData==0 ){
         1330  +    flushHook(pHook);
         1331  +    fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db"));
         1332  +  }else if( pHook->nData 
         1333  +         && bLog==pHook->bLog 
         1334  +         && iOff==(pHook->iOff+pHook->nData) 
         1335  +  ){
         1336  +    pHook->nData += nData;
         1337  +  }else{
         1338  +    flushHook(pHook);
         1339  +    pHook->bLog = bLog;
         1340  +    pHook->iOff = iOff;
         1341  +    pHook->nData = nData;
         1342  +  }
         1343  +}
         1344  +
         1345  +static int do_replay(int nArg, char **azArg){
         1346  +  char aBuf[4096];
         1347  +  FILE *pInput;
         1348  +  FILE *pClose = 0;
         1349  +  const char *zDb;
         1350  +
         1351  +  lsm_env *pEnv;
         1352  +  lsm_file *pOut;
         1353  +  int rc;
         1354  +
         1355  +  if( nArg!=2 ){
         1356  +    testPrintError("Usage: replay WRITELOG FILE\n");
         1357  +    return 1;
         1358  +  }
         1359  +
         1360  +  if( strcmp(azArg[0], "-")==0 ){
         1361  +    pInput = stdin;
         1362  +  }else{
         1363  +    pClose = pInput = fopen(azArg[0], "r");
         1364  +  }
         1365  +  zDb = azArg[1];
         1366  +  pEnv = tdb_lsm_env();
         1367  +  rc = pEnv->xOpen(pEnv, zDb, 0, &pOut);
         1368  +  if( rc!=LSM_OK ) return rc;
         1369  +
         1370  +  while( feof(pInput)==0 ){
         1371  +    char zLine[80];
         1372  +    fgets(zLine, sizeof(zLine)-1, pInput);
         1373  +    zLine[sizeof(zLine)-1] = '\0';
         1374  +
         1375  +    if( 0==memcmp("sync db", zLine, 7) ){
         1376  +      rc = pEnv->xSync(pOut);
         1377  +      if( rc!=0 ) break;
         1378  +    }else{
         1379  +      int iOff;
         1380  +      int nData;
         1381  +      int nMatch;
         1382  +      nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData);
         1383  +      if( nMatch==2 ){
         1384  +        int i;
         1385  +        for(i=0; i<nData; i+=sizeof(aBuf)){
         1386  +          memset(aBuf, i&0xFF, sizeof(aBuf));
         1387  +          rc = pEnv->xWrite(pOut, iOff+i, aBuf, sizeof(aBuf));
         1388  +          if( rc!=0 ) break;
         1389  +        }
         1390  +      }
         1391  +    }
         1392  +  }
         1393  +  if( pClose ) fclose(pClose);
         1394  +  pEnv->xClose(pOut);
         1395  +
         1396  +  return rc;
         1397  +}
         1398  +
         1399  +static int do_insert(int nArg, char **azArg){
         1400  +  const char *zDb = "lsm";
         1401  +  TestDb *pDb = 0;
         1402  +  int i;
         1403  +  int rc;
         1404  +  const int nRow = 1 * 1000 * 1000;
         1405  +
         1406  +  DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 };
         1407  +  Datasource *pData = 0;
         1408  +
         1409  +  if( nArg>1 ){
         1410  +    testPrintError("Usage: insert ?DATABASE?\n");
         1411  +    return 1;
         1412  +  }
         1413  +  if( nArg==1 ){ zDb = azArg[0]; }
         1414  +
         1415  +  testMallocUninstall(tdb_lsm_env());
         1416  +  for(i=0; zDb[i] && zDb[i]!='='; i++);
         1417  +  if( zDb[i] ){
         1418  +    rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb);
         1419  +  }else{
         1420  +    rc = tdb_open(zDb, 0, 1, &pDb);
         1421  +  }
         1422  +
         1423  +  if( rc!=0 ){
         1424  +    testPrintError("Error opening db \"%s\": %d\n", zDb, rc);
         1425  +  }else{
         1426  +    InsertWriteHook hook;
         1427  +    memset(&hook, 0, sizeof(hook));
         1428  +    hook.pOut = fopen("writelog.txt", "w");
         1429  +
         1430  +    pData = testDatasourceNew(&defn);
         1431  +    tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0);
         1432  +    tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook);
         1433  +
         1434  +    if( rc==0 ){
         1435  +      for(i=0; i<nRow; i++){
         1436  +        void *pKey; int nKey;     /* Database key to insert */
         1437  +        void *pVal; int nVal;     /* Database value to insert */
         1438  +        testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
         1439  +        tdb_write(pDb, pKey, nKey, pVal, nVal);
         1440  +      }
         1441  +    }
         1442  +
         1443  +    testDatasourceFree(pData);
         1444  +    tdb_close(pDb);
         1445  +    flushHook(&hook);
         1446  +    fclose(hook.pOut);
         1447  +  }
         1448  +  testMallocInstall(tdb_lsm_env());
         1449  +
         1450  +  return rc;
         1451  +}
         1452  +
         1453  +static int st_do_show(int a, char **b)      { return do_show(a, b); }
         1454  +static int st_do_work(int a, char **b)      { return do_work(a, b); }
         1455  +static int st_do_io(int a, char **b)        { return do_io(a, b); }
         1456  +
         1457  +#ifdef __linux__
         1458  +#include <sys/time.h>
         1459  +#include <sys/resource.h>
         1460  +
         1461  +static void lsmtest_rusage_report(void){
         1462  +  struct rusage r;
         1463  +  memset(&r, 0, sizeof(r));
         1464  +
         1465  +  getrusage(RUSAGE_SELF, &r);
         1466  +  printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", 
         1467  +      (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock
         1468  +  );
         1469  +}
         1470  +#else
         1471  +static void lsmtest_rusage_report(void){
         1472  +  /* no-op */
         1473  +}
         1474  +#endif
         1475  +
         1476  +int main(int argc, char **argv){
         1477  +  struct TestFunc {
         1478  +    const char *zName;
         1479  +    int bRusageReport;
         1480  +    int (*xFunc)(int, char **);
         1481  +  } aTest[] = {
         1482  +    {"random",      1, do_random_tests},
         1483  +    {"writespeed",  1, do_writer_test},
         1484  +    {"io",          1, st_do_io},
         1485  +
         1486  +    {"insert",      1, do_insert},
         1487  +    {"replay",      1, do_replay},
         1488  +
         1489  +    {"speed",       1, do_speed_tests},
         1490  +    {"speed2",      1, do_speed_test2},
         1491  +    {"show",        0, st_do_show},
         1492  +    {"work",        1, st_do_work},
         1493  +    {"test",        1, do_test},
         1494  +
         1495  +    {0, 0}
         1496  +  };
         1497  +  int rc;                         /* Return Code */
         1498  +  int iFunc;                      /* Index into aTest[] */
         1499  +
         1500  +  int nLeakAlloc = 0;             /* Allocations leaked by lsm */
         1501  +  int nLeakByte = 0;              /* Bytes leaked by lsm */
         1502  +
         1503  +#ifdef LSM_DEBUG_MEM
         1504  +  FILE *pReport = 0;              /* lsm malloc() report file */
         1505  +  const char *zReport = "malloc.txt generated";
         1506  +#else
         1507  +  const char *zReport = "malloc.txt NOT generated";
         1508  +#endif
         1509  +
         1510  +  testMallocInstall(tdb_lsm_env());
         1511  +
         1512  +  if( argc<2 ){
         1513  +    testPrintError("Usage: %s sub-command ?args...?\n", argv[0]);
         1514  +    return -1;
         1515  +  }
         1516  +
         1517  +  /* Initialize error reporting */
         1518  +  testErrorInit(argc, argv);
         1519  +
         1520  +  /* Initialize PRNG system */
         1521  +  testPrngInit();
         1522  +
         1523  +  rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc);
         1524  +  if( rc==0 ){
         1525  +    rc = aTest[iFunc].xFunc(argc-2, &argv[2]);
         1526  +  }
         1527  +
         1528  +#ifdef LSM_DEBUG_MEM
         1529  +  pReport = fopen("malloc.txt", "w");
         1530  +  testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport);
         1531  +  fclose(pReport);
         1532  +#else
         1533  +  testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0);
         1534  +#endif
         1535  +
         1536  +  if( nLeakAlloc ){
         1537  +    testPrintError("Leaked %d bytes in %d allocations (%s)\n", 
         1538  +        nLeakByte, nLeakAlloc, zReport
         1539  +    );
         1540  +    if( rc==0 ) rc = -1;
         1541  +  }
         1542  +  testMallocUninstall(tdb_lsm_env());
         1543  +
         1544  +  if( aTest[iFunc].bRusageReport ){
         1545  +    lsmtest_rusage_report();
         1546  +  }
         1547  +  return rc;
         1548  +}

Added ext/lsm1/lsm-test/lsmtest_mem.c.

            1  +
            2  +#include <stdio.h>
            3  +#include <assert.h>
            4  +#include <string.h>
            5  +
            6  +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0])))
            7  +
            8  +#define MIN(x,y) ((x)<(y) ? (x) : (y))
            9  +
           10  +typedef unsigned int  u32;
           11  +typedef unsigned char u8;
           12  +typedef long long int i64;
           13  +typedef unsigned long long int u64;
           14  +
           15  +#if defined(__GLIBC__) && defined(LSM_DEBUG_MEM)
           16  +  extern int backtrace(void**,int);
           17  +  extern void backtrace_symbols_fd(void*const*,int,int);
           18  +# define TM_BACKTRACE 12
           19  +#else
           20  +# define backtrace(A,B) 1
           21  +# define backtrace_symbols_fd(A,B,C)
           22  +#endif
           23  +
           24  +
           25  +typedef struct TmBlockHdr TmBlockHdr;
           26  +typedef struct TmAgg TmAgg;
           27  +typedef struct TmGlobal TmGlobal;
           28  +
           29  +struct TmGlobal {
           30  +  /* Linked list of all currently outstanding allocations. And a table of
           31  +  ** all allocations, past and present, indexed by backtrace() info.  */
           32  +  TmBlockHdr *pFirst;
           33  +#ifdef TM_BACKTRACE
           34  +  TmAgg *aHash[10000];
           35  +#endif
           36  +
           37  +  /* Underlying malloc/realloc/free functions */
           38  +  void *(*xMalloc)(int);          /* underlying malloc(3) function */
           39  +  void *(*xRealloc)(void *, int); /* underlying realloc(3) function */
           40  +  void (*xFree)(void *);          /* underlying free(3) function */
           41  +
           42  +  /* Mutex to protect pFirst and aHash */
           43  +  void (*xEnterMutex)(TmGlobal*); /* Call this to enter the mutex */
           44  +  void (*xLeaveMutex)(TmGlobal*); /* Call this to leave mutex */
           45  +  void (*xDelMutex)(TmGlobal*);   /* Call this to delete mutex */
           46  +  void *pMutex;                   /* Mutex handle */
           47  +
           48  +  void *(*xSaveMalloc)(void *, size_t);
           49  +  void *(*xSaveRealloc)(void *, void *, size_t);
           50  +  void (*xSaveFree)(void *, void *);
           51  +
           52  +  /* OOM injection scheduling. If nCountdown is greater than zero when a 
           53  +  ** malloc attempt is made, it is decremented. If this means nCountdown 
           54  +  ** transitions from 1 to 0, then the allocation fails. If bPersist is true 
           55  +  ** when this happens, nCountdown is then incremented back to 1 (so that the 
           56  +  ** next attempt fails too).  
           57  +  */
           58  +  int nCountdown;
           59  +  int bPersist;
           60  +  int bEnable;
           61  +  void (*xHook)(void *);
           62  +  void *pHookCtx;
           63  +};
           64  +
           65  +struct TmBlockHdr {
           66  +  TmBlockHdr *pNext;
           67  +  TmBlockHdr *pPrev;
           68  +  int nByte;
           69  +#ifdef TM_BACKTRACE
           70  +  TmAgg *pAgg;
           71  +#endif
           72  +  u32 iForeGuard;
           73  +};
           74  +
           75  +#ifdef TM_BACKTRACE
           76  +struct TmAgg {
           77  +  int nAlloc;                     /* Number of allocations at this path */
           78  +  int nByte;                      /* Total number of bytes allocated */
           79  +  int nOutAlloc;                  /* Number of outstanding allocations */
           80  +  int nOutByte;                   /* Number of outstanding bytes */
           81  +  void *aFrame[TM_BACKTRACE];     /* backtrace() output */
           82  +  TmAgg *pNext;                   /* Next object in hash-table collision */
           83  +};
           84  +#endif
           85  +
           86  +#define FOREGUARD 0x80F5E153
           87  +#define REARGUARD 0xE4676B53
           88  +static const u32 rearguard = REARGUARD;
           89  +
           90  +#define ROUND8(x) (((x)+7)&~7)
           91  +
           92  +#define BLOCK_HDR_SIZE (ROUND8( sizeof(TmBlockHdr) ))
           93  +
           94  +static void lsmtest_oom_error(void){
           95  +  static int nErr = 0;
           96  +  nErr++;
           97  +}
           98  +
           99  +static void tmEnterMutex(TmGlobal *pTm){
          100  +  pTm->xEnterMutex(pTm);
          101  +}
          102  +static void tmLeaveMutex(TmGlobal *pTm){
          103  +  pTm->xLeaveMutex(pTm);
          104  +}
          105  +
          106  +static void *tmMalloc(TmGlobal *pTm, int nByte){
          107  +  TmBlockHdr *pNew;               /* New allocation header block */
          108  +  u8 *pUser;                      /* Return value */
          109  +  int nReq;                       /* Total number of bytes requested */
          110  +
          111  +  assert( sizeof(rearguard)==4 );
          112  +  nReq = BLOCK_HDR_SIZE + nByte + 4;
          113  +  pNew = (TmBlockHdr *)pTm->xMalloc(nReq);
          114  +  memset(pNew, 0, sizeof(TmBlockHdr));
          115  +
          116  +  tmEnterMutex(pTm);
          117  +  assert( pTm->nCountdown>=0 );
          118  +  assert( pTm->bPersist==0 || pTm->bPersist==1 );
          119  +
          120  +  if( pTm->bEnable && pTm->nCountdown==1 ){
          121  +    /* Simulate an OOM error. */
          122  +    lsmtest_oom_error();
          123  +    pTm->xFree(pNew);
          124  +    pTm->nCountdown = pTm->bPersist;
          125  +    if( pTm->xHook ) pTm->xHook(pTm->pHookCtx);
          126  +    pUser = 0;
          127  +  }else{
          128  +    if( pTm->bEnable && pTm->nCountdown ) pTm->nCountdown--;
          129  +
          130  +    pNew->iForeGuard = FOREGUARD;
          131  +    pNew->nByte = nByte;
          132  +    pNew->pNext = pTm->pFirst;
          133  +
          134  +    if( pTm->pFirst ){
          135  +      pTm->pFirst->pPrev = pNew;
          136  +    }
          137  +    pTm->pFirst = pNew;
          138  +
          139  +    pUser = &((u8 *)pNew)[BLOCK_HDR_SIZE];
          140  +    memset(pUser, 0x56, nByte);
          141  +    memcpy(&pUser[nByte], &rearguard, 4);
          142  +
          143  +#ifdef TM_BACKTRACE
          144  +    {
          145  +      TmAgg *pAgg;
          146  +      int i;
          147  +      u32 iHash = 0;
          148  +      void *aFrame[TM_BACKTRACE];
          149  +      memset(aFrame, 0, sizeof(aFrame));
          150  +      backtrace(aFrame, TM_BACKTRACE);
          151  +
          152  +      for(i=0; i<ArraySize(aFrame); i++){
          153  +        iHash += (u64)(aFrame[i]) + (iHash<<3);
          154  +      }
          155  +      iHash = iHash % ArraySize(pTm->aHash);
          156  +
          157  +      for(pAgg=pTm->aHash[iHash]; pAgg; pAgg=pAgg->pNext){
          158  +        if( memcmp(pAgg->aFrame, aFrame, sizeof(aFrame))==0 ) break;
          159  +      }
          160  +      if( !pAgg ){
          161  +        pAgg = (TmAgg *)pTm->xMalloc(sizeof(TmAgg));
          162  +        memset(pAgg, 0, sizeof(TmAgg));
          163  +        memcpy(pAgg->aFrame, aFrame, sizeof(aFrame));
          164  +        pAgg->pNext = pTm->aHash[iHash];
          165  +        pTm->aHash[iHash] = pAgg;
          166  +      }
          167  +      pAgg->nAlloc++;
          168  +      pAgg->nByte += nByte;
          169  +      pAgg->nOutAlloc++;
          170  +      pAgg->nOutByte += nByte;
          171  +      pNew->pAgg = pAgg;
          172  +    }
          173  +#endif
          174  +  }
          175  +
          176  +  tmLeaveMutex(pTm);
          177  +  return pUser;
          178  +}
          179  +
          180  +static void tmFree(TmGlobal *pTm, void *p){
          181  +  if( p ){
          182  +    TmBlockHdr *pHdr;
          183  +    u8 *pUser = (u8 *)p;
          184  +
          185  +    tmEnterMutex(pTm);
          186  +    pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE);
          187  +    assert( pHdr->iForeGuard==FOREGUARD );
          188  +    assert( 0==memcmp(&pUser[pHdr->nByte], &rearguard, 4) );
          189  +
          190  +    if( pHdr->pPrev ){
          191  +      assert( pHdr->pPrev->pNext==pHdr );
          192  +      pHdr->pPrev->pNext = pHdr->pNext;
          193  +    }else{
          194  +      assert( pHdr==pTm->pFirst );
          195  +      pTm->pFirst = pHdr->pNext;
          196  +    }
          197  +    if( pHdr->pNext ){
          198  +      assert( pHdr->pNext->pPrev==pHdr );
          199  +      pHdr->pNext->pPrev = pHdr->pPrev;
          200  +    }
          201  +
          202  +#ifdef TM_BACKTRACE
          203  +    pHdr->pAgg->nOutAlloc--;
          204  +    pHdr->pAgg->nOutByte -= pHdr->nByte;
          205  +#endif
          206  +
          207  +    tmLeaveMutex(pTm);
          208  +    memset(pUser, 0x58, pHdr->nByte);
          209  +    memset(pHdr, 0x57, sizeof(TmBlockHdr));
          210  +    pTm->xFree(pHdr);
          211  +  }
          212  +}
          213  +
          214  +static void *tmRealloc(TmGlobal *pTm, void *p, int nByte){
          215  +  void *pNew;
          216  +
          217  +  pNew = tmMalloc(pTm, nByte);
          218  +  if( pNew && p ){
          219  +    TmBlockHdr *pHdr;
          220  +    u8 *pUser = (u8 *)p;
          221  +    pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE);
          222  +    memcpy(pNew, p, MIN(nByte, pHdr->nByte));
          223  +    tmFree(pTm, p);
          224  +  }
          225  +  return pNew;
          226  +}
          227  +
          228  +static void tmMallocOom(
          229  +  TmGlobal *pTm, 
          230  +  int nCountdown, 
          231  +  int bPersist,
          232  +  void (*xHook)(void *),
          233  +  void *pHookCtx
          234  +){
          235  +  assert( nCountdown>=0 );
          236  +  assert( bPersist==0 || bPersist==1 );
          237  +  pTm->nCountdown = nCountdown;
          238  +  pTm->bPersist = bPersist;
          239  +  pTm->xHook = xHook;
          240  +  pTm->pHookCtx = pHookCtx;
          241  +  pTm->bEnable = 1;
          242  +}
          243  +
          244  +static void tmMallocOomEnable(
          245  +  TmGlobal *pTm, 
          246  +  int bEnable
          247  +){
          248  +  pTm->bEnable = bEnable;
          249  +}
          250  +
          251  +static void tmMallocCheck(
          252  +  TmGlobal *pTm,
          253  +  int *pnLeakAlloc,
          254  +  int *pnLeakByte,
          255  +  FILE *pFile
          256  +){
          257  +  TmBlockHdr *pHdr;
          258  +  int nLeak = 0;
          259  +  int nByte = 0;
          260  +
          261  +  if( pTm==0 ) return;
          262  +
          263  +  for(pHdr=pTm->pFirst; pHdr; pHdr=pHdr->pNext){
          264  +    nLeak++; 
          265  +    nByte += pHdr->nByte;
          266  +  }
          267  +  if( pnLeakAlloc ) *pnLeakAlloc = nLeak;
          268  +  if( pnLeakByte ) *pnLeakByte = nByte;
          269  +
          270  +#ifdef TM_BACKTRACE
          271  +  if( pFile ){
          272  +    int i;
          273  +    fprintf(pFile, "LEAKS\n");
          274  +    for(i=0; i<ArraySize(pTm->aHash); i++){
          275  +      TmAgg *pAgg;
          276  +      for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){
          277  +        if( pAgg->nOutAlloc ){
          278  +          int j;
          279  +          fprintf(pFile, "%d %d ", pAgg->nOutByte, pAgg->nOutAlloc);
          280  +          for(j=0; j<TM_BACKTRACE; j++){
          281  +            fprintf(pFile, "%p ", pAgg->aFrame[j]);
          282  +          }
          283  +          fprintf(pFile, "\n");
          284  +        }
          285  +      }
          286  +    }
          287  +    fprintf(pFile, "\nALLOCATIONS\n");
          288  +    for(i=0; i<ArraySize(pTm->aHash); i++){
          289  +      TmAgg *pAgg;
          290  +      for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){
          291  +        int j;
          292  +        fprintf(pFile, "%d %d ", pAgg->nByte, pAgg->nAlloc);
          293  +        for(j=0; j<TM_BACKTRACE; j++) fprintf(pFile, "%p ", pAgg->aFrame[j]);
          294  +        fprintf(pFile, "\n");
          295  +      }
          296  +    }
          297  +  }
          298  +#else
          299  +  (void)pFile;
          300  +#endif
          301  +}
          302  +
          303  +
          304  +#include "lsm.h"
          305  +#include "stdlib.h"
          306  +
          307  +typedef struct LsmMutex LsmMutex;
          308  +struct LsmMutex {
          309  +  lsm_env *pEnv;
          310  +  lsm_mutex *pMutex;
          311  +};
          312  +
          313  +static void tmLsmMutexEnter(TmGlobal *pTm){
          314  +  LsmMutex *p = (LsmMutex *)pTm->pMutex;
          315  +  p->pEnv->xMutexEnter(p->pMutex);
          316  +}
          317  +static void tmLsmMutexLeave(TmGlobal *pTm){
          318  +  LsmMutex *p = (LsmMutex *)(pTm->pMutex);
          319  +  p->pEnv->xMutexLeave(p->pMutex);
          320  +}
          321  +static void tmLsmMutexDel(TmGlobal *pTm){
          322  +  LsmMutex *p = (LsmMutex *)pTm->pMutex;
          323  +  pTm->xFree(p);
          324  +}
          325  +static void *tmLsmMalloc(int n){ return malloc(n); }
          326  +static void tmLsmFree(void *ptr){ free(ptr); }
          327  +static void *tmLsmRealloc(void *ptr, int n){ return realloc(ptr, n); }
          328  +
          329  +static void *tmLsmEnvMalloc(lsm_env *p, size_t n){ 
          330  +  return tmMalloc((TmGlobal *)(p->pMemCtx), n); 
          331  +}
          332  +static void tmLsmEnvFree(lsm_env *p, void *ptr){ 
          333  +  tmFree((TmGlobal *)(p->pMemCtx), ptr); 
          334  +}
          335  +static void *tmLsmEnvRealloc(lsm_env *p, void *ptr, size_t n){ 
          336  +  return tmRealloc((TmGlobal *)(p->pMemCtx), ptr, n);
          337  +}
          338  +
          339  +void testMallocInstall(lsm_env *pEnv){
          340  +  TmGlobal *pGlobal;
          341  +  LsmMutex *pMutex;
          342  +  assert( pEnv->pMemCtx==0 );
          343  +
          344  +  /* Allocate and populate a TmGlobal structure. */
          345  +  pGlobal = (TmGlobal *)tmLsmMalloc(sizeof(TmGlobal));
          346  +  memset(pGlobal, 0, sizeof(TmGlobal));
          347  +  pGlobal->xMalloc = tmLsmMalloc;
          348  +  pGlobal->xRealloc = tmLsmRealloc;
          349  +  pGlobal->xFree = tmLsmFree;
          350  +  pMutex = (LsmMutex *)pGlobal->xMalloc(sizeof(LsmMutex));
          351  +  pMutex->pEnv = pEnv;
          352  +  pEnv->xMutexStatic(pEnv, LSM_MUTEX_HEAP, &pMutex->pMutex);
          353  +  pGlobal->xEnterMutex = tmLsmMutexEnter;
          354  +  pGlobal->xLeaveMutex = tmLsmMutexLeave;
          355  +  pGlobal->xDelMutex = tmLsmMutexDel;
          356  +  pGlobal->pMutex = (void *)pMutex;
          357  +
          358  +  pGlobal->xSaveMalloc = pEnv->xMalloc;
          359  +  pGlobal->xSaveRealloc = pEnv->xRealloc;
          360  +  pGlobal->xSaveFree = pEnv->xFree;
          361  +
          362  +  /* Set up pEnv to the use the new TmGlobal */
          363  +  pEnv->pMemCtx = (void *)pGlobal;
          364  +  pEnv->xMalloc = tmLsmEnvMalloc;
          365  +  pEnv->xRealloc = tmLsmEnvRealloc;
          366  +  pEnv->xFree = tmLsmEnvFree;
          367  +}
          368  +
          369  +void testMallocUninstall(lsm_env *pEnv){
          370  +  TmGlobal *p = (TmGlobal *)pEnv->pMemCtx;
          371  +  pEnv->pMemCtx = 0;
          372  +  if( p ){
          373  +    pEnv->xMalloc = p->xSaveMalloc;
          374  +    pEnv->xRealloc = p->xSaveRealloc;
          375  +    pEnv->xFree = p->xSaveFree;
          376  +    p->xDelMutex(p);
          377  +    tmLsmFree(p);
          378  +  }
          379  +}
          380  +
          381  +void testMallocCheck(
          382  +  lsm_env *pEnv,
          383  +  int *pnLeakAlloc,
          384  +  int *pnLeakByte,
          385  +  FILE *pFile
          386  +){
          387  +  if( pEnv->pMemCtx==0 ){
          388  +    *pnLeakAlloc = 0;
          389  +    *pnLeakByte = 0;
          390  +  }else{
          391  +    tmMallocCheck((TmGlobal *)(pEnv->pMemCtx), pnLeakAlloc, pnLeakByte, pFile);
          392  +  }
          393  +}
          394  +
          395  +void testMallocOom(
          396  +  lsm_env *pEnv, 
          397  +  int nCountdown, 
          398  +  int bPersist,
          399  +  void (*xHook)(void *),
          400  +  void *pHookCtx
          401  +){
          402  +  TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx);
          403  +  tmMallocOom(pTm, nCountdown, bPersist, xHook, pHookCtx);
          404  +}
          405  +
          406  +void testMallocOomEnable(lsm_env *pEnv, int bEnable){
          407  +  TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx);
          408  +  tmMallocOomEnable(pTm, bEnable);
          409  +}

Added ext/lsm1/lsm-test/lsmtest_tdb.c.

            1  +
            2  +/*
            3  +** This program attempts to test the correctness of some facets of the 
            4  +** LSM database library. Specifically, that the contents of the database
            5  +** are maintained correctly during a series of inserts and deletes.
            6  +*/
            7  +
            8  +
            9  +#include "lsmtest_tdb.h"
           10  +#include "lsm.h"
           11  +
           12  +#include "lsmtest.h"
           13  +
           14  +#include <stdlib.h>
           15  +#include <string.h>
           16  +#include <assert.h>
           17  +#ifndef _WIN32
           18  +# include <unistd.h>
           19  +#endif
           20  +#include <stdio.h>
           21  +
           22  +
           23  +typedef struct SqlDb SqlDb;
           24  +
           25  +static int error_transaction_function(TestDb *p, int iLevel){ 
           26  +  unused_parameter(p);
           27  +  unused_parameter(iLevel);
           28  +  return -1; 
           29  +}
           30  +
           31  +
           32  +/*************************************************************************
           33  +** Begin wrapper for LevelDB.
           34  +*/
           35  +#ifdef HAVE_LEVELDB
           36  +
           37  +#include <leveldb/c.h>
           38  +
           39  +typedef struct LevelDb LevelDb;
           40  +struct LevelDb {
           41  +  TestDb base;
           42  +  leveldb_t *db;
           43  +  leveldb_options_t *pOpt;
           44  +  leveldb_writeoptions_t *pWriteOpt;
           45  +  leveldb_readoptions_t *pReadOpt;
           46  +
           47  +  char *pVal;
           48  +};
           49  +
           50  +static int test_leveldb_close(TestDb *pTestDb){
           51  +  LevelDb *pDb = (LevelDb *)pTestDb;
           52  +
           53  +  leveldb_close(pDb->db);
           54  +  leveldb_writeoptions_destroy(pDb->pWriteOpt);
           55  +  leveldb_readoptions_destroy(pDb->pReadOpt);
           56  +  leveldb_options_destroy(pDb->pOpt);
           57  +  free(pDb->pVal);
           58  +  free(pDb);
           59  +
           60  +  return 0;
           61  +}
           62  +
           63  +static int test_leveldb_write(
           64  +  TestDb *pTestDb, 
           65  +  void *pKey, 
           66  +  int nKey, 
           67  +  void *pVal, 
           68  +  int nVal
           69  +){
           70  +  LevelDb *pDb = (LevelDb *)pTestDb;
           71  +  char *zErr = 0;
           72  +  leveldb_put(pDb->db, pDb->pWriteOpt, pKey, nKey, pVal, nVal, &zErr);
           73  +  return (zErr!=0);
           74  +}
           75  +
           76  +static int test_leveldb_delete(TestDb *pTestDb, void *pKey, int nKey){
           77  +  LevelDb *pDb = (LevelDb *)pTestDb;
           78  +  char *zErr = 0;
           79  +  leveldb_delete(pDb->db, pDb->pWriteOpt, pKey, nKey, &zErr);
           80  +  return (zErr!=0);
           81  +}
           82  +
           83  +static int test_leveldb_fetch(
           84  +  TestDb *pTestDb, 
           85  +  void *pKey, 
           86  +  int nKey, 
           87  +  void **ppVal, 
           88  +  int *pnVal
           89  +){
           90  +  LevelDb *pDb = (LevelDb *)pTestDb;
           91  +  char *zErr = 0;
           92  +  size_t nVal = 0;
           93  +
           94  +  if( pKey==0 ) return 0;
           95  +  free(pDb->pVal);
           96  +  pDb->pVal = leveldb_get(pDb->db, pDb->pReadOpt, pKey, nKey, &nVal, &zErr);
           97  +  *ppVal = (void *)(pDb->pVal);
           98  +  if( pDb->pVal==0 ){
           99  +    *pnVal = -1;
          100  +  }else{
          101  +    *pnVal = (int)nVal;
          102  +  }
          103  +
          104  +  return (zErr!=0);
          105  +}
          106  +
          107  +static int test_leveldb_scan(
          108  +  TestDb *pTestDb,
          109  +  void *pCtx,
          110  +  int bReverse,
          111  +  void *pKey1, int nKey1,         /* Start of search */
          112  +  void *pKey2, int nKey2,         /* End of search */
          113  +  void (*xCallback)(void *, void *, int , void *, int)
          114  +){
          115  +  LevelDb *pDb = (LevelDb *)pTestDb;
          116  +  leveldb_iterator_t *iter;
          117  +
          118  +  iter = leveldb_create_iterator(pDb->db, pDb->pReadOpt);
          119  +
          120  +  if( bReverse==0 ){
          121  +    if( pKey1 ){
          122  +      leveldb_iter_seek(iter, pKey1, nKey1);
          123  +    }else{
          124  +      leveldb_iter_seek_to_first(iter);
          125  +    }
          126  +  }else{
          127  +    if( pKey2 ){
          128  +      leveldb_iter_seek(iter, pKey2, nKey2);
          129  +
          130  +      if( leveldb_iter_valid(iter)==0 ){
          131  +        leveldb_iter_seek_to_last(iter);
          132  +      }else{
          133  +        const char *k; size_t n;
          134  +        int res;
          135  +        k = leveldb_iter_key(iter, &n);
          136  +        res = memcmp(k, pKey2, MIN(n, nKey2));
          137  +        if( res==0 ) res = n - nKey2;
          138  +        assert( res>=0 );
          139  +        if( res>0 ){
          140  +          leveldb_iter_prev(iter);
          141  +        }
          142  +      }
          143  +    }else{
          144  +      leveldb_iter_seek_to_last(iter);
          145  +    }
          146  +  }
          147  +
          148  +
          149  +  while( leveldb_iter_valid(iter) ){
          150  +    const char *k; size_t n;
          151  +    const char *v; size_t n2;
          152  +    int res;
          153  +
          154  +    k = leveldb_iter_key(iter, &n);
          155  +    if( bReverse==0 && pKey2 ){
          156  +      res = memcmp(k, pKey2, MIN(n, nKey2));
          157  +      if( res==0 ) res = n - nKey2;
          158  +      if( res>0 ) break;
          159  +    }
          160  +    if( bReverse!=0 && pKey1 ){
          161  +      res = memcmp(k, pKey1, MIN(n, nKey1));
          162  +      if( res==0 ) res = n - nKey1;
          163  +      if( res<0 ) break;
          164  +    }
          165  +
          166  +    v = leveldb_iter_value(iter, &n2);
          167  +
          168  +    xCallback(pCtx, (void *)k, n, (void *)v, n2);
          169  +
          170  +    if( bReverse==0 ){
          171  +      leveldb_iter_next(iter);
          172  +    }else{
          173  +      leveldb_iter_prev(iter);
          174  +    }
          175  +  }
          176  +
          177  +  leveldb_iter_destroy(iter);
          178  +  return 0;
          179  +}
          180  +
          181  +static int test_leveldb_open(
          182  +  const char *zSpec, 
          183  +  const char *zFilename, 
          184  +  int bClear, 
          185  +  TestDb **ppDb
          186  +){
          187  +  static const DatabaseMethods LeveldbMethods = {
          188  +    test_leveldb_close,
          189  +    test_leveldb_write,
          190  +    test_leveldb_delete,
          191  +    0,
          192  +    test_leveldb_fetch,
          193  +    test_leveldb_scan,
          194  +    error_transaction_function,
          195  +    error_transaction_function,
          196  +    error_transaction_function
          197  +  };
          198  +
          199  +  LevelDb *pLevelDb;
          200  +  char *zErr = 0;
          201  +
          202  +  if( bClear ){
          203  +    char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename);
          204  +    system(zCmd);
          205  +    sqlite3_free(zCmd);
          206  +  }
          207  +
          208  +  pLevelDb = (LevelDb *)malloc(sizeof(LevelDb));
          209  +  memset(pLevelDb, 0, sizeof(LevelDb));
          210  +
          211  +  pLevelDb->pOpt = leveldb_options_create();
          212  +  leveldb_options_set_create_if_missing(pLevelDb->pOpt, 1);
          213  +  pLevelDb->pWriteOpt = leveldb_writeoptions_create();
          214  +  pLevelDb->pReadOpt = leveldb_readoptions_create();
          215  +
          216  +  pLevelDb->db = leveldb_open(pLevelDb->pOpt, zFilename, &zErr);
          217  +
          218  +  if( zErr ){
          219  +    test_leveldb_close((TestDb *)pLevelDb);
          220  +    *ppDb = 0;
          221  +    return 1;
          222  +  }
          223  +
          224  +  *ppDb = (TestDb *)pLevelDb;
          225  +  pLevelDb->base.pMethods = &LeveldbMethods;
          226  +  return 0;
          227  +}
          228  +#endif  /* HAVE_LEVELDB */
          229  +/* 
          230  +** End wrapper for LevelDB.
          231  +*************************************************************************/
          232  +
          233  +#ifdef HAVE_KYOTOCABINET
          234  +static int kc_close(TestDb *pTestDb){
          235  +  return test_kc_close(pTestDb);
          236  +}
          237  +
          238  +static int kc_write(
          239  +  TestDb *pTestDb, 
          240  +  void *pKey, 
          241  +  int nKey, 
          242  +  void *pVal, 
          243  +  int nVal
          244  +){
          245  +  return test_kc_write(pTestDb, pKey, nKey, pVal, nVal);
          246  +}
          247  +
          248  +static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){
          249  +  return test_kc_delete(pTestDb, pKey, nKey);
          250  +}
          251  +
          252  +static int kc_delete_range(
          253  +  TestDb *pTestDb, 
          254  +  void *pKey1, int nKey1,
          255  +  void *pKey2, int nKey2
          256  +){
          257  +  return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2);
          258  +}
          259  +
          260  +static int kc_fetch(
          261  +  TestDb *pTestDb, 
          262  +  void *pKey, 
          263  +  int nKey, 
          264  +  void **ppVal, 
          265  +  int *pnVal
          266  +){
          267  +  if( pKey==0 ) return LSM_OK;
          268  +  return test_kc_fetch(pTestDb, pKey, nKey, ppVal, pnVal);
          269  +}
          270  +
          271  +static int kc_scan(
          272  +  TestDb *pTestDb,
          273  +  void *pCtx,
          274  +  int bReverse,
          275  +  void *pFirst, int nFirst,
          276  +  void *pLast, int nLast,
          277  +  void (*xCallback)(void *, void *, int , void *, int)
          278  +){
          279  +  return test_kc_scan(
          280  +      pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback
          281  +  );
          282  +}
          283  +
          284  +static int kc_open(
          285  +  const char *zSpec, 
          286  +  const char *zFilename, 
          287  +  int bClear, 
          288  +  TestDb **ppDb
          289  +){
          290  +  static const DatabaseMethods KcdbMethods = {
          291  +    kc_close,
          292  +    kc_write,
          293  +    kc_delete,
          294  +    kc_delete_range,
          295  +    kc_fetch,
          296  +    kc_scan,
          297  +    error_transaction_function,
          298  +    error_transaction_function,
          299  +    error_transaction_function
          300  +  };
          301  +
          302  +  int rc;
          303  +  TestDb *pTestDb = 0;
          304  +
          305  +  rc = test_kc_open(zFilename, bClear, &pTestDb);
          306  +  if( rc!=0 ){
          307  +    *ppDb = 0;
          308  +    return rc;
          309  +  }
          310  +  pTestDb->pMethods = &KcdbMethods;
          311  +  *ppDb = pTestDb;
          312  +  return 0;
          313  +}
          314  +#endif /* HAVE_KYOTOCABINET */
          315  +/* 
          316  +** End wrapper for Kyoto cabinet.
          317  +*************************************************************************/
          318  +
          319  +#ifdef HAVE_MDB
          320  +static int mdb_close(TestDb *pTestDb){
          321  +  return test_mdb_close(pTestDb);
          322  +}
          323  +
          324  +static int mdb_write(
          325  +  TestDb *pTestDb, 
          326  +  void *pKey, 
          327  +  int nKey, 
          328  +  void *pVal, 
          329  +  int nVal
          330  +){
          331  +  return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal);
          332  +}
          333  +
          334  +static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){
          335  +  return test_mdb_delete(pTestDb, pKey, nKey);
          336  +}
          337  +
          338  +static int mdb_fetch(
          339  +  TestDb *pTestDb, 
          340  +  void *pKey, 
          341  +  int nKey, 
          342  +  void **ppVal, 
          343  +  int *pnVal
          344  +){
          345  +  if( pKey==0 ) return LSM_OK;
          346  +  return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal);
          347  +}
          348  +
          349  +static int mdb_scan(
          350  +  TestDb *pTestDb,
          351  +  void *pCtx,
          352  +  int bReverse,
          353  +  void *pFirst, int nFirst,
          354  +  void *pLast, int nLast,
          355  +  void (*xCallback)(void *, void *, int , void *, int)
          356  +){
          357  +  return test_mdb_scan(
          358  +      pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback
          359  +  );
          360  +}
          361  +
          362  +static int mdb_open(
          363  +  const char *zSpec, 
          364  +  const char *zFilename, 
          365  +  int bClear, 
          366  +  TestDb **ppDb
          367  +){
          368  +  static const DatabaseMethods KcdbMethods = {
          369  +    mdb_close,
          370  +    mdb_write,
          371  +    mdb_delete,
          372  +    0,
          373  +    mdb_fetch,
          374  +    mdb_scan,
          375  +    error_transaction_function,
          376  +    error_transaction_function,
          377  +    error_transaction_function
          378  +  };
          379  +
          380  +  int rc;
          381  +  TestDb *pTestDb = 0;
          382  +
          383  +  rc = test_mdb_open(zSpec, zFilename, bClear, &pTestDb);
          384  +  if( rc!=0 ){
          385  +    *ppDb = 0;
          386  +    return rc;
          387  +  }
          388  +  pTestDb->pMethods = &KcdbMethods;
          389  +  *ppDb = pTestDb;
          390  +  return 0;
          391  +}
          392  +#endif /* HAVE_MDB */
          393  +
          394  +/*************************************************************************
          395  +** Begin wrapper for SQLite.
          396  +*/
          397  +
          398  +/*
          399  +** nOpenTrans:
          400  +**   The number of open nested transactions, in the same sense as used
          401  +**   by the tdb_begin/commit/rollback and SQLite 4 KV interfaces. If this
          402  +**   value is 0, there are no transactions open at all. If it is 1, then
          403  +**   there is a read transaction. If it is 2 or greater, then there are
          404  +**   (nOpenTrans-1) nested write transactions open.
          405  +*/
          406  +struct SqlDb {
          407  +  TestDb base;
          408  +  sqlite3 *db;
          409  +  sqlite3_stmt *pInsert;
          410  +  sqlite3_stmt *pDelete;
          411  +  sqlite3_stmt *pDeleteRange;
          412  +  sqlite3_stmt *pFetch;
          413  +  sqlite3_stmt *apScan[8];
          414  +
          415  +  int nOpenTrans;
          416  +
          417  +  /* Used by sql_fetch() to allocate space for results */
          418  +  int nAlloc;
          419  +  u8 *aAlloc;
          420  +};
          421  +
          422  +static int sql_close(TestDb *pTestDb){
          423  +  SqlDb *pDb = (SqlDb *)pTestDb;
          424  +  sqlite3_finalize(pDb->pInsert);
          425  +  sqlite3_finalize(pDb->pDelete);
          426  +  sqlite3_finalize(pDb->pDeleteRange);
          427  +  sqlite3_finalize(pDb->pFetch);
          428  +  sqlite3_finalize(pDb->apScan[0]);
          429  +  sqlite3_finalize(pDb->apScan[1]);
          430  +  sqlite3_finalize(pDb->apScan[2]);
          431  +  sqlite3_finalize(pDb->apScan[3]);
          432  +  sqlite3_finalize(pDb->apScan[4]);
          433  +  sqlite3_finalize(pDb->apScan[5]);
          434  +  sqlite3_finalize(pDb->apScan[6]);
          435  +  sqlite3_finalize(pDb->apScan[7]);
          436  +  sqlite3_close(pDb->db);
          437  +  free((char *)pDb->aAlloc);
          438  +  free((char *)pDb);
          439  +  return SQLITE_OK;
          440  +}
          441  +
          442  +static int sql_write(
          443  +  TestDb *pTestDb, 
          444  +  void *pKey, 
          445  +  int nKey, 
          446  +  void *pVal, 
          447  +  int nVal
          448  +){
          449  +  SqlDb *pDb = (SqlDb *)pTestDb;
          450  +  sqlite3_bind_blob(pDb->pInsert, 1, pKey, nKey, SQLITE_STATIC);
          451  +  sqlite3_bind_blob(pDb->pInsert, 2, pVal, nVal, SQLITE_STATIC);
          452  +  sqlite3_step(pDb->pInsert);
          453  +  return sqlite3_reset(pDb->pInsert);
          454  +}
          455  +
          456  +static int sql_delete(TestDb *pTestDb, void *pKey, int nKey){
          457  +  SqlDb *pDb = (SqlDb *)pTestDb;
          458  +  sqlite3_bind_blob(pDb->pDelete, 1, pKey, nKey, SQLITE_STATIC);
          459  +  sqlite3_step(pDb->pDelete);
          460  +  return sqlite3_reset(pDb->pDelete);
          461  +}
          462  +
          463  +static int sql_delete_range(
          464  +  TestDb *pTestDb, 
          465  +  void *pKey1, int nKey1,
          466  +  void *pKey2, int nKey2
          467  +){
          468  +  SqlDb *pDb = (SqlDb *)pTestDb;
          469  +  sqlite3_bind_blob(pDb->pDeleteRange, 1, pKey1, nKey1, SQLITE_STATIC);
          470  +  sqlite3_bind_blob(pDb->pDeleteRange, 2, pKey2, nKey2, SQLITE_STATIC);
          471  +  sqlite3_step(pDb->pDeleteRange);
          472  +  return sqlite3_reset(pDb->pDeleteRange);
          473  +}
          474  +
          475  +static int sql_fetch(
          476  +  TestDb *pTestDb, 
          477  +  void *pKey, 
          478  +  int nKey, 
          479  +  void **ppVal, 
          480  +  int *pnVal
          481  +){
          482  +  SqlDb *pDb = (SqlDb *)pTestDb;
          483  +  int rc;
          484  +
          485  +  sqlite3_reset(pDb->pFetch);
          486  +  if( pKey==0 ){
          487  +    assert( ppVal==0 );
          488  +    assert( pnVal==0 );
          489  +    return LSM_OK;
          490  +  }
          491  +
          492  +  sqlite3_bind_blob(pDb->pFetch, 1, pKey, nKey, SQLITE_STATIC);
          493  +  rc = sqlite3_step(pDb->pFetch);
          494  +  if( rc==SQLITE_ROW ){
          495  +    int nVal = sqlite3_column_bytes(pDb->pFetch, 0);
          496  +    u8 *aVal = (void *)sqlite3_column_blob(pDb->pFetch, 0);
          497  +
          498  +    if( nVal>pDb->nAlloc ){
          499  +      free(pDb->aAlloc);
          500  +      pDb->aAlloc = (u8 *)malloc(nVal*2);
          501  +      pDb->nAlloc = nVal*2;
          502  +    }
          503  +    memcpy(pDb->aAlloc, aVal, nVal);
          504  +    *pnVal = nVal;
          505  +    *ppVal = (void *)pDb->aAlloc;
          506  +  }else{
          507  +    *pnVal = -1;
          508  +    *ppVal = 0;
          509  +  }
          510  +
          511  +  rc = sqlite3_reset(pDb->pFetch);
          512  +  return rc;
          513  +}
          514  +
          515  +static int sql_scan(
          516  +  TestDb *pTestDb,
          517  +  void *pCtx,
          518  +  int bReverse,
          519  +  void *pFirst, int nFirst,
          520  +  void *pLast, int nLast,
          521  +  void (*xCallback)(void *, void *, int , void *, int)
          522  +){
          523  +  SqlDb *pDb = (SqlDb *)pTestDb;
          524  +  sqlite3_stmt *pScan;
          525  +
          526  +  assert( bReverse==1 || bReverse==0 );
          527  +  pScan = pDb->apScan[(pFirst==0) + (pLast==0)*2 + bReverse*4];
          528  +
          529  +  if( pFirst ) sqlite3_bind_blob(pScan, 1, pFirst, nFirst, SQLITE_STATIC);
          530  +  if( pLast ) sqlite3_bind_blob(pScan, 2, pLast, nLast, SQLITE_STATIC);
          531  +
          532  +  while( SQLITE_ROW==sqlite3_step(pScan) ){
          533  +    void *pKey; int nKey;
          534  +    void *pVal; int nVal;
          535  +
          536  +    nKey = sqlite3_column_bytes(pScan, 0);
          537  +    pKey = (void *)sqlite3_column_blob(pScan, 0);
          538  +    nVal = sqlite3_column_bytes(pScan, 1);
          539  +    pVal = (void *)sqlite3_column_blob(pScan, 1);
          540  +
          541  +    xCallback(pCtx, pKey, nKey, pVal, nVal);
          542  +  }
          543  +  return sqlite3_reset(pScan);
          544  +}
          545  +
          546  +static int sql_begin(TestDb *pTestDb, int iLevel){
          547  +  int i;
          548  +  SqlDb *pDb = (SqlDb *)pTestDb;
          549  +
          550  +  /* iLevel==0 is a no-op */
          551  +  if( iLevel==0 ) return 0;
          552  +
          553  +  /* If there are no transactions at all open, open a read transaction. */
          554  +  if( pDb->nOpenTrans==0 ){
          555  +    int rc = sqlite3_exec(pDb->db, 
          556  +        "BEGIN; SELECT * FROM sqlite_master LIMIT 1;" , 0, 0, 0
          557  +    );
          558  +    if( rc!=0 ) return rc;
          559  +    pDb->nOpenTrans = 1;
          560  +  }
          561  +
          562  +  /* Open any required write transactions */
          563  +  for(i=pDb->nOpenTrans; i<iLevel; i++){
          564  +    char *zSql = sqlite3_mprintf("SAVEPOINT x%d", i);
          565  +    int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0);
          566  +    sqlite3_free(zSql);
          567  +    if( rc!=SQLITE_OK ) return rc;
          568  +  }
          569  +
          570  +  pDb->nOpenTrans = iLevel;
          571  +  return 0;
          572  +}
          573  +
          574  +static int sql_commit(TestDb *pTestDb, int iLevel){
          575  +  SqlDb *pDb = (SqlDb *)pTestDb;
          576  +  assert( iLevel>=0 );
          577  +
          578  +  /* Close the read transaction if requested. */
          579  +  if( pDb->nOpenTrans>=1 && iLevel==0 ){
          580  +    int rc = sqlite3_exec(pDb->db, "COMMIT", 0, 0, 0);
          581  +    if( rc!=0 ) return rc;
          582  +    pDb->nOpenTrans = 0;
          583  +  }
          584  +
          585  +  /* Close write transactions as required */
          586  +  if( pDb->nOpenTrans>iLevel ){
          587  +    char *zSql = sqlite3_mprintf("RELEASE x%d", iLevel);
          588  +    int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0);
          589  +    sqlite3_free(zSql);
          590  +    if( rc!=0 ) return rc;
          591  +  }
          592  +
          593  +  pDb->nOpenTrans = iLevel;
          594  +  return 0;
          595  +}
          596  +
          597  +static int sql_rollback(TestDb *pTestDb, int iLevel){
          598  +  SqlDb *pDb = (SqlDb *)pTestDb;
          599  +  assert( iLevel>=0 );
          600  +
          601  +  if( pDb->nOpenTrans>=1 && iLevel==0 ){
          602  +    /* Close the read transaction if requested. */
          603  +    int rc = sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0);
          604  +    if( rc!=0 ) return rc;
          605  +  }else if( pDb->nOpenTrans>1 && iLevel==1 ){
          606  +    /* Or, rollback and close the top-level write transaction */
          607  +    int rc = sqlite3_exec(pDb->db, "ROLLBACK TO x1; RELEASE x1;", 0, 0, 0);
          608  +    if( rc!=0 ) return rc;
          609  +  }else{
          610  +    /* Or, just roll back some nested transactions */
          611  +    char *zSql = sqlite3_mprintf("ROLLBACK TO x%d", iLevel-1);
          612  +    int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0);
          613  +    sqlite3_free(zSql);
          614  +    if( rc!=0 ) return rc;
          615  +  }
          616  +
          617  +  pDb->nOpenTrans = iLevel;
          618  +  return 0;
          619  +}
          620  +
          621  +static int sql_open(
          622  +  const char *zSpec, 
          623  +  const char *zFilename, 
          624  +  int bClear, 
          625  +  TestDb **ppDb
          626  +){
          627  +  static const DatabaseMethods SqlMethods = {
          628  +    sql_close,
          629  +    sql_write,
          630  +    sql_delete,
          631  +    sql_delete_range,
          632  +    sql_fetch,
          633  +    sql_scan,
          634  +    sql_begin,
          635  +    sql_commit,
          636  +    sql_rollback
          637  +  };
          638  +  const char *zCreate = "CREATE TABLE IF NOT EXISTS t1(k PRIMARY KEY, v)";
          639  +  const char *zInsert = "REPLACE INTO t1 VALUES(?, ?)";
          640  +  const char *zDelete = "DELETE FROM t1 WHERE k = ?";
          641  +  const char *zRange = "DELETE FROM t1 WHERE k>? AND k<?";
          642  +  const char *zFetch  = "SELECT v FROM t1 WHERE k = ?";
          643  +
          644  +  const char *zScan0  = "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k";
          645  +  const char *zScan1  = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k";
          646  +  const char *zScan2  = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k";
          647  +  const char *zScan3  = "SELECT * FROM t1 ORDER BY k";
          648  +
          649  +  const char *zScan4  = 
          650  +    "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k DESC";
          651  +  const char *zScan5  = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k DESC";
          652  +  const char *zScan6  = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k DESC";
          653  +  const char *zScan7  = "SELECT * FROM t1 ORDER BY k DESC";
          654  +
          655  +  int rc;
          656  +  SqlDb *pDb;
          657  +  char *zPragma;
          658  +
          659  +  if( bClear && zFilename && zFilename[0] ){
          660  +    unlink(zFilename);
          661  +  }
          662  +
          663  +  pDb = (SqlDb *)malloc(sizeof(SqlDb));
          664  +  memset(pDb, 0, sizeof(SqlDb));
          665  +  pDb->base.pMethods = &SqlMethods;
          666  +
          667  +  if( 0!=(rc = sqlite3_open(zFilename, &pDb->db))
          668  +   || 0!=(rc = sqlite3_exec(pDb->db, zCreate, 0, 0, 0))
          669  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zInsert, -1, &pDb->pInsert, 0))
          670  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zDelete, -1, &pDb->pDelete, 0))
          671  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zRange, -1, &pDb->pDeleteRange, 0))
          672  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zFetch, -1, &pDb->pFetch, 0))
          673  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan0, -1, &pDb->apScan[0], 0))
          674  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan1, -1, &pDb->apScan[1], 0))
          675  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan2, -1, &pDb->apScan[2], 0))
          676  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan3, -1, &pDb->apScan[3], 0))
          677  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan4, -1, &pDb->apScan[4], 0))
          678  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan5, -1, &pDb->apScan[5], 0))
          679  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan6, -1, &pDb->apScan[6], 0))
          680  +   || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan7, -1, &pDb->apScan[7], 0))
          681  +  ){
          682  +    *ppDb = 0;
          683  +    sql_close((TestDb *)pDb);
          684  +    return rc;
          685  +  }
          686  +
          687  +  zPragma = sqlite3_mprintf("PRAGMA page_size=%d", TESTDB_DEFAULT_PAGE_SIZE);
          688  +  sqlite3_exec(pDb->db, zPragma, 0, 0, 0);
          689  +  sqlite3_free(zPragma);
          690  +  zPragma = sqlite3_mprintf("PRAGMA cache_size=%d", TESTDB_DEFAULT_CACHE_SIZE);
          691  +  sqlite3_exec(pDb->db, zPragma, 0, 0, 0);
          692  +  sqlite3_free(zPragma);
          693  +
          694  +  /* sqlite3_exec(pDb->db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0); */
          695  +  sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0);
          696  +  sqlite3_exec(pDb->db, "PRAGMA journal_mode=WAL", 0, 0, 0);
          697  +  sqlite3_exec(pDb->db, "PRAGMA wal_autocheckpoint=4096", 0, 0, 0);
          698  +  if( zSpec ){
          699  +    rc = sqlite3_exec(pDb->db, zSpec, 0, 0, 0);
          700  +    if( rc!=SQLITE_OK ){
          701  +      sql_close((TestDb *)pDb);
          702  +      return rc;
          703  +    }
          704  +  }
          705  +
          706  +  *ppDb = (TestDb *)pDb;
          707  +  return 0;
          708  +}
          709  +/* 
          710  +** End wrapper for SQLite.
          711  +*************************************************************************/
          712  +
          713  +/*************************************************************************
          714  +** Begin exported functions.
          715  +*/
          716  +static struct Lib {
          717  +  const char *zName;
          718  +  const char *zDefaultDb;
          719  +  int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb);
          720  +} aLib[] = {
          721  +  { "sqlite3",      "testdb.sqlite",    sql_open },
          722  +  { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
          723  +  { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
          724  +#ifdef HAVE_ZLIB
          725  +  { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
          726  +#endif
          727  +  { "lsm",          "testdb.lsm",       test_lsm_open },
          728  +#ifdef LSM_MUTEX_PTHREADS
          729  +  { "lsm_mt2",      "testdb.lsm_mt2",   test_lsm_mt2 },
          730  +  { "lsm_mt3",      "testdb.lsm_mt3",   test_lsm_mt3 },
          731  +#endif
          732  +#ifdef HAVE_LEVELDB
          733  +  { "leveldb",      "testdb.leveldb",   test_leveldb_open },
          734  +#endif
          735  +#ifdef HAVE_KYOTOCABINET
          736  +  { "kyotocabinet", "testdb.kc",        kc_open },
          737  +#endif
          738  +#ifdef HAVE_MDB
          739  +  { "mdb", "./testdb.mdb",        mdb_open }
          740  +#endif
          741  +};
          742  +
          743  +const char *tdb_system_name(int i){
          744  +  if( i<0 || i>=ArraySize(aLib) ) return 0;
          745  +  return aLib[i].zName;
          746  +}
          747  +
          748  +const char *tdb_default_db(const char *zSys){
          749  +  int i;
          750  +  for(i=0; i<ArraySize(aLib); i++){
          751  +    if( strcmp(aLib[i].zName, zSys)==0 ) return aLib[i].zDefaultDb;
          752  +  }
          753  +  return 0;
          754  +}
          755  +
          756  +int tdb_open(const char *zLib, const char *zDb, int bClear, TestDb **ppDb){
          757  +  int i;
          758  +  int rc = 1;
          759  +  const char *zSpec = 0;
          760  +
          761  +  int nLib = 0;
          762  +  while( zLib[nLib] && zLib[nLib]!=' ' ){
          763  +    nLib++;
          764  +  }
          765  +  zSpec = &zLib[nLib];
          766  +  while( *zSpec==' ' ) zSpec++;
          767  +  if( *zSpec=='\0' ) zSpec = 0;
          768  +
          769  +  for(i=0; i<ArraySize(aLib); i++){
          770  +    if( (int)strlen(aLib[i].zName)==nLib
          771  +        && 0==memcmp(zLib, aLib[i].zName, nLib) ){
          772  +      rc = aLib[i].xOpen(zSpec, (zDb ? zDb : aLib[i].zDefaultDb), bClear, ppDb);
          773  +      if( rc==0 ){
          774  +        (*ppDb)->zLibrary = aLib[i].zName;
          775  +      }
          776  +      break;
          777  +    }
          778  +  }
          779  +
          780  +  if( rc ){
          781  +    /* Failed to find the requested database library. Return an error. */
          782  +    *ppDb = 0;
          783  +  }
          784  +  return rc;
          785  +}
          786  +
          787  +int tdb_close(TestDb *pDb){
          788  +  if( pDb ){
          789  +    return pDb->pMethods->xClose(pDb);
          790  +  }
          791  +  return 0;
          792  +}
          793  +
          794  +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){
          795  +  return pDb->pMethods->xWrite(pDb, pKey, nKey, pVal, nVal);
          796  +}
          797  +
          798  +int tdb_delete(TestDb *pDb, void *pKey, int nKey){
          799  +  return pDb->pMethods->xDelete(pDb, pKey, nKey);
          800  +}
          801  +
          802  +int tdb_delete_range(
          803  +    TestDb *pDb, void *pKey1, int nKey1, void *pKey2, int nKey2
          804  +){
          805  +  return pDb->pMethods->xDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2);
          806  +}
          807  +
          808  +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal){
          809  +  return pDb->pMethods->xFetch(pDb, pKey, nKey, ppVal, pnVal);
          810  +}
          811  +
          812  +int tdb_scan(
          813  +  TestDb *pDb,                    /* Database handle */
          814  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          815  +  int bReverse,                   /* True to scan in reverse order */
          816  +  void *pKey1, int nKey1,         /* Start of search */
          817  +  void *pKey2, int nKey2,         /* End of search */
          818  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          819  +){
          820  +  return pDb->pMethods->xScan(
          821  +      pDb, pCtx, bReverse, pKey1, nKey1, pKey2, nKey2, xCallback
          822  +  );
          823  +}
          824  +
          825  +int tdb_begin(TestDb *pDb, int iLevel){
          826  +  return pDb->pMethods->xBegin(pDb, iLevel);
          827  +}
          828  +int tdb_commit(TestDb *pDb, int iLevel){
          829  +  return pDb->pMethods->xCommit(pDb, iLevel);
          830  +}
          831  +int tdb_rollback(TestDb *pDb, int iLevel){
          832  +  return pDb->pMethods->xRollback(pDb, iLevel);
          833  +}
          834  +
          835  +int tdb_transaction_support(TestDb *pDb){
          836  +  return (pDb->pMethods->xBegin != error_transaction_function);
          837  +}
          838  +
          839  +const char *tdb_library_name(TestDb *pDb){
          840  +  return pDb->zLibrary;
          841  +}
          842  +
          843  +/* 
          844  +** End exported functions.
          845  +*************************************************************************/

Added ext/lsm1/lsm-test/lsmtest_tdb.h.

            1  +
            2  +/*
            3  +** This file is the interface to a very simple database library used for
            4  +** testing. The interface is similar to that of the LSM. The main virtue 
            5  +** of this library is that the same API may be used to access a key-value
            6  +** store implemented by LSM, SQLite or another database system. Which 
            7  +** makes it easy to use for correctness and performance tests.
            8  +*/
            9  +
           10  +#ifndef __WRAPPER_H_
           11  +#define __WRAPPER_H_
           12  +
           13  +#ifdef __cplusplus
           14  +extern "C" {
           15  +#endif
           16  +
           17  +#include "lsm.h"
           18  +
           19  +typedef struct TestDb TestDb;
           20  +
           21  +/*
           22  +** Open a new database connection. The first argument is the name of the
           23  +** database library to use. e.g. something like:
           24  +**
           25  +**     "sqlite3"
           26  +**     "lsm"
           27  +**
           28  +** See function tdb_system_name() for a list of available database systems.
           29  +**
           30  +** The second argument is the name of the database to open (e.g. a filename).
           31  +**
           32  +** If the third parameter is non-zero, then any existing database by the
           33  +** name of zDb is removed before opening a new one. If it is zero, then an
           34  +** existing database may be opened.
           35  +*/
           36  +int tdb_open(const char *zLibrary, const char *zDb, int bClear, TestDb **ppDb);
           37  +
           38  +/*
           39  +** Close a database handle.
           40  +*/
           41  +int tdb_close(TestDb *pDb);
           42  +
           43  +/*
           44  +** Write a new key/value into the database.
           45  +*/
           46  +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal);
           47  +
           48  +/*
           49  +** Delete a key from the database.
           50  +*/
           51  +int tdb_delete(TestDb *pDb, void *pKey, int nKey);
           52  +
           53  +/*
           54  +** Delete a range of keys from the database.
           55  +*/
           56  +int tdb_delete_range(TestDb *, void *pKey1, int nKey1, void *pKey2, int nKey2);
           57  +
           58  +/*
           59  +** Query the database for key (pKey/nKey). If no entry is found, set *ppVal
           60  +** to 0 and *pnVal to -1 before returning. Otherwise, set *ppVal and *pnVal
           61  +** to a pointer to and size of the value associated with (pKey/nKey).
           62  +*/
           63  +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal);
           64  +
           65  +/*
           66  +** Open and close nested transactions. Currently, these functions only 
           67  +** work for SQLite3 and LSM systems. Use the tdb_transaction_support() 
           68  +** function to determine if a given TestDb handle supports these methods.
           69  +**
           70  +** These functions and the iLevel parameter follow the same conventions as
           71  +** the SQLite 4 transaction interface. Note that this is slightly different
           72  +** from the way LSM does things. As follows:
           73  +**
           74  +** tdb_begin():
           75  +**   A successful call to tdb_begin() with (iLevel>1) guarantees that 
           76  +**   there are at least (iLevel-1) write transactions open. If iLevel==1,
           77  +**   then it guarantees that at least a read-transaction is open. Calling
           78  +**   tdb_begin() with iLevel==0 is a no-op.
           79  +**
           80  +** tdb_commit():
           81  +**   A successful call to tdb_commit() with (iLevel>1) guarantees that 
           82  +**   there are at most (iLevel-1) write transactions open. If iLevel==1,
           83  +**   then it guarantees that there are no write transactions open (although
           84  +**   a read-transaction may remain open).  Calling tdb_commit() with 
           85  +**   iLevel==0 ensures that all transactions, read or write, have been 
           86  +**   closed and committed.
           87  +**
           88  +** tdb_rollback():
           89  +**   This call is similar to tdb_commit(), except that instead of committing
           90  +**   transactions, it reverts them. For example, calling tdb_rollback() with
           91  +**   iLevel==2 ensures that there is at most one write transaction open, and
           92  +**   restores the database to the state that it was in when that transaction
           93  +**   was opened.
           94  +**
           95  +**   In other words, tdb_commit() just closes transactions - tdb_rollback()
           96  +**   closes transactions and then restores the database to the state it
           97  +**   was in before those transactions were even opened.
           98  +*/
           99  +int tdb_begin(TestDb *pDb, int iLevel);
          100  +int tdb_commit(TestDb *pDb, int iLevel);
          101  +int tdb_rollback(TestDb *pDb, int iLevel);
          102  +
          103  +/*
          104  +** Return true if transactions are supported, or false otherwise.
          105  +*/
          106  +int tdb_transaction_support(TestDb *pDb);
          107  +
          108  +/*
          109  +** Return the name of the database library (as passed to tdb_open()) used
          110  +** by the handled passed as the first argument.
          111  +*/
          112  +const char *tdb_library_name(TestDb *pDb);
          113  +
          114  +/*
          115  +** Scan a range of database keys. Invoke the callback function for each
          116  +** key visited.
          117  +*/
          118  +int tdb_scan(
          119  +  TestDb *pDb,                    /* Database handle */
          120  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          121  +  int bReverse,                   /* True to scan in reverse order */
          122  +  void *pKey1, int nKey1,         /* Start of search */
          123  +  void *pKey2, int nKey2,         /* End of search */
          124  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          125  +);
          126  +
          127  +const char *tdb_system_name(int i);
          128  +const char *tdb_default_db(const char *zSys);
          129  +
          130  +int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb);
          131  +
          132  +/*
          133  +** If the TestDb handle passed as an argument is a wrapper around an LSM
          134  +** database, return the LSM handle. Otherwise, if the argument is some other
          135  +** database system, return NULL.
          136  +*/
          137  +lsm_db *tdb_lsm(TestDb *pDb);
          138  +
          139  +/*
          140  +** Return true if the db passed as an argument is a multi-threaded LSM
          141  +** connection.
          142  +*/
          143  +int tdb_lsm_multithread(TestDb *pDb);
          144  +
          145  +/*
          146  +** Return a pointer to the lsm_env object used by all lsm database
          147  +** connections initialized as a copy of the object returned by 
          148  +** lsm_default_env(). It may be modified (e.g. to override functions)
          149  +** if the caller can guarantee that it is not already in use.
          150  +*/
          151  +lsm_env *tdb_lsm_env(void);
          152  +
          153  +/*
          154  +** The following functions only work with LSM database handles. It is
          155  +** illegal to call them with any other type of database handle specified
          156  +** as an argument.
          157  +*/
          158  +void tdb_lsm_enable_log(TestDb *pDb, int bEnable);
          159  +void tdb_lsm_application_crash(TestDb *pDb);
          160  +void tdb_lsm_prepare_system_crash(TestDb *pDb);
          161  +void tdb_lsm_system_crash(TestDb *pDb);
          162  +void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync);
          163  +
          164  +
          165  +void tdb_lsm_safety(TestDb *pDb, int eMode);
          166  +void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *);
          167  +void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*);
          168  +int tdb_lsm_config_str(TestDb *pDb, const char *zStr);
          169  +
          170  +#ifdef __cplusplus
          171  +}  /* End of the 'extern "C"' block */
          172  +#endif
          173  +
          174  +#endif

Added ext/lsm1/lsm-test/lsmtest_tdb2.cc.

            1  +
            2  +
            3  +#include "lsmtest.h"
            4  +#include <stdlib.h>
            5  +
            6  +#ifdef HAVE_KYOTOCABINET
            7  +#include "kcpolydb.h"
            8  +extern "C" {
            9  +  struct KcDb {
           10  +    TestDb base;
           11  +    kyotocabinet::TreeDB* db;
           12  +    char *pVal;
           13  +  };
           14  +}
           15  +
           16  +int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb){
           17  +  KcDb *pKcDb;
           18  +  int ok;
           19  +  int rc = 0;
           20  +
           21  +  if( bClear ){
           22  +    char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename);
           23  +    system(zCmd);
           24  +    sqlite3_free(zCmd);
           25  +  }
           26  +
           27  +  pKcDb = (KcDb *)malloc(sizeof(KcDb));
           28  +  memset(pKcDb, 0, sizeof(KcDb));
           29  +
           30  +
           31  +  pKcDb->db = new kyotocabinet::TreeDB();
           32  +  pKcDb->db->tune_page(TESTDB_DEFAULT_PAGE_SIZE);
           33  +  pKcDb->db->tune_page_cache(
           34  +      TESTDB_DEFAULT_PAGE_SIZE * TESTDB_DEFAULT_CACHE_SIZE
           35  +  );
           36  +  ok = pKcDb->db->open(zFilename,
           37  +      kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE
           38  +  );
           39  +  if( ok==0 ){
           40  +    free(pKcDb);
           41  +    pKcDb = 0;
           42  +    rc = 1;
           43  +  }
           44  +
           45  +  *ppDb = (TestDb *)pKcDb;
           46  +  return rc;
           47  +}
           48  +
           49  +int test_kc_close(TestDb *pDb){
           50  +  KcDb *pKcDb = (KcDb *)pDb;
           51  +  if( pKcDb->pVal ){
           52  +    delete [] pKcDb->pVal;
           53  +  }
           54  +  pKcDb->db->close();
           55  +  delete pKcDb->db;
           56  +  free(pKcDb);
           57  +  return 0;
           58  +}
           59  +
           60  +int test_kc_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){
           61  +  KcDb *pKcDb = (KcDb *)pDb;
           62  +  int ok;
           63  +
           64  +  ok = pKcDb->db->set((const char *)pKey, nKey, (const char *)pVal, nVal);
           65  +  return (ok ? 0 : 1);
           66  +}
           67  +
           68  +int test_kc_delete(TestDb *pDb, void *pKey, int nKey){
           69  +  KcDb *pKcDb = (KcDb *)pDb;
           70  +  int ok;
           71  +
           72  +  ok = pKcDb->db->remove((const char *)pKey, nKey);
           73  +  return (ok ? 0 : 1);
           74  +}
           75  +
           76  +int test_kc_delete_range(
           77  +  TestDb *pDb, 
           78  +  void *pKey1, int nKey1,
           79  +  void *pKey2, int nKey2
           80  +){
           81  +  int res;
           82  +  KcDb *pKcDb = (KcDb *)pDb;
           83  +  kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor();
           84  +
           85  +  if( pKey1 ){
           86  +    res = pCur->jump((const char *)pKey1, nKey1);
           87  +  }else{
           88  +    res = pCur->jump();
           89  +  }
           90  +
           91  +  while( 1 ){
           92  +    const char *pKey; size_t nKey;
           93  +    const char *pVal; size_t nVal;
           94  +
           95  +    pKey = pCur->get(&nKey, &pVal, &nVal);
           96  +    if( pKey==0 ) break;
           97  +
           98  +#ifndef NDEBUG
           99  +    if( pKey1 ){
          100  +      res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey));
          101  +      assert( res>0 || (res==0 && nKey>nKey1) );
          102  +    }
          103  +#endif
          104  +
          105  +    if( pKey2 ){
          106  +      res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey));
          107  +      if( res>0 || (res==0 && (size_t)nKey2<nKey) ){
          108  +        delete [] pKey;
          109  +        break;
          110  +      }
          111  +    }
          112  +    pCur->remove();
          113  +    delete [] pKey;
          114  +  }
          115  +
          116  +  delete pCur;
          117  +  return 0;
          118  +}
          119  +
          120  +int test_kc_fetch(
          121  +  TestDb *pDb, 
          122  +  void *pKey, 
          123  +  int nKey, 
          124  +  void **ppVal,
          125  +  int *pnVal
          126  +){
          127  +  KcDb *pKcDb = (KcDb *)pDb;
          128  +  size_t nVal;
          129  +
          130  +  if( pKcDb->pVal ){
          131  +    delete [] pKcDb->pVal;
          132  +    pKcDb->pVal = 0;
          133  +  }
          134  +
          135  +  pKcDb->pVal = pKcDb->db->get((const char *)pKey, nKey, &nVal);
          136  +  if( pKcDb->pVal ){
          137  +    *ppVal = pKcDb->pVal;
          138  +    *pnVal = nVal;
          139  +  }else{
          140  +    *ppVal = 0;
          141  +    *pnVal = -1;
          142  +  }
          143  +
          144  +  return 0;
          145  +}
          146  +
          147  +int test_kc_scan(
          148  +  TestDb *pDb,                    /* Database handle */
          149  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          150  +  int bReverse,                   /* True for a reverse order scan */
          151  +  void *pKey1, int nKey1,         /* Start of search */
          152  +  void *pKey2, int nKey2,         /* End of search */
          153  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          154  +){
          155  +  KcDb *pKcDb = (KcDb *)pDb;
          156  +  kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor();
          157  +  int res;
          158  +
          159  +  if( bReverse==0 ){
          160  +    if( pKey1 ){
          161  +      res = pCur->jump((const char *)pKey1, nKey1);
          162  +    }else{
          163  +      res = pCur->jump();
          164  +    }
          165  +  }else{
          166  +    if( pKey2 ){
          167  +      res = pCur->jump_back((const char *)pKey2, nKey2);
          168  +    }else{
          169  +      res = pCur->jump_back();
          170  +    }
          171  +  }
          172  +
          173  +  while( res ){
          174  +    const char *pKey; size_t nKey;
          175  +    const char *pVal; size_t nVal;
          176  +    pKey = pCur->get(&nKey, &pVal, &nVal);
          177  +
          178  +    if( bReverse==0 && pKey2 ){
          179  +      res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey));
          180  +      if( res>0 || (res==0 && (size_t)nKey2<nKey) ){
          181  +        delete [] pKey;
          182  +        break;
          183  +      }
          184  +    }else if( bReverse!=0 && pKey1 ){
          185  +      res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey));
          186  +      if( res<0 || (res==0 && (size_t)nKey1>nKey) ){
          187  +        delete [] pKey;
          188  +        break;
          189  +      }
          190  +    }
          191  +
          192  +    xCallback(pCtx, (void *)pKey, (int)nKey, (void *)pVal, (int)nVal);
          193  +    delete [] pKey;
          194  +
          195  +    if( bReverse ){
          196  +      res = pCur->step_back();
          197  +    }else{
          198  +      res = pCur->step();
          199  +    }
          200  +  }
          201  +
          202  +  delete pCur;
          203  +  return 0;
          204  +}
          205  +#endif /* HAVE_KYOTOCABINET */
          206  +
          207  +#ifdef HAVE_MDB 
          208  +#include "lmdb.h"
          209  +
          210  +extern "C" {
          211  +  struct MdbDb {
          212  +    TestDb base;
          213  +    MDB_env *env;
          214  +    MDB_dbi dbi;
          215  +  };
          216  +}
          217  +
          218  +int test_mdb_open(
          219  +  const char *zSpec, 
          220  +  const char *zFilename, 
          221  +  int bClear, 
          222  +  TestDb **ppDb
          223  +){
          224  +  MDB_txn *txn;
          225  +  MdbDb *pMdb;
          226  +  int rc;
          227  +
          228  +  if( bClear ){
          229  +    char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename);
          230  +    system(zCmd);
          231  +    sqlite3_free(zCmd);
          232  +  }
          233  +
          234  +  pMdb = (MdbDb *)malloc(sizeof(MdbDb));
          235  +  memset(pMdb, 0, sizeof(MdbDb));
          236  +
          237  +  rc = mdb_env_create(&pMdb->env);
          238  +  if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024);
          239  +  if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600);
          240  +  if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn);
          241  +  if( rc==0 ){
          242  +    rc = mdb_open(txn, NULL, 0, &pMdb->dbi);
          243  +    mdb_txn_commit(txn);
          244  +  }
          245  +
          246  +  *ppDb = (TestDb *)pMdb;
          247  +  return rc;
          248  +}
          249  +
          250  +int test_mdb_close(TestDb *pDb){
          251  +  MdbDb *pMdb = (MdbDb *)pDb;
          252  +
          253  +  mdb_close(pMdb->env, pMdb->dbi);
          254  +  mdb_env_close(pMdb->env);
          255  +  free(pMdb);
          256  +  return 0;
          257  +}
          258  +
          259  +int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){
          260  +  int rc;
          261  +  MdbDb *pMdb = (MdbDb *)pDb;
          262  +  MDB_val val;
          263  +  MDB_val key;
          264  +  MDB_txn *txn;
          265  +
          266  +  val.mv_size = nVal; 
          267  +  val.mv_data = pVal;
          268  +  key.mv_size = nKey; 
          269  +  key.mv_data = pKey;
          270  +
          271  +  rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn);
          272  +  if( rc==0 ){
          273  +    rc = mdb_put(txn, pMdb->dbi, &key, &val, 0);
          274  +    if( rc==0 ){
          275  +      rc = mdb_txn_commit(txn);
          276  +    }else{
          277  +      mdb_txn_abort(txn);
          278  +    }
          279  +  }
          280  +  
          281  +  return rc;
          282  +}
          283  +
          284  +int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){
          285  +  int rc;
          286  +  MdbDb *pMdb = (MdbDb *)pDb;
          287  +  MDB_val key;
          288  +  MDB_txn *txn;
          289  +
          290  +  key.mv_size = nKey; 
          291  +  key.mv_data = pKey;
          292  +  rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn);
          293  +  if( rc==0 ){
          294  +    rc = mdb_del(txn, pMdb->dbi, &key, 0);
          295  +    if( rc==0 ){
          296  +      rc = mdb_txn_commit(txn);
          297  +    }else{
          298  +      mdb_txn_abort(txn);
          299  +    }
          300  +  }
          301  +  
          302  +  return rc;
          303  +}
          304  +
          305  +int test_mdb_fetch(
          306  +  TestDb *pDb, 
          307  +  void *pKey, 
          308  +  int nKey, 
          309  +  void **ppVal,
          310  +  int *pnVal
          311  +){
          312  +  int rc;
          313  +  MdbDb *pMdb = (MdbDb *)pDb;
          314  +  MDB_val key;
          315  +  MDB_txn *txn;
          316  +
          317  +  key.mv_size = nKey;
          318  +  key.mv_data = pKey;
          319  +
          320  +  rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn);
          321  +  if( rc==0 ){
          322  +    MDB_val val = {0, 0};
          323  +    rc = mdb_get(txn, pMdb->dbi, &key, &val);
          324  +    if( rc==MDB_NOTFOUND ){
          325  +      rc = 0;
          326  +      *ppVal = 0;
          327  +      *pnVal = -1;
          328  +    }else{
          329  +      *ppVal = val.mv_data;
          330  +      *pnVal = val.mv_size;
          331  +    }
          332  +    mdb_txn_commit(txn);
          333  +  }
          334  +
          335  +  return rc;
          336  +}
          337  +
          338  +int test_mdb_scan(
          339  +  TestDb *pDb,                    /* Database handle */
          340  +  void *pCtx,                     /* Context pointer to pass to xCallback */
          341  +  int bReverse,                   /* True for a reverse order scan */
          342  +  void *pKey1, int nKey1,         /* Start of search */
          343  +  void *pKey2, int nKey2,         /* End of search */
          344  +  void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal)
          345  +){
          346  +  MdbDb *pMdb = (MdbDb *)pDb;
          347  +  int rc;
          348  +  MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT;
          349  +  MDB_txn *txn;
          350  +
          351  +  rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn);
          352  +  if( rc==0 ){
          353  +    MDB_cursor *csr;
          354  +    MDB_val key = {0, 0};
          355  +    MDB_val val = {0, 0};
          356  +
          357  +    rc = mdb_cursor_open(txn, pMdb->dbi, &csr);
          358  +    if( rc==0 ){
          359  +      while( mdb_cursor_get(csr, &key, &val, op)==0 ){
          360  +        xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size);
          361  +      }
          362  +      mdb_cursor_close(csr);
          363  +    }
          364  +  }
          365  +
          366  +  return rc;
          367  +}
          368  +
          369  +#endif /* HAVE_MDB */
          370  +

Added ext/lsm1/lsm-test/lsmtest_tdb3.c.

            1  +
            2  +#include "lsmtest_tdb.h"
            3  +#include "lsm.h"
            4  +#include "lsmtest.h"
            5  +
            6  +#include <stdlib.h>
            7  +#include <string.h>
            8  +#include <assert.h>
            9  +#ifndef _WIN32
           10  +# include <unistd.h>
           11  +#endif
           12  +#include <stdio.h>
           13  +
           14  +#ifndef _WIN32
           15  +# include <sys/time.h>
           16  +#endif
           17  +
           18  +typedef struct LsmDb LsmDb;
           19  +typedef struct LsmWorker LsmWorker;
           20  +typedef struct LsmFile LsmFile;
           21  +
           22  +#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024)
           23  +#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024)
           24  +
           25  +#ifdef LSM_MUTEX_PTHREADS
           26  +#include <pthread.h>
           27  +
           28  +#define LSMTEST_THREAD_CKPT      1
           29  +#define LSMTEST_THREAD_WORKER    2
           30  +#define LSMTEST_THREAD_WORKER_AC 3
           31  +
           32  +/*
           33  +** There are several different types of worker threads that run in different
           34  +** test configurations, depending on the value of LsmWorker.eType.
           35  +**
           36  +**   1. Checkpointer.
           37  +**   2. Worker with auto-checkpoint.
           38  +**   3. Worker without auto-checkpoint.
           39  +*/
           40  +struct LsmWorker {
           41  +  LsmDb *pDb;                     /* Main database structure */
           42  +  lsm_db *pWorker;                /* Worker database handle */
           43  +  pthread_t worker_thread;        /* Worker thread */
           44  +  pthread_cond_t worker_cond;     /* Condition var the worker waits on */
           45  +  pthread_mutex_t worker_mutex;   /* Mutex used with worker_cond */
           46  +  int bDoWork;                    /* Set to true by client when there is work */
           47  +  int worker_rc;                  /* Store error code here */
           48  +  int eType;                      /* LSMTEST_THREAD_XXX constant */
           49  +  int bBlock;
           50  +};
           51  +#else
           52  +struct LsmWorker { int worker_rc; int bBlock; };
           53  +#endif
           54  +
           55  +static void mt_shutdown(LsmDb *);
           56  +
           57  +lsm_env *tdb_lsm_env(void){
           58  +  static int bInit = 0;
           59  +  static lsm_env env;
           60  +  if( bInit==0 ){
           61  +    memcpy(&env, lsm_default_env(), sizeof(env));
           62  +    bInit = 1;
           63  +  }
           64  +  return &env;
           65  +}
           66  +
           67  +typedef struct FileSector FileSector;
           68  +typedef struct FileData FileData;
           69  +
           70  +struct FileSector {
           71  +  u8 *aOld;                       /* Old data for this sector */
           72  +};
           73  +
           74  +struct FileData {
           75  +  int nSector;                    /* Allocated size of apSector[] array */
           76  +  FileSector *aSector;            /* Array of file sectors */
           77  +};
           78  +
           79  +/*
           80  +** bPrepareCrash:
           81  +**   If non-zero, the file wrappers maintain enough in-memory data to
           82  +**   simulate the effect of a power-failure on the file-system (i.e. that
           83  +**   unsynced sectors may be written, not written, or overwritten with
           84  +**   arbitrary data when the crash occurs).
           85  +**
           86  +** bCrashed:
           87  +**   Set to true after a crash is simulated. Once this variable is true, all
           88  +**   VFS methods other than xClose() return LSM_IOERR as soon as they are
           89  +**   called (without affecting the contents of the file-system).
           90  +**
           91  +** env:
           92  +**   The environment object used by all lsm_db* handles opened by this
           93  +**   object (i.e. LsmDb.db plus any worker connections). Variable env.pVfsCtx
           94  +**   always points to the containing LsmDb structure.
           95  +*/
           96  +struct LsmDb {
           97  +  TestDb base;                    /* Base class - methods table */
           98  +  lsm_env env;                    /* Environment used by connection db */
           99  +  char *zName;                    /* Database file name */
          100  +  lsm_db *db;                     /* LSM database handle */
          101  +
          102  +  lsm_cursor *pCsr;               /* Cursor held open during read transaction */
          103  +  void *pBuf;                     /* Buffer for tdb_fetch() output */
          104  +  int nBuf;                       /* Allocated (not used) size of pBuf */
          105  +
          106  +  /* Crash testing related state */
          107  +  int bCrashed;                   /* True once a crash has occurred */
          108  +  int nAutoCrash;                 /* Number of syncs until a crash */
          109  +  int bPrepareCrash;              /* True to store writes in memory */
          110  +
          111  +  /* Unsynced data (while crash testing) */
          112  +  int szSector;                   /* Assumed size of disk sectors (512B) */
          113  +  FileData aFile[2];              /* Database and log file data */
          114  +
          115  +  /* Other test instrumentation */
          116  +  int bNoRecovery;                /* If true, assume DMS2 is locked */
          117  +
          118  +  /* Work hook redirection */
          119  +  void (*xWork)(lsm_db *, void *);
          120  +  void *pWorkCtx;
          121  +
          122  +  /* IO logging hook */
          123  +  void (*xWriteHook)(void *, int, lsm_i64, int, int);
          124  +  void *pWriteCtx;
          125  +  
          126  +  /* Worker threads (for lsm_mt) */
          127  +  int nMtMinCkpt;
          128  +  int nMtMaxCkpt;
          129  +  int eMode;
          130  +  int nWorker;
          131  +  LsmWorker *aWorker;
          132  +};
          133  +
          134  +#define LSMTEST_MODE_SINGLETHREAD    1
          135  +#define LSMTEST_MODE_BACKGROUND_CKPT 2
          136  +#define LSMTEST_MODE_BACKGROUND_WORK 3
          137  +#define LSMTEST_MODE_BACKGROUND_BOTH 4
          138  +
          139  +/*************************************************************************
          140  +**************************************************************************
          141  +** Begin test VFS code.
          142  +*/
          143  +
          144  +struct LsmFile {
          145  +  lsm_file *pReal;                /* Real underlying file */
          146  +  int bLog;                       /* True for log file. False for db file */
          147  +  LsmDb *pDb;                     /* Database handle that uses this file */
          148  +};
          149  +
          150  +static int testEnvFullpath(
          151  +  lsm_env *pEnv,                  /* Environment for current LsmDb */
          152  +  const char *zFile,              /* Relative path name */
          153  +  char *zOut,                     /* Output buffer */
          154  +  int *pnOut                      /* IN/OUT: Size of output buffer */
          155  +){
          156  +  lsm_env *pRealEnv = tdb_lsm_env();
          157  +  return pRealEnv->xFullpath(pRealEnv, zFile, zOut, pnOut);
          158  +}
          159  +
          160  +static int testEnvOpen(
          161  +  lsm_env *pEnv,                  /* Environment for current LsmDb */
          162  +  const char *zFile,              /* Name of file to open */
          163  +  int flags,
          164  +  lsm_file **ppFile               /* OUT: New file handle object */
          165  +){
          166  +  lsm_env *pRealEnv = tdb_lsm_env();
          167  +  LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx;
          168  +  int rc;                         /* Return Code */
          169  +  LsmFile *pRet;                  /* The new file handle */
          170  +  int nFile;                      /* Length of string zFile in bytes */
          171  +
          172  +  nFile = strlen(zFile);
          173  +  pRet = (LsmFile *)testMalloc(sizeof(LsmFile));
          174  +  pRet->pDb = pDb;
          175  +  pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4));
          176  +
          177  +  rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal);
          178  +  if( rc!=LSM_OK ){
          179  +    testFree(pRet);
          180  +    pRet = 0;
          181  +  }
          182  +
          183  +  *ppFile = (lsm_file *)pRet;
          184  +  return rc;
          185  +}
          186  +
          187  +static int testEnvRead(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){
          188  +  lsm_env *pRealEnv = tdb_lsm_env();
          189  +  LsmFile *p = (LsmFile *)pFile;
          190  +  if( p->pDb->bCrashed ) return LSM_IOERR;
          191  +  return pRealEnv->xRead(p->pReal, iOff, pData, nData);
          192  +}
          193  +
          194  +static int testEnvWrite(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){
          195  +  lsm_env *pRealEnv = tdb_lsm_env();
          196  +  LsmFile *p = (LsmFile *)pFile;
          197  +  LsmDb *pDb = p->pDb;
          198  +
          199  +  if( pDb->bCrashed ) return LSM_IOERR;
          200  +
          201  +  if( pDb->bPrepareCrash ){
          202  +    FileData *pData2 = &pDb->aFile[p->bLog];
          203  +    int iFirst;                 
          204  +    int iLast;
          205  +    int iSector;
          206  +
          207  +    iFirst = (int)(iOff / pDb->szSector);
          208  +    iLast =  (int)((iOff + nData - 1) / pDb->szSector);
          209  +
          210  +    if( pData2->nSector<(iLast+1) ){
          211  +      int nNew = ( ((iLast + 1) + 63) / 64 ) * 64;
          212  +      assert( nNew>iLast );
          213  +      pData2->aSector = (FileSector *)testRealloc(
          214  +          pData2->aSector, nNew*sizeof(FileSector)
          215  +      );
          216  +      memset(&pData2->aSector[pData2->nSector], 
          217  +          0, (nNew - pData2->nSector) * sizeof(FileSector)
          218  +      );
          219  +      pData2->nSector = nNew;
          220  +    }
          221  +
          222  +    for(iSector=iFirst; iSector<=iLast; iSector++){
          223  +      if( pData2->aSector[iSector].aOld==0 ){
          224  +        u8 *aOld = (u8 *)testMalloc(pDb->szSector);
          225  +        pRealEnv->xRead(
          226  +            p->pReal, (lsm_i64)iSector*pDb->szSector, aOld, pDb->szSector
          227  +        );
          228  +        pData2->aSector[iSector].aOld = aOld;
          229  +      }
          230  +    }
          231  +  }
          232  +
          233  +  if( pDb->xWriteHook ){
          234  +    int rc;
          235  +    int nUs;
          236  +    struct timeval t1;
          237  +    struct timeval t2;
          238  +
          239  +    gettimeofday(&t1, 0);
          240  +    assert( nData>0 );
          241  +    rc = pRealEnv->xWrite(p->pReal, iOff, pData, nData);
          242  +    gettimeofday(&t2, 0);
          243  +
          244  +    nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec);
          245  +    pDb->xWriteHook(pDb->pWriteCtx, p->bLog, iOff, nData, nUs);
          246  +    return rc;
          247  +  }
          248  +
          249  +  return pRealEnv->xWrite(p->pReal, iOff, pData, nData);
          250  +}
          251  +
          252  +static void doSystemCrash(LsmDb *pDb);
          253  +
          254  +static int testEnvSync(lsm_file *pFile){
          255  +  lsm_env *pRealEnv = tdb_lsm_env();
          256  +  LsmFile *p = (LsmFile *)pFile;
          257  +  LsmDb *pDb = p->pDb;
          258  +  FileData *pData = &pDb->aFile[p->bLog];
          259  +  int i;
          260  +
          261  +  if( pDb->bCrashed ) return LSM_IOERR;
          262  +
          263  +  if( pDb->nAutoCrash ){
          264  +    pDb->nAutoCrash--;
          265  +    if( pDb->nAutoCrash==0 ){
          266  +      doSystemCrash(pDb);
          267  +      pDb->bCrashed = 1;
          268  +      return LSM_IOERR;
          269  +    }
          270  +  }
          271  +
          272  +  if( pDb->bPrepareCrash ){
          273  +    for(i=0; i<pData->nSector; i++){
          274  +      testFree(pData->aSector[i].aOld);
          275  +      pData->aSector[i].aOld = 0;
          276  +    }
          277  +  }
          278  +
          279  +  if( pDb->xWriteHook ){
          280  +    int rc;
          281  +    int nUs;
          282  +    struct timeval t1;
          283  +    struct timeval t2;
          284  +
          285  +    gettimeofday(&t1, 0);
          286  +    rc = pRealEnv->xSync(p->pReal);
          287  +    gettimeofday(&t2, 0);
          288  +
          289  +    nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec);
          290  +    pDb->xWriteHook(pDb->pWriteCtx, p->bLog, 0, 0, nUs);
          291  +    return rc;
          292  +  }
          293  +
          294  +  return pRealEnv->xSync(p->pReal);
          295  +}
          296  +
          297  +static int testEnvTruncate(lsm_file *pFile, lsm_i64 iOff){
          298  +  lsm_env *pRealEnv = tdb_lsm_env();
          299  +  LsmFile *p = (LsmFile *)pFile;
          300  +  if( p->pDb->bCrashed ) return LSM_IOERR;
          301  +  return pRealEnv->xTruncate(p->pReal, iOff);
          302  +}
          303  +
          304  +static int testEnvSectorSize(lsm_file *pFile){
          305  +  lsm_env *pRealEnv = tdb_lsm_env();
          306  +  LsmFile *p = (LsmFile *)pFile;
          307  +  return pRealEnv->xSectorSize(p->pReal);
          308  +}
          309  +
          310  +static int testEnvRemap(
          311  +  lsm_file *pFile, 
          312  +  lsm_i64 iMin, 
          313  +  void **ppOut,
          314  +  lsm_i64 *pnOut
          315  +){
          316  +  lsm_env *pRealEnv = tdb_lsm_env();
          317  +  LsmFile *p = (LsmFile *)pFile;
          318  +  return pRealEnv->xRemap(p->pReal, iMin, ppOut, pnOut);
          319  +}
          320  +
          321  +static int testEnvFileid(
          322  +  lsm_file *pFile, 
          323  +  void *ppOut,
          324  +  int *pnOut
          325  +){
          326  +  lsm_env *pRealEnv = tdb_lsm_env();
          327  +  LsmFile *p = (LsmFile *)pFile;
          328  +  return pRealEnv->xFileid(p->pReal, ppOut, pnOut);
          329  +}
          330  +
          331  +static int testEnvClose(lsm_file *pFile){
          332  +  lsm_env *pRealEnv = tdb_lsm_env();
          333  +  LsmFile *p = (LsmFile *)pFile;
          334  +
          335  +  pRealEnv->xClose(p->pReal);
          336  +  testFree(p);
          337  +  return LSM_OK;
          338  +}
          339  +
          340  +static int testEnvUnlink(lsm_env *pEnv, const char *zFile){
          341  +  lsm_env *pRealEnv = tdb_lsm_env();
          342  +  unused_parameter(pEnv);
          343  +  return pRealEnv->xUnlink(pRealEnv, zFile);
          344  +}
          345  +
          346  +static int testEnvLock(lsm_file *pFile, int iLock, int eType){
          347  +  LsmFile *p = (LsmFile *)pFile;
          348  +  lsm_env *pRealEnv = tdb_lsm_env();
          349  +
          350  +  if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){
          351  +    return LSM_BUSY;
          352  +  }
          353  +  return pRealEnv->xLock(p->pReal, iLock, eType);
          354  +}
          355  +
          356  +static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){
          357  +  LsmFile *p = (LsmFile *)pFile;
          358  +  lsm_env *pRealEnv = tdb_lsm_env();
          359  +
          360  +  if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){
          361  +    return LSM_BUSY;
          362  +  }
          363  +  return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType);
          364  +}
          365  +
          366  +static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){
          367  +  LsmFile *p = (LsmFile *)pFile;
          368  +  lsm_env *pRealEnv = tdb_lsm_env();
          369  +  return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp);
          370  +}
          371  +
          372  +static void testEnvShmBarrier(void){
          373  +}
          374  +
          375  +static int testEnvShmUnmap(lsm_file *pFile, int bDel){
          376  +  LsmFile *p = (LsmFile *)pFile;
          377  +  lsm_env *pRealEnv = tdb_lsm_env();
          378  +  return pRealEnv->xShmUnmap(p->pReal, bDel);
          379  +}
          380  +
          381  +static int testEnvSleep(lsm_env *pEnv, int us){
          382  +  lsm_env *pRealEnv = tdb_lsm_env();
          383  +  return pRealEnv->xSleep(pRealEnv, us);
          384  +}
          385  +
          386  +static void doSystemCrash(LsmDb *pDb){
          387  +  lsm_env *pEnv = tdb_lsm_env();
          388  +  int iFile;
          389  +  int iSeed = pDb->aFile[0].nSector + pDb->aFile[1].nSector;
          390  +
          391  +  char *zFile = pDb->zName;
          392  +  char *zFree = 0;
          393  +
          394  +  for(iFile=0; iFile<2; iFile++){
          395  +    lsm_file *pFile = 0;
          396  +    int i;
          397  +
          398  +    pEnv->xOpen(pEnv, zFile, 0, &pFile);
          399  +    for(i=0; i<pDb->aFile[iFile].nSector; i++){
          400  +      u8 *aOld = pDb->aFile[iFile].aSector[i].aOld;
          401  +      if( aOld ){
          402  +        int iOpt = testPrngValue(iSeed++) % 3;
          403  +        switch( iOpt ){
          404  +          case 0:
          405  +            break;
          406  +
          407  +          case 1:
          408  +            testPrngArray(iSeed++, (u32 *)aOld, pDb->szSector/4);
          409  +            /* Fall-through */
          410  +
          411  +          case 2:
          412  +            pEnv->xWrite(
          413  +                pFile, (lsm_i64)i * pDb->szSector, aOld, pDb->szSector
          414  +            );
          415  +            break;
          416  +        }
          417  +        testFree(aOld);
          418  +        pDb->aFile[iFile].aSector[i].aOld = 0;
          419  +      }
          420  +    }
          421  +    pEnv->xClose(pFile);
          422  +    zFree = zFile = sqlite3_mprintf("%s-log", pDb->zName);
          423  +  }
          424  +
          425  +  sqlite3_free(zFree);
          426  +}
          427  +/*
          428  +** End test VFS code.
          429  +**************************************************************************
          430  +*************************************************************************/
          431  +
          432  +/*************************************************************************
          433  +**************************************************************************
          434  +** Begin test compression hooks.
          435  +*/
          436  +
          437  +#ifdef HAVE_ZLIB
          438  +#include <zlib.h>
          439  +
          440  +static int testZipBound(void *pCtx, int nSrc){
          441  +  return compressBound(nSrc);
          442  +}
          443  +
          444  +static int testZipCompress(
          445  +  void *pCtx,                     /* Context pointer */
          446  +  char *aOut, int *pnOut,         /* OUT: Buffer containing compressed data */
          447  +  const char *aIn, int nIn        /* Buffer containing input data */
          448  +){
          449  +  uLongf n = *pnOut;              /* In/out buffer size for compress() */
          450  +  int rc;                         /* compress() return code */
          451  + 
          452  +  rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn);
          453  +  *pnOut = n;
          454  +  return (rc==Z_OK ? 0 : LSM_ERROR);
          455  +}
          456  +
          457  +static int testZipUncompress(
          458  +  void *pCtx,                     /* Context pointer */
          459  +  char *aOut, int *pnOut,         /* OUT: Buffer containing uncompressed data */
          460  +  const char *aIn, int nIn        /* Buffer containing input data */
          461  +){
          462  +  uLongf n = *pnOut;              /* In/out buffer size for uncompress() */
          463  +  int rc;                         /* uncompress() return code */
          464  +
          465  +  rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn);
          466  +  *pnOut = n;
          467  +  return (rc==Z_OK ? 0 : LSM_ERROR);
          468  +}
          469  +
          470  +static int testConfigureCompression(lsm_db *pDb){
          471  +  static lsm_compress zip = {
          472  +    0,                            /* Context pointer (unused) */
          473  +    1,                            /* Id value */
          474  +    testZipBound,                 /* xBound method */
          475  +    testZipCompress,              /* xCompress method */
          476  +    testZipUncompress             /* xUncompress method */
          477  +  };
          478  +  return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip);
          479  +}
          480  +#endif /* ifdef HAVE_ZLIB */
          481  +
          482  +/*
          483  +** End test compression hooks.
          484  +**************************************************************************
          485  +*************************************************************************/
          486  +
          487  +static int test_lsm_close(TestDb *pTestDb){
          488  +  int i;
          489  +  int rc = LSM_OK;
          490  +  LsmDb *pDb = (LsmDb *)pTestDb;
          491  +
          492  +  lsm_csr_close(pDb->pCsr);
          493  +  lsm_close(pDb->db);
          494  +
          495  +  /* If this is a multi-threaded database, wait on the worker threads. */
          496  +  mt_shutdown(pDb);
          497  +  for(i=0; i<pDb->nWorker && rc==LSM_OK; i++){
          498  +    rc = pDb->aWorker[i].worker_rc;
          499  +  }
          500  +
          501  +  for(i=0; i<pDb->aFile[0].nSector; i++){
          502  +    testFree(pDb->aFile[0].aSector[i].aOld);
          503  +  }
          504  +  testFree(pDb->aFile[0].aSector);
          505  +  for(i=0; i<pDb->aFile[1].nSector; i++){
          506  +    testFree(pDb->aFile[1].aSector[i].aOld);
          507  +  }
          508  +  testFree(pDb->aFile[1].aSector);
          509  +
          510  +  memset(pDb, sizeof(LsmDb), 0x11);
          511  +  testFree((char *)pDb->pBuf);
          512  +  testFree((char *)pDb);
          513  +  return rc;
          514  +}
          515  +
          516  +static void mt_signal_worker(LsmDb*, int);
          517  +
          518  +static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){
          519  +  int nSleep = 0;
          520  +  int nKB;
          521  +  int rc;
          522  +
          523  +  do {
          524  +    nKB = 0;
          525  +    rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB);
          526  +    if( rc!=LSM_OK || nKB<pDb->nMtMaxCkpt ) break;
          527  +#ifdef LSM_MUTEX_PTHREADS
          528  +    mt_signal_worker(pDb, 
          529  +        (pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ? 0 : 1)
          530  +    );
          531  +#endif
          532  +    usleep(5000);
          533  +    nSleep += 5;
          534  +  }while( 1 );
          535  +
          536  +#if 0
          537  +    if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep);
          538  +#endif
          539  +
          540  +  return rc;
          541  +}
          542  +
          543  +static int waitOnWorker(LsmDb *pDb){
          544  +  int rc;
          545  +  int nLimit = -1;
          546  +  int nSleep = 0;
          547  +
          548  +  rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit);
          549  +  do {
          550  +    int nOld, nNew, rc2;
          551  +    rc2 = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew);
          552  +    if( rc2!=LSM_OK ) return rc2;
          553  +    if( nOld==0 || nNew<(nLimit/2) ) break;
          554  +#ifdef LSM_MUTEX_PTHREADS
          555  +    mt_signal_worker(pDb, 0);
          556  +#endif
          557  +    usleep(5000);
          558  +    nSleep += 5;
          559  +  }while( 1 );
          560  +
          561  +#if 0
          562  +  if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep);
          563  +#endif
          564  +
          565  +  return rc;
          566  +}
          567  +
          568  +static int test_lsm_write(
          569  +  TestDb *pTestDb, 
          570  +  void *pKey, 
          571  +  int nKey, 
          572  +  void *pVal,
          573  +  int nVal
          574  +){
          575  +  LsmDb *pDb = (LsmDb *)pTestDb;
          576  +  int rc = LSM_OK;
          577  +
          578  +  if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){
          579  +    rc = waitOnCheckpointer(pDb, pDb->db);
          580  +  }else if( 
          581  +      pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK
          582  +   || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH 
          583  +  ){
          584  +    rc = waitOnWorker(pDb);
          585  +  }
          586  +
          587  +  if( rc==LSM_OK ){
          588  +    rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal);
          589  +  }
          590  +  return rc;
          591  +}
          592  +
          593  +static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){
          594  +  LsmDb *pDb = (LsmDb *)pTestDb;
          595  +  return lsm_delete(pDb->db, pKey, nKey);
          596  +}
          597  +
          598  +static int test_lsm_delete_range(
          599  +  TestDb *pTestDb, 
          600  +  void *pKey1, int nKey1,
          601  +  void *pKey2, int nKey2
          602  +){
          603  +  LsmDb *pDb = (LsmDb *)pTestDb;
          604  +  return lsm_delete_range(pDb->db, pKey1, nKey1, pKey2, nKey2);
          605  +}
          606  +
          607  +static int test_lsm_fetch(
          608  +  TestDb *pTestDb, 
          609  +  void *pKey, 
          610  +  int nKey, 
          611  +  void **ppVal, 
          612  +  int *pnVal
          613  +){
          614  +  int rc;
          615  +  LsmDb *pDb = (LsmDb *)pTestDb;
          616  +  lsm_cursor *csr;
          617  +