/ Changes On Branch est_count_pragma
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch est_count_pragma Excluding Merge-Ins

This is equivalent to a diff from fd81d8a4 to 8d2a1cca

2017-09-22
20:18
Merge in all the trunk enhancements of the previous 7 months. The LIKE optimization has stopped working when there is an ESCAPE - that problem will be addressed in a subsequent check-in. (Leaf check-in: 8d2a1cca user: drh tags: est_count_pragma)
16:23
Use the updated Win32 VFS semantics for winOpen from check-in [5d03c738e9] for WinRT, et al, as well. (check-in: 2c03d8b8 user: mistachkin tags: trunk)
2017-02-16
14:02
Merge recent enhancements from trunk. (check-in: 325ccfa9 user: drh tags: est_count_pragma)
2016-10-20
22:02
Experimental est_count pragma. (check-in: 340822af user: drh tags: est_count_pragma)
18:20
Add the ability for the PRAGMA statement to accept multiple arguments. Currently all arguments other than the first are ignored. (Leaf check-in: fd81d8a4 user: drh tags: multi-arg-pragma)
2016-10-18
16:36
Minor simplification of the comparison opcodes. (check-in: 56474ebc user: drh tags: trunk)

Changes to Makefile.in.

   177    177            icu.lo insert.lo json1.lo legacy.lo loadext.lo \
   178    178            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   179    179            memjournal.lo \
   180    180            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   181    181            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   182    182            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   183    183            random.lo resolve.lo rowset.lo rtree.lo \
   184         -         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
          184  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
   185    185            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   186    186            update.lo util.lo vacuum.lo \
   187    187            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   188    188            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   189    189            utf.lo vtab.lo
   190    190   
   191    191   # Object files for the amalgamation.
................................................................................
   346    346   SRC += \
   347    347     $(TOP)/ext/session/sqlite3session.c \
   348    348     $(TOP)/ext/session/sqlite3session.h
   349    349   SRC += \
   350    350     $(TOP)/ext/rbu/sqlite3rbu.h \
   351    351     $(TOP)/ext/rbu/sqlite3rbu.c
   352    352   SRC += \
   353         -  $(TOP)/ext/misc/json1.c
          353  +  $(TOP)/ext/misc/json1.c \
          354  +  $(TOP)/ext/misc/stmt.c
   354    355   
   355    356   # Generated source code files
   356    357   #
   357    358   SRC += \
   358    359     keywordhash.h \
   359    360     opcodes.c \
   360    361     opcodes.h \
................................................................................
   423    424     $(TOP)/ext/misc/eval.c \
   424    425     $(TOP)/ext/misc/fileio.c \
   425    426     $(TOP)/ext/misc/fuzzer.c \
   426    427     $(TOP)/ext/fts5/fts5_tcl.c \
   427    428     $(TOP)/ext/fts5/fts5_test_mi.c \
   428    429     $(TOP)/ext/fts5/fts5_test_tok.c \
   429    430     $(TOP)/ext/misc/ieee754.c \
          431  +  $(TOP)/ext/misc/mmapwarm.c \
   430    432     $(TOP)/ext/misc/nextchar.c \
   431    433     $(TOP)/ext/misc/percentile.c \
   432    434     $(TOP)/ext/misc/regexp.c \
          435  +  $(TOP)/ext/misc/remember.c \
   433    436     $(TOP)/ext/misc/series.c \
   434    437     $(TOP)/ext/misc/spellfix.c \
   435    438     $(TOP)/ext/misc/totype.c \
          439  +  $(TOP)/ext/misc/unionvtab.c \
   436    440     $(TOP)/ext/misc/wholenumber.c
   437    441   
   438    442   # Source code to the library files needed by the test fixture
   439    443   #
   440    444   TESTSRC2 = \
   441    445     $(TOP)/src/attach.c \
   442    446     $(TOP)/src/backup.c \
................................................................................
   478    482     $(TOP)/ext/fts3/fts3.c \
   479    483     $(TOP)/ext/fts3/fts3_aux.c \
   480    484     $(TOP)/ext/fts3/fts3_expr.c \
   481    485     $(TOP)/ext/fts3/fts3_term.c \
   482    486     $(TOP)/ext/fts3/fts3_tokenizer.c \
   483    487     $(TOP)/ext/fts3/fts3_write.c \
   484    488     $(TOP)/ext/async/sqlite3async.c \
   485         -  $(TOP)/ext/session/sqlite3session.c 
          489  +  $(TOP)/ext/session/sqlite3session.c \
          490  +  $(TOP)/ext/misc/stmt.c 
   486    491   
   487    492   # Header files used by all library source files.
   488    493   #
   489    494   HDR = \
   490    495      $(TOP)/src/btree.h \
   491    496      $(TOP)/src/btreeInt.h \
   492    497      $(TOP)/src/hash.h \
................................................................................
   546    551   
   547    552   # Databases containing fuzzer test cases
   548    553   #
   549    554   FUZZDATA = \
   550    555     $(TOP)/test/fuzzdata1.db \
   551    556     $(TOP)/test/fuzzdata2.db \
   552    557     $(TOP)/test/fuzzdata3.db \
   553         -  $(TOP)/test/fuzzdata4.db
          558  +  $(TOP)/test/fuzzdata4.db \
          559  +  $(TOP)/test/fuzzdata5.db
   554    560   
   555    561   # Standard options to testfixture
   556    562   #
   557    563   TESTOPTS = --verbose=file --output=test-out.txt
   558    564   
   559    565   # Extra compiler options for various shell tools
   560    566   #
   561    567   SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
   562    568   # SHELL_OPT += -DSQLITE_ENABLE_FTS5
   563    569   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   564    570   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
          571  +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   565    572   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   566         -FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
          573  +FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
          574  +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
          575  +FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
          576  +DBFUZZ_OPT = 
   567    577   
   568    578   # This is the default Makefile target.  The objects listed here
   569    579   # are what get build when you type just "make" with no arguments.
   570    580   #
   571    581   all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
   572    582   
   573    583   Makefile: $(TOP)/Makefile.in
................................................................................
   608    618   sourcetest:	srcck1$(BEXE) sqlite3.c
   609    619   	./srcck1 sqlite3.c
   610    620   
   611    621   fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
   612    622   	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
   613    623   	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
   614    624   
   615         -fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
   616         -	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
          625  +fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
          626  +	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
          627  +
          628  +ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
          629  +	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
          630  +             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
          631  +
          632  +dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
          633  +	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
   617    634   
   618    635   mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
   619    636   	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
   620    637   		$(TLIBS) -rpath "$(libdir)"
   621    638   
   622    639   MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
   623    640   MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20
................................................................................
   672    689   	$(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c
   673    690   
   674    691   # Rules to build the LEMON compiler generator
   675    692   #
   676    693   lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
   677    694   	$(BCC) -o $@ $(TOP)/tool/lemon.c
   678    695   	cp $(TOP)/tool/lempar.c .
          696  +
          697  +# Rules to build the program that generates the source-id
          698  +#
          699  +mksourceid$(BEXE):	$(TOP)/tool/mksourceid.c
          700  +	$(BCC) -o $@ $(TOP)/tool/mksourceid.c
   679    701   
   680    702   # Rules to build individual *.o files from generated *.c files. This
   681    703   # applies to:
   682    704   #
   683    705   #     parse.o
   684    706   #     opcodes.o
   685    707   #
................................................................................
   938    960   parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
   939    961   	cp $(TOP)/src/parse.y .
   940    962   	rm -f parse.h
   941    963   	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
   942    964   	mv parse.h parse.h.temp
   943    965   	$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h
   944    966   
   945         -sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
          967  +sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
   946    968   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
   947    969   
   948    970   keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
   949    971   	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
   950    972   	./mkkeywordhash$(BEXE) >keywordhash.h
   951    973   
   952    974   
................................................................................
  1018   1040   
  1019   1041   sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
  1020   1042   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
  1021   1043   
  1022   1044   json1.lo:	$(TOP)/ext/misc/json1.c
  1023   1045   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
  1024   1046   
         1047  +stmt.lo:	$(TOP)/ext/misc/stmt.c
         1048  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
         1049  +
  1025   1050   # FTS5 things
  1026   1051   #
  1027   1052   FTS5_SRC = \
  1028   1053      $(TOP)/ext/fts5/fts5.h \
  1029   1054      $(TOP)/ext/fts5/fts5Int.h \
  1030   1055      $(TOP)/ext/fts5/fts5_aux.c \
  1031   1056      $(TOP)/ext/fts5/fts5_buffer.c \
................................................................................
  1067   1092   # hidden when the library is built via the amalgamation).
  1068   1093   #
  1069   1094   TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1070   1095   TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
  1071   1096   TESTFIXTURE_FLAGS += -DBUILD_sqlite
  1072   1097   TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  1073   1098   TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
         1099  +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
  1074   1100   
  1075   1101   TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
  1076   1102   TESTFIXTURE_SRC1 = sqlite3.c
  1077   1103   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
  1078   1104   TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
  1079   1105   
  1080   1106   testfixture$(TEXE):	$(TESTFIXTURE_SRC)
................................................................................
  1099   1125   
  1100   1126   fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
  1101   1127   	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
  1102   1128   
  1103   1129   valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
  1104   1130   	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
  1105   1131   
         1132  +# The veryquick.test TCL tests.
         1133  +#
         1134  +tcltest:	./testfixture$(TEXE)
         1135  +	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
         1136  +
  1106   1137   # Minimal testing that runs in less than 3 minutes
  1107   1138   #
  1108   1139   quicktest:	./testfixture$(TEXE)
  1109   1140   	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
  1110   1141   
  1111   1142   # This is the common case.  Run many tests that do not take too long,
  1112   1143   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1113   1144   #
  1114         -test:	$(TESTPROGS) sourcetest fastfuzztest
  1115         -	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
         1145  +test:	fastfuzztest sourcetest $(TESTPROGS) tcltest
  1116   1146   
  1117   1147   # Run a test using valgrind.  This can take a really long time
  1118   1148   # because valgrind is so much slower than a native machine.
  1119   1149   #
  1120   1150   valgrindtest:	$(TESTPROGS) valgrindfuzz
  1121   1151   	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS)
  1122   1152   
................................................................................
  1135   1165   	echo "static const char *zMainloop = " >> $@
  1136   1166   	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
  1137   1167   	echo "; return zMainloop; }" >> $@
  1138   1168   
  1139   1169   sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
  1140   1170   	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
  1141   1171   
         1172  +dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
         1173  +	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
         1174  +           $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
         1175  +
  1142   1176   showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
  1143   1177   	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
  1144   1178   
  1145   1179   showstat4$(TEXE):	$(TOP)/tool/showstat4.c sqlite3.lo
  1146   1180   	$(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS)
  1147   1181   
  1148   1182   showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
................................................................................
  1159   1193   
  1160   1194   LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
  1161   1195   	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
  1162   1196   
  1163   1197   wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
  1164   1198   	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
  1165   1199   
  1166         -speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
  1167         -	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
         1200  +speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
         1201  +	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
         1202  +
         1203  +KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
         1204  +
         1205  +kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
         1206  +	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
  1168   1207   
  1169   1208   rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
  1170   1209   	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
  1171   1210   
  1172   1211   loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
  1173   1212   	$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
  1174   1213   

Changes to Makefile.msc.

    17     17   USE_AMALGAMATION = 1
    18     18   !ENDIF
    19     19   # <</mark>>
    20     20   
    21     21   # Set this non-0 to enable full warnings (-W4, etc) when compiling.
    22     22   #
    23     23   !IFNDEF USE_FULLWARN
    24         -USE_FULLWARN = 0
           24  +USE_FULLWARN = 1
           25  +!ENDIF
           26  +
           27  +# Set this non-0 to enable treating warnings as errors (-WX, etc) when
           28  +# compiling.
           29  +#
           30  +!IFNDEF USE_FATAL_WARN
           31  +USE_FATAL_WARN = 0
    25     32   !ENDIF
    26     33   
    27     34   # Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
    28     35   # has no effect if (any) optimizations are enabled.
    29     36   #
    30     37   !IFNDEF USE_RUNTIME_CHECKS
    31     38   USE_RUNTIME_CHECKS = 0
................................................................................
   488    495   # same unless your are cross-compiling.)
   489    496   #
   490    497   !IF $(USE_FULLWARN)!=0
   491    498   TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
   492    499   !ELSE
   493    500   TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
   494    501   !ENDIF
          502  +
          503  +# Check if warnings should be treated as errors when compiling.
          504  +#
          505  +!IF $(USE_FATAL_WARN)!=0
          506  +TCC = $(TCC) -WX
          507  +!ENDIF
   495    508   
   496    509   TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise
   497    510   RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
   498    511   
   499    512   # Check if we want to use the "stdcall" calling convention when compiling.
   500    513   # This is not supported by the compilers for non-x86 platforms.  It should
   501    514   # also be noted here that building any target with these "stdcall" options
................................................................................
   729    742   TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
   730    743   RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
   731    744   !ENDIF
   732    745   
   733    746   !IF $(DEBUG)>2
   734    747   TCC = $(TCC) -DSQLITE_DEBUG=1
   735    748   RCC = $(RCC) -DSQLITE_DEBUG=1
          749  +!IF $(DYNAMIC_SHELL)==0
          750  +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          751  +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          752  +!ENDIF
   736    753   !ENDIF
   737    754   
   738    755   !IF $(DEBUG)>4 || $(OSTRACE)!=0
   739    756   TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   740    757   RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   741    758   !ENDIF
   742    759   
................................................................................
   790    807   !ENDIF
   791    808   
   792    809   !IFNDEF TCLLIBDIR
   793    810   TCLLIBDIR = c:\tcl\lib
   794    811   !ENDIF
   795    812   
   796    813   !IFNDEF LIBTCL
   797         -LIBTCL = tcl85.lib
          814  +LIBTCL = tcl86.lib
   798    815   !ENDIF
   799    816   
   800    817   !IFNDEF LIBTCLSTUB
   801         -LIBTCLSTUB = tclstub85.lib
          818  +LIBTCLSTUB = tclstub86.lib
   802    819   !ENDIF
   803    820   
   804    821   !IFNDEF LIBTCLPATH
   805    822   LIBTCLPATH = c:\tcl\bin
   806    823   !ENDIF
   807    824   
   808    825   # The locations of the ICU header and library files.  These variables
................................................................................
   824    841   
   825    842   # This is the command to use for tclsh - normally just "tclsh", but we may
   826    843   # know the specific version we want to use.  This variable (TCLSH_CMD) may be
   827    844   # overridden via the environment prior to running nmake in order to select a
   828    845   # specific Tcl shell to use.
   829    846   #
   830    847   !IFNDEF TCLSH_CMD
   831         -TCLSH_CMD = tclsh85
          848  +TCLSH_CMD = tclsh
   832    849   !ENDIF
   833    850   # <</mark>>
   834    851   
   835    852   # Compiler options needed for programs that use the readline() library.
   836    853   #
   837    854   !IFNDEF READLINE_FLAGS
   838    855   READLINE_FLAGS = -DHAVE_READLINE=0
................................................................................
  1273   1290     $(TOP)\ext\fts3\fts3_unicode.c \
  1274   1291     $(TOP)\ext\fts3\fts3_unicode2.c \
  1275   1292     $(TOP)\ext\fts3\fts3_write.c \
  1276   1293     $(TOP)\ext\icu\icu.c \
  1277   1294     $(TOP)\ext\rtree\rtree.c \
  1278   1295     $(TOP)\ext\session\sqlite3session.c \
  1279   1296     $(TOP)\ext\rbu\sqlite3rbu.c \
  1280         -  $(TOP)\ext\misc\json1.c
         1297  +  $(TOP)\ext\misc\json1.c \
         1298  +  $(TOP)\ext\misc\stmt.c
  1281   1299   
  1282   1300   # Extension header files, part 1.
  1283   1301   #
  1284   1302   SRC08 = \
  1285   1303     $(TOP)\ext\fts1\fts1.h \
  1286   1304     $(TOP)\ext\fts1\fts1_hash.h \
  1287   1305     $(TOP)\ext\fts1\fts1_tokenizer.h \
................................................................................
  1389   1407     $(TOP)\ext\misc\eval.c \
  1390   1408     $(TOP)\ext\misc\fileio.c \
  1391   1409     $(TOP)\ext\misc\fuzzer.c \
  1392   1410     $(TOP)\ext\fts5\fts5_tcl.c \
  1393   1411     $(TOP)\ext\fts5\fts5_test_mi.c \
  1394   1412     $(TOP)\ext\fts5\fts5_test_tok.c \
  1395   1413     $(TOP)\ext\misc\ieee754.c \
         1414  +  $(TOP)\ext\misc\mmapwarm.c \
  1396   1415     $(TOP)\ext\misc\nextchar.c \
  1397   1416     $(TOP)\ext\misc\percentile.c \
  1398   1417     $(TOP)\ext\misc\regexp.c \
         1418  +  $(TOP)\ext\misc\remember.c \
  1399   1419     $(TOP)\ext\misc\series.c \
  1400   1420     $(TOP)\ext\misc\spellfix.c \
  1401   1421     $(TOP)\ext\misc\totype.c \
         1422  +  $(TOP)\ext\misc\unionvtab.c \
  1402   1423     $(TOP)\ext\misc\wholenumber.c
  1403   1424   
  1404   1425   # Source code to the library files needed by the test fixture
  1405   1426   # (non-amalgamation)
  1406   1427   #
  1407   1428   TESTSRC2 = \
  1408   1429     $(SRC00) \
................................................................................
  1475   1496   
  1476   1497   # Databases containing fuzzer test cases
  1477   1498   #
  1478   1499   FUZZDATA = \
  1479   1500     $(TOP)\test\fuzzdata1.db \
  1480   1501     $(TOP)\test\fuzzdata2.db \
  1481   1502     $(TOP)\test\fuzzdata3.db \
  1482         -  $(TOP)\test\fuzzdata4.db
         1503  +  $(TOP)\test\fuzzdata4.db \
         1504  +  $(TOP)\test\fuzzdata5.db
  1483   1505   # <</mark>>
  1484   1506   
  1485   1507   # Additional compiler options for the shell.  These are only effective
  1486   1508   # when the shell is not being dynamically linked.
  1487   1509   #
  1488   1510   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1489         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
         1511  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
  1490   1512   !ENDIF
  1491   1513   
  1492   1514   # <<mark>>
  1493   1515   # Extra compiler options for various test tools.
  1494   1516   #
  1495   1517   MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
  1496   1518   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
  1497         -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
         1519  +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
         1520  +FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
         1521  +OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
         1522  +DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
         1523  +KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
         1524  +DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
         1525  +ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0
  1498   1526   
  1499   1527   # Standard options to testfixture.
  1500   1528   #
  1501   1529   TESTOPTS = --verbose=file --output=test-out.txt
  1502   1530   
  1503   1531   # Extra targets for the "all" target that require Tcl.
  1504   1532   #
................................................................................
  1533   1561   $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
  1534   1562   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1535   1563   
  1536   1564   # <<block2>>
  1537   1565   sqlite3.def:	libsqlite3.lib
  1538   1566   	echo EXPORTS > sqlite3.def
  1539   1567   	dumpbin /all libsqlite3.lib \
  1540         -		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \
         1568  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
  1541   1569   		| sort >> sqlite3.def
  1542   1570   # <</block2>>
  1543   1571   
  1544   1572   $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
  1545   1573   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
  1546   1574   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1547   1575   
................................................................................
  1560   1588   
  1561   1589   sourcetest:	srcck1.exe sqlite3.c
  1562   1590   	srcck1.exe sqlite3.c
  1563   1591   
  1564   1592   fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
  1565   1593   	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1566   1594   
  1567         -fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
  1568         -	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1595  +dbfuzz.exe:	$(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
         1596  +	$(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1597  +
         1598  +fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
         1599  +	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1600  +
         1601  +ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
         1602  +	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1569   1603   
  1570   1604   mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
  1571   1605   	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1572   1606   
  1573   1607   MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20
  1574   1608   MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20
  1575   1609   
................................................................................
  1633   1667   lempar.c:	$(TOP)\tool\lempar.c
  1634   1668   	copy $(TOP)\tool\lempar.c .
  1635   1669   
  1636   1670   lemon.exe:	$(TOP)\tool\lemon.c lempar.c
  1637   1671   	$(BCC) $(NO_WARN) -Daccess=_access \
  1638   1672   		-Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
  1639   1673   
         1674  +# <<mark>>
         1675  +# Rules to build the source-id generator tool
         1676  +#
         1677  +mksourceid.exe:	$(TOP)\tool\mksourceid.c
         1678  +	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\mksourceid.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
         1679  +
  1640   1680   # Rules to build individual *.lo files from generated *.c files. This
  1641   1681   # applies to:
  1642   1682   #
  1643   1683   #     parse.lo
  1644   1684   #     opcodes.lo
  1645   1685   #
  1646   1686   parse.lo:	parse.c $(HDR)
................................................................................
  1911   1951   parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl
  1912   1952   	del /Q parse.y parse.h parse.h.temp 2>NUL
  1913   1953   	copy $(TOP)\src\parse.y .
  1914   1954   	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y
  1915   1955   	move parse.h parse.h.temp
  1916   1956   	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h
  1917   1957   
  1918         -$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
         1958  +$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION
  1919   1959   	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS)
  1920   1960   
  1921   1961   sqlite3ext.h:	.target_source
  1922   1962   !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
  1923   1963   	type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \
  1924   1964   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_APICALL *" > sqlite3ext.h
  1925   1965   	copy /Y sqlite3ext.h tsrc\sqlite3ext.h
................................................................................
  2054   2094   # hidden when the library is built via the amalgamation).
  2055   2095   #
  2056   2096   TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  2057   2097   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  2058   2098   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
  2059   2099   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  2060   2100   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
         2101  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
  2061   2102   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
  2062   2103   
  2063   2104   TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  2064   2105   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  2065   2106   !IF $(USE_AMALGAMATION)==0
  2066   2107   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  2067   2108   !ELSE
................................................................................
  2146   2187   	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
  2147   2188   	echo ; return zMainloop; } >> $@
  2148   2189   
  2149   2190   sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
  2150   2191   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
  2151   2192   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  2152   2193   
         2194  +dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
         2195  +	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
         2196  +		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
         2197  +
  2153   2198   testloadext.lo:	$(TOP)\src\test_loadext.c
  2154   2199   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
  2155   2200   
  2156   2201   testloadext.dll:	testloadext.lo
  2157   2202   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  2158   2203   
  2159   2204   showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
  2160         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2205  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2161   2206   		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2162   2207   
  2163   2208   showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
  2164         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2209  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2165   2210   		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2166   2211   
  2167   2212   showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H)
  2168         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2213  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2169   2214   		$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2170   2215   
  2171   2216   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  2172         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2217  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2173   2218   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2174   2219   
  2175         -changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C)
  2176         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2220  +changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
         2221  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2222  +		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
  2177   2223   		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2178   2224   
  2179   2225   fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
  2180         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2226  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2181   2227   		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2182   2228   
  2183   2229   rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
  2184         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2230  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2185   2231   		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2186   2232   
  2187   2233   LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
  2188         -	$(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
         2234  +	$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
  2189   2235   
  2190   2236   wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H)
  2191         -	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2237  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2192   2238   		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2193   2239   
  2194   2240   speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H)
  2195         -	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2241  +	$(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \
  2196   2242   		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2197   2243   
         2244  +kvtest.exe:	$(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H)
         2245  +	$(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \
         2246  +		$(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2247  +
         2248  +dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
         2249  +	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
         2250  +
  2198   2251   rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
  2199         -	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \
         2252  +	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
  2200   2253   		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2201   2254   
         2255  +LSMDIR=$(TOP)\ext\lsm1
         2256  +!INCLUDE $(LSMDIR)\Makefile.msc
         2257  +
  2202   2258   moreclean:	clean
  2203   2259   	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
  2204   2260   # <</mark>>
  2205   2261   
  2206   2262   clean:
  2207   2263   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  2208   2264   	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
  2209   2265   	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
  2210   2266   # <<mark>>
  2211   2267   	del /Q sqlite3.c sqlite3.h 2>NUL
  2212   2268   	del /Q opcodes.c opcodes.h 2>NUL
  2213   2269   	del /Q lemon.* lempar.c parse.* 2>NUL
  2214         -	del /Q mkkeywordhash.* keywordhash.h 2>NUL
         2270  +	del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL
  2215   2271   	del /Q notasharedlib.* 2>NUL
  2216   2272   	-rmdir /Q/S .deps 2>NUL
  2217   2273   	-rmdir /Q/S .libs 2>NUL
  2218   2274   	-rmdir /Q/S tsrc 2>NUL
  2219   2275   	del /Q .target_source 2>NUL
  2220   2276   	del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL
         2277  +	del /Q lsm.dll lsmtest.exe 2>NUL
  2221   2278   	del /Q testloadext.dll 2>NUL
  2222   2279   	del /Q testfixture.exe test.db 2>NUL
  2223         -	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
         2280  +	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL
  2224   2281   	del /Q changeset.exe 2>NUL
  2225   2282   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  2226   2283   	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2227   2284   	del /Q sqlite3.c sqlite3-*.c 2>NUL
  2228   2285   	del /Q sqlite3rc.h 2>NUL
  2229   2286   	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
  2230   2287   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  2231   2288   	del /Q sqlite-*-output.vsix 2>NUL
  2232   2289   	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
  2233   2290   	del /Q fts5.* fts5parse.* 2>NUL
  2234   2291   # <</mark>>

Changes to README.md.

     4      4   engine.  Some test scripts are also include.  However, many other test scripts
     5      5   and most of the documentation are managed separately.
     6      6   
     7      7   If you are reading this on a Git mirror someplace, you are doing it wrong.
     8      8   The [official repository](https://www.sqlite.org/src/) is better.  Go there
     9      9   now.
    10     10   
           11  +## Obtaining The Code
           12  +
           13  +SQLite sources are managed using the
           14  +[Fossil](https://www.fossil-scm.org/), a distributed version control system
           15  +that was specifically designed to support SQLite development.
           16  +If you do not want to use Fossil, you can download tarballs or ZIP
           17  +archives as follows:
           18  +
           19  +  *  Lastest trunk check-in:
           20  +     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
           21  +     <https://www.sqlite.org/src/zip/sqlite.zip>.
           22  +
           23  +  *  Latest release:
           24  +     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
           25  +     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
           26  +
           27  +  *  For other check-ins, substitute an appropriate branch name or
           28  +     tag or hash prefix for "release" in the URLs of the previous
           29  +     bullet.  Or browse the [timeline](https://www.sqlite.org/src/timeline)
           30  +     to locate the check-in desired, click on its information page link,
           31  +     then click on the "Tarball" or "ZIP Archive" links on the information
           32  +     page.
           33  +
           34  +If you do want to use Fossil to check out the source tree, 
           35  +first install Fossil version 2.0 or later.
           36  +(Source tarballs and precompiled binaries available
           37  +[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
           38  +a stand-alone program.  To install, simply download or build the single 
           39  +executable file and put that file someplace on your $PATH.)
           40  +Then run commands like this:
           41  +
           42  +        mkdir ~/sqlite
           43  +        cd ~/sqlite
           44  +        fossil clone https://www.sqlite.org/src sqlite.fossil
           45  +        fossil open sqlite.fossil
           46  +    
           47  +After setting up a repository using the steps above, you can always
           48  +update to the lastest version using:
           49  +
           50  +        fossil update trunk   ;# latest trunk check-in
           51  +        fossil update release ;# latest official release
           52  +
           53  +Or type "fossil ui" to get a web-based user interface.
           54  +
    11     55   ## Compiling
    12     56   
    13     57   First create a directory in which to place
    14     58   the build products.  It is recommended, but not required, that the
    15     59   build directory be separate from the source directory.  Cd into the
    16     60   build directory and then from the build directory run the configure
    17     61   script found at the root of the source tree.  Then run "make".
    18     62   
    19     63   For example:
    20     64   
    21         -    tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
    22         -    mkdir bld                ;#  Build will occur in a sibling directory
    23         -    cd bld                   ;#  Change to the build directory
    24         -    ../sqlite/configure      ;#  Run the configure script
    25         -    make                     ;#  Run the makefile.
    26         -    make sqlite3.c           ;#  Build the "amalgamation" source file
    27         -    make test                ;#  Run some tests (requires Tcl)
           65  +        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
           66  +        mkdir bld                ;#  Build will occur in a sibling directory
           67  +        cd bld                   ;#  Change to the build directory
           68  +        ../sqlite/configure      ;#  Run the configure script
           69  +        make                     ;#  Run the makefile.
           70  +        make sqlite3.c           ;#  Build the "amalgamation" source file
           71  +        make test                ;#  Run some tests (requires Tcl)
    28     72   
    29     73   See the makefile for additional targets.
    30     74   
    31     75   The configure script uses autoconf 2.61 and libtool.  If the configure
    32     76   script does not work out for you, there is a generic makefile named
    33     77   "Makefile.linux-gcc" in the top directory of the source tree that you
    34     78   can copy and edit to suit your needs.  Comments on the generic makefile
................................................................................
    39     83   On Windows, all applicable build products can be compiled with MSVC.
    40     84   First open the command prompt window associated with the desired compiler
    41     85   version (e.g. "Developer Command Prompt for VS2013").  Next, use NMAKE
    42     86   with the provided "Makefile.msc" to build one of the supported targets.
    43     87   
    44     88   For example:
    45     89   
    46         -    mkdir bld
    47         -    cd bld
    48         -    nmake /f Makefile.msc TOP=..\sqlite
    49         -    nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
    50         -    nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
    51         -    nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
    52         -    nmake /f Makefile.msc test TOP=..\sqlite
           90  +        mkdir bld
           91  +        cd bld
           92  +        nmake /f Makefile.msc TOP=..\sqlite
           93  +        nmake /f Makefile.msc sqlite3.c TOP=..\sqlite
           94  +        nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite
           95  +        nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite
           96  +        nmake /f Makefile.msc test TOP=..\sqlite
    53     97   
    54     98   There are several build options that can be set via the NMAKE command
    55     99   line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
    56    100   to the "sqlite3.dll" command line above.  When debugging into the SQLite
    57    101   code, adding the "DEBUG=1" argument to one of the above command lines is
    58    102   recommended.
    59    103   
................................................................................
    60    104   SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
    61    105   is required by the makefiles (including those for MSVC).  SQLite contains
    62    106   a lot of generated code and Tcl is used to do much of that code generation.
    63    107   The makefiles also require AWK.
    64    108   
    65    109   ## Source Code Tour
    66    110   
    67         -Most of the core source files are in the **src/** subdirectory.  But
    68         -src/ also contains files used to build the "testfixture" test harness;
    69         -those file all begin with "test".  And src/ contains the "shell.c" file
    70         -which is the main program for the "sqlite3.exe" command-line shell and
    71         -the "tclsqlite.c" file which implements the bindings to SQLite from the
    72         -Tcl programming language.  (Historical note:  SQLite began as a Tcl
          111  +Most of the core source files are in the **src/** subdirectory.  The
          112  +**src/** folder also contains files used to build the "testfixture" test
          113  +harness. The names of the source files used by "testfixture" all begin
          114  +with "test".
          115  +The **src/** also contains the "shell.c" file
          116  +which is the main program for the "sqlite3.exe"
          117  +[command-line shell](https://sqlite.org/cli.html) and
          118  +the "tclsqlite.c" file which implements the
          119  +[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
          120  +(Historical note:  SQLite began as a Tcl
    73    121   extension and only later escaped to the wild as an independent library.)
    74    122   
    75    123   Test scripts and programs are found in the **test/** subdirectory.
    76         -There are other test suites for SQLite (see
    77         -[How SQLite Is Tested](http://www.sqlite.org/testing.html))
    78         -but those other test suites are
    79         -in separate source repositories.
          124  +Addtional test code is found in other source repositories.
          125  +See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
          126  +additional information.
    80    127   
    81    128   The **ext/** subdirectory contains code for extensions.  The
    82    129   Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
    83    130   **ext/rtree**.  The **ext/misc** subdirectory contains a number of
    84    131   smaller, single-file extensions, such as a REGEXP operator.
    85    132   
    86    133   The **tool/** subdirectory contains various scripts and programs used
................................................................................
    96    143   The "target&#95;source" make target will create a subdirectory "tsrc/" and
    97    144   fill it with all the source files needed to build SQLite, both
    98    145   manually-edited files and automatically-generated files.
    99    146   
   100    147   The SQLite interface is defined by the **sqlite3.h** header file, which is
   101    148   generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
   102    149   [Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
   103         -The manifest.uuid file contains the SHA1 hash of the particular check-in
          150  +The manifest.uuid file contains the SHA3 hash of the particular check-in
   104    151   and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
   105    152   contains the current SQLite version number.  The sqlite3.h header is really
   106    153   just a copy of src/sqlite.h.in with the source-id and version number inserted
   107    154   at just the right spots. Note that comment text in the sqlite3.h file is
   108    155   used to generate much of the SQLite API documentation.  The Tcl scripts
   109    156   used to generate that documentation are in a separate source repository.
   110    157   
   111    158   The SQL language parser is **parse.c** which is generate from a grammar in
   112    159   the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
   113    160   by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
   114         -for lemon is at tool/lemon.c.  Lemon uses a
   115         -template for generating its parser.  A generic template is in tool/lempar.c,
   116         -but SQLite uses a slightly modified template found in src/lempar.c.
          161  +for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
          162  +template for generating its parser.
   117    163   
   118    164   Lemon also generates the **parse.h** header file, at the same time it
   119    165   generates parse.c. But the parse.h header file is
   120    166   modified further (to add additional symbols) using the ./addopcodes.awk
   121    167   AWK script.
   122    168   
   123    169   The **opcodes.h** header file contains macros that define the numbers
................................................................................
   129    175   opcode-number to opcode-name that is used for EXPLAIN output.
   130    176   
   131    177   The **keywordhash.h** header file contains the definition of a hash table
   132    178   that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
   133    179   the numeric codes used by the parse.c parser.  The keywordhash.h file is
   134    180   generated by a C-language program at tool mkkeywordhash.c.
   135    181   
          182  +The **pragma.h** header file contains various definitions used to parse
          183  +and implement the PRAGMA statements.  The header is generated by a
          184  +script **tool/mkpragmatab.tcl**. If you want to add a new PRAGMA, edit
          185  +the **tool/mkpragmatab.tcl** file to insert the information needed by the
          186  +parser for your new PRAGMA, then run the script to regenerate the
          187  +**pragma.h** header file.
          188  +
   136    189   ### The Amalgamation
   137    190   
   138    191   All of the individual C source code and header files (both manually-edited
   139    192   and automatically-generated) can be combined into a single big source file
   140    193   **sqlite3.c** called "the amalgamation".  The amalgamation is the recommended
   141    194   way of using SQLite in a larger application.  Combining all individual
   142    195   source code files into a single big source code file allows the C compiler
................................................................................
   146    199   
   147    200   The amalgamation is generated from the tool/mksqlite3c.tcl Tcl script.
   148    201   First, all of the individual source files must be gathered into the tsrc/
   149    202   subdirectory (using the equivalent of "make target_source") then the
   150    203   tool/mksqlite3c.tcl script is run to copy them all together in just the
   151    204   right order while resolving internal "#include" references.
   152    205   
   153         -The amalgamation source file is more than 100K lines long.  Some symbolic
          206  +The amalgamation source file is more than 200K lines long.  Some symbolic
   154    207   debuggers (most notably MSVC) are unable to deal with files longer than 64K
   155    208   lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
   156    209   can be run on the amalgamation to break it up into a single small C file
   157    210   called **sqlite3-all.c** that does #include on about five other files
   158    211   named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
   159    212   all of the source code is contained within a single translation unit so
   160    213   that the compiler can do extra cross-procedure optimization, but no
................................................................................
   163    216   ## How It All Fits Together
   164    217   
   165    218   SQLite is modular in design.
   166    219   See the [architectural description](http://www.sqlite.org/arch.html)
   167    220   for details. Other documents that are useful in
   168    221   (helping to understand how SQLite works include the
   169    222   [file format](http://www.sqlite.org/fileformat2.html) description,
   170         -the [virtual machine](http://www.sqlite.org/vdbe.html) that runs
          223  +the [virtual machine](http://www.sqlite.org/opcode.html) that runs
   171    224   prepared statements, the description of
   172    225   [how transactions work](http://www.sqlite.org/atomiccommit.html), and
   173    226   the [overview of the query planner](http://www.sqlite.org/optoverview.html).
   174    227   
   175         -Unfortunately, years of effort have gone into optimizating SQLite, both
          228  +Years of effort have gone into optimizating SQLite, both
   176    229   for small size and high performance.  And optimizations tend to result in
   177         -complex code.  So there is a lot of complexity in the SQLite implementation.
          230  +complex code.  So there is a lot of complexity in the current SQLite
          231  +implementation.  It will not be the easiest library in the world to hack.
   178    232   
   179    233   Key files:
   180    234   
   181    235     *  **sqlite.h.in** - This file defines the public interface to the SQLite
   182    236        library.  Readers will need to be familiar with this interface before
   183    237        trying to understand how the library works internally.
   184    238   
   185    239     *  **sqliteInt.h** - this header file defines many of the data objects
   186    240        used internally by SQLite.
   187    241   
   188         -  *  **parse.y** - This file describes the LALR(1) grammer that SQLite uses
          242  +  *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
   189    243        to parse SQL statements, and the actions that are taken at each step
   190    244        in the parsing process.
   191    245   
   192    246     *  **vdbe.c** - This file implements the virtual machine that runs
   193    247        prepared statements.  There are various helper files whose names
   194    248        begin with "vdbe".  The VDBE has access to the vdbeInt.h header file
   195    249        which defines internal data objects.  The rest of SQLite interacts
................................................................................
   214    268        is the file that, when linked against sqlite3.a, generates the
   215    269        "sqlite3.exe" command-line shell.
   216    270   
   217    271     *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
   218    272        is not part of the core SQLite library.  But as most of the tests in this
   219    273        repository are written in Tcl, the Tcl language bindings are important.
   220    274   
   221         -There are many other source files.  Each has a suscinct header comment that
          275  +There are many other source files.  Each has a succinct header comment that
   222    276   describes its purpose and role within the larger system.
   223    277   
   224    278   
   225    279   ## Contacts
   226    280   
   227    281   The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
   228         -with geographically distributed backup servers at
          282  +with geographically distributed backups at
   229    283   [http://www2.sqlite.org/](http://www2.sqlite.org) and
   230    284   [http://www3.sqlite.org/](http://www3.sqlite.org).

Changes to VERSION.

     1         -3.16.0
            1  +3.21.0

Changes to autoconf/Makefile.msc.

    17     17   #
    18     18   TOP = .
    19     19   
    20     20   
    21     21   # Set this non-0 to enable full warnings (-W4, etc) when compiling.
    22     22   #
    23     23   !IFNDEF USE_FULLWARN
    24         -USE_FULLWARN = 0
           24  +USE_FULLWARN = 1
           25  +!ENDIF
           26  +
           27  +# Set this non-0 to enable treating warnings as errors (-WX, etc) when
           28  +# compiling.
           29  +#
           30  +!IFNDEF USE_FATAL_WARN
           31  +USE_FATAL_WARN = 0
    25     32   !ENDIF
    26     33   
    27     34   # Set this non-0 to enable full runtime error checks (-RTC1, etc).  This
    28     35   # has no effect if (any) optimizations are enabled.
    29     36   #
    30     37   !IFNDEF USE_RUNTIME_CHECKS
    31     38   USE_RUNTIME_CHECKS = 0
................................................................................
   449    456   # same unless your are cross-compiling.)
   450    457   #
   451    458   !IF $(USE_FULLWARN)!=0
   452    459   TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
   453    460   !ELSE
   454    461   TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
   455    462   !ENDIF
          463  +
          464  +# Check if warnings should be treated as errors when compiling.
          465  +#
          466  +!IF $(USE_FATAL_WARN)!=0
          467  +TCC = $(TCC) -WX
          468  +!ENDIF
   456    469   
   457    470   TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
   458    471   RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
   459    472   
   460    473   # Check if we want to use the "stdcall" calling convention when compiling.
   461    474   # This is not supported by the compilers for non-x86 platforms.  It should
   462    475   # also be noted here that building any target with these "stdcall" options
................................................................................
   628    641   TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
   629    642   RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
   630    643   !ENDIF
   631    644   
   632    645   !IF $(DEBUG)>2
   633    646   TCC = $(TCC) -DSQLITE_DEBUG=1
   634    647   RCC = $(RCC) -DSQLITE_DEBUG=1
          648  +!IF $(DYNAMIC_SHELL)==0
          649  +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          650  +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE
          651  +!ENDIF
   635    652   !ENDIF
   636    653   
   637    654   !IF $(DEBUG)>4 || $(OSTRACE)!=0
   638    655   TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   639    656   RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
   640    657   !ENDIF
   641    658   
................................................................................
   906    923   !ENDIF
   907    924   
   908    925   
   909    926   # Additional compiler options for the shell.  These are only effective
   910    927   # when the shell is not being dynamically linked.
   911    928   #
   912    929   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   913         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
          930  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
   914    931   !ENDIF
   915    932   
   916    933   
   917    934   # This is the default Makefile target.  The objects listed here
   918    935   # are what get build when you type just "make" with no arguments.
   919    936   #
   920    937   all:	dll shell
................................................................................
   933    950   
   934    951   Replace.exe:
   935    952   	$(CSC) /target:exe $(TOP)\Replace.cs
   936    953   
   937    954   sqlite3.def:	Replace.exe $(LIBOBJ)
   938    955   	echo EXPORTS > sqlite3.def
   939    956   	dumpbin /all $(LIBOBJ) \
   940         -		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
          957  +		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
   941    958   		| sort >> sqlite3.def
   942    959   
   943    960   $(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
   944    961   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
   945    962   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
   946    963   
   947    964   

Changes to autoconf/configure.ac.

    51     51   
    52     52   AS_IF([ test x"$enable_editline" != xno ],[
    53     53     AC_CHECK_HEADERS([editline/readline.h],[
    54     54       sLIBS=$LIBS
    55     55       LIBS=""
    56     56       AC_SEARCH_LIBS([readline],[edit],[
    57     57         AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
    58         -      READLINE_LIBS=$LIBS
           58  +      READLINE_LIBS="$LIBS -ltinfo"
    59     59         enable_readline=no
    60         -    ])
           60  +    ],[],[-ltinfo])
    61     61       AS_UNSET(ac_cv_search_readline)
    62     62       LIBS=$sLIBS
    63     63     ])
    64     64   ])
    65     65   
    66     66   AS_IF([ test x"$enable_readline" != xno ],[
    67     67     AC_CHECK_HEADERS([readline/readline.h],[

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.16.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.21.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.16.0'
   730         -PACKAGE_STRING='sqlite 3.16.0'
          729  +PACKAGE_VERSION='3.21.0'
          730  +PACKAGE_STRING='sqlite 3.21.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
   905    905   enable_load_extension
   906    906   enable_memsys5
   907    907   enable_memsys3
   908    908   enable_fts3
   909    909   enable_fts4
   910    910   enable_fts5
   911    911   enable_json1
          912  +enable_update_limit
   912    913   enable_rtree
   913    914   enable_session
   914    915   enable_gcov
   915    916   '
   916    917         ac_precious_vars='build_alias
   917    918   host_alias
   918    919   target_alias
................................................................................
  1459   1460   #
  1460   1461   # Report the --help message.
  1461   1462   #
  1462   1463   if test "$ac_init_help" = "long"; then
  1463   1464     # Omit some internal or obsolete options to make the list less imposing.
  1464   1465     # This message is too long to be a string in the A/UX 3.1 sh.
  1465   1466     cat <<_ACEOF
  1466         -\`configure' configures sqlite 3.16.0 to adapt to many kinds of systems.
         1467  +\`configure' configures sqlite 3.21.0 to adapt to many kinds of systems.
  1467   1468   
  1468   1469   Usage: $0 [OPTION]... [VAR=VALUE]...
  1469   1470   
  1470   1471   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1471   1472   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1472   1473   
  1473   1474   Defaults for the options are specified in brackets.
................................................................................
  1524   1525     --build=BUILD     configure for building on BUILD [guessed]
  1525   1526     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1526   1527   _ACEOF
  1527   1528   fi
  1528   1529   
  1529   1530   if test -n "$ac_init_help"; then
  1530   1531     case $ac_init_help in
  1531         -     short | recursive ) echo "Configuration of sqlite 3.16.0:";;
         1532  +     short | recursive ) echo "Configuration of sqlite 3.21.0:";;
  1532   1533      esac
  1533   1534     cat <<\_ACEOF
  1534   1535   
  1535   1536   Optional Features:
  1536   1537     --disable-option-checking  ignore unrecognized --enable/--with options
  1537   1538     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1538   1539     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1556   1557                             Disable loading of external extensions
  1557   1558     --enable-memsys5        Enable MEMSYS5
  1558   1559     --enable-memsys3        Enable MEMSYS3
  1559   1560     --enable-fts3           Enable the FTS3 extension
  1560   1561     --enable-fts4           Enable the FTS4 extension
  1561   1562     --enable-fts5           Enable the FTS5 extension
  1562   1563     --enable-json1          Enable the JSON1 extension
         1564  +  --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
  1563   1565     --enable-rtree          Enable the RTREE extension
  1564   1566     --enable-session        Enable the SESSION extension
  1565   1567     --enable-gcov           Enable coverage testing using gcov
  1566   1568   
  1567   1569   Optional Packages:
  1568   1570     --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
  1569   1571     --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
................................................................................
  1648   1650       cd "$ac_pwd" || { ac_status=$?; break; }
  1649   1651     done
  1650   1652   fi
  1651   1653   
  1652   1654   test -n "$ac_init_help" && exit $ac_status
  1653   1655   if $ac_init_version; then
  1654   1656     cat <<\_ACEOF
  1655         -sqlite configure 3.16.0
         1657  +sqlite configure 3.21.0
  1656   1658   generated by GNU Autoconf 2.69
  1657   1659   
  1658   1660   Copyright (C) 2012 Free Software Foundation, Inc.
  1659   1661   This configure script is free software; the Free Software Foundation
  1660   1662   gives unlimited permission to copy, distribute and modify it.
  1661   1663   _ACEOF
  1662   1664     exit
................................................................................
  2067   2069     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2068   2070   
  2069   2071   } # ac_fn_c_check_header_mongrel
  2070   2072   cat >config.log <<_ACEOF
  2071   2073   This file contains any messages produced by compilers while
  2072   2074   running configure, to aid debugging if configure makes a mistake.
  2073   2075   
  2074         -It was created by sqlite $as_me 3.16.0, which was
         2076  +It was created by sqlite $as_me 3.21.0, which was
  2075   2077   generated by GNU Autoconf 2.69.  Invocation command line was
  2076   2078   
  2077   2079     $ $0 $@
  2078   2080   
  2079   2081   _ACEOF
  2080   2082   exec 5>>config.log
  2081   2083   {
................................................................................
  3925   3927   { $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5
  3926   3928   $as_echo_n "checking the name lister ($NM) interface... " >&6; }
  3927   3929   if ${lt_cv_nm_interface+:} false; then :
  3928   3930     $as_echo_n "(cached) " >&6
  3929   3931   else
  3930   3932     lt_cv_nm_interface="BSD nm"
  3931   3933     echo "int some_variable = 0;" > conftest.$ac_ext
  3932         -  (eval echo "\"\$as_me:3932: $ac_compile\"" >&5)
         3934  +  (eval echo "\"\$as_me:3934: $ac_compile\"" >&5)
  3933   3935     (eval "$ac_compile" 2>conftest.err)
  3934   3936     cat conftest.err >&5
  3935         -  (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
         3937  +  (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  3936   3938     (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  3937   3939     cat conftest.err >&5
  3938         -  (eval echo "\"\$as_me:3938: output\"" >&5)
         3940  +  (eval echo "\"\$as_me:3940: output\"" >&5)
  3939   3941     cat conftest.out >&5
  3940   3942     if $GREP 'External.*some_variable' conftest.out > /dev/null; then
  3941   3943       lt_cv_nm_interface="MS dumpbin"
  3942   3944     fi
  3943   3945     rm -f conftest*
  3944   3946   fi
  3945   3947   { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5
................................................................................
  5137   5139   	;;
  5138   5140       esac
  5139   5141     fi
  5140   5142     rm -rf conftest*
  5141   5143     ;;
  5142   5144   *-*-irix6*)
  5143   5145     # Find out which ABI we are using.
  5144         -  echo '#line 5144 "configure"' > conftest.$ac_ext
         5146  +  echo '#line 5146 "configure"' > conftest.$ac_ext
  5145   5147     if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  5146   5148     (eval $ac_compile) 2>&5
  5147   5149     ac_status=$?
  5148   5150     $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
  5149   5151     test $ac_status = 0; }; then
  5150   5152       if test "$lt_cv_prog_gnu_ld" = yes; then
  5151   5153         case `/usr/bin/file conftest.$ac_objext` in
................................................................................
  6662   6664      # Note that $ac_compile itself does not contain backslashes and begins
  6663   6665      # with a dollar sign (not a hyphen), so the echo should work correctly.
  6664   6666      # The option is referenced via a variable to avoid confusing sed.
  6665   6667      lt_compile=`echo "$ac_compile" | $SED \
  6666   6668      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  6667   6669      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  6668   6670      -e 's:$: $lt_compiler_flag:'`
  6669         -   (eval echo "\"\$as_me:6669: $lt_compile\"" >&5)
         6671  +   (eval echo "\"\$as_me:6671: $lt_compile\"" >&5)
  6670   6672      (eval "$lt_compile" 2>conftest.err)
  6671   6673      ac_status=$?
  6672   6674      cat conftest.err >&5
  6673         -   echo "$as_me:6673: \$? = $ac_status" >&5
         6675  +   echo "$as_me:6675: \$? = $ac_status" >&5
  6674   6676      if (exit $ac_status) && test -s "$ac_outfile"; then
  6675   6677        # The compiler can only warn and ignore the option if not recognized
  6676   6678        # So say no if there are warnings other than the usual output.
  6677   6679        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  6678   6680        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  6679   6681        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  6680   6682          lt_cv_prog_compiler_rtti_exceptions=yes
................................................................................
  7001   7003      # Note that $ac_compile itself does not contain backslashes and begins
  7002   7004      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7003   7005      # The option is referenced via a variable to avoid confusing sed.
  7004   7006      lt_compile=`echo "$ac_compile" | $SED \
  7005   7007      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7006   7008      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7007   7009      -e 's:$: $lt_compiler_flag:'`
  7008         -   (eval echo "\"\$as_me:7008: $lt_compile\"" >&5)
         7010  +   (eval echo "\"\$as_me:7010: $lt_compile\"" >&5)
  7009   7011      (eval "$lt_compile" 2>conftest.err)
  7010   7012      ac_status=$?
  7011   7013      cat conftest.err >&5
  7012         -   echo "$as_me:7012: \$? = $ac_status" >&5
         7014  +   echo "$as_me:7014: \$? = $ac_status" >&5
  7013   7015      if (exit $ac_status) && test -s "$ac_outfile"; then
  7014   7016        # The compiler can only warn and ignore the option if not recognized
  7015   7017        # So say no if there are warnings other than the usual output.
  7016   7018        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  7017   7019        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  7018   7020        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  7019   7021          lt_cv_prog_compiler_pic_works=yes
................................................................................
  7106   7108      # (2) before a word containing "conftest.", or (3) at the end.
  7107   7109      # Note that $ac_compile itself does not contain backslashes and begins
  7108   7110      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7109   7111      lt_compile=`echo "$ac_compile" | $SED \
  7110   7112      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7111   7113      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7112   7114      -e 's:$: $lt_compiler_flag:'`
  7113         -   (eval echo "\"\$as_me:7113: $lt_compile\"" >&5)
         7115  +   (eval echo "\"\$as_me:7115: $lt_compile\"" >&5)
  7114   7116      (eval "$lt_compile" 2>out/conftest.err)
  7115   7117      ac_status=$?
  7116   7118      cat out/conftest.err >&5
  7117         -   echo "$as_me:7117: \$? = $ac_status" >&5
         7119  +   echo "$as_me:7119: \$? = $ac_status" >&5
  7118   7120      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7119   7121      then
  7120   7122        # The compiler can only warn and ignore the option if not recognized
  7121   7123        # So say no if there are warnings
  7122   7124        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7123   7125        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7124   7126        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  7161   7163      # (2) before a word containing "conftest.", or (3) at the end.
  7162   7164      # Note that $ac_compile itself does not contain backslashes and begins
  7163   7165      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7164   7166      lt_compile=`echo "$ac_compile" | $SED \
  7165   7167      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7166   7168      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7167   7169      -e 's:$: $lt_compiler_flag:'`
  7168         -   (eval echo "\"\$as_me:7168: $lt_compile\"" >&5)
         7170  +   (eval echo "\"\$as_me:7170: $lt_compile\"" >&5)
  7169   7171      (eval "$lt_compile" 2>out/conftest.err)
  7170   7172      ac_status=$?
  7171   7173      cat out/conftest.err >&5
  7172         -   echo "$as_me:7172: \$? = $ac_status" >&5
         7174  +   echo "$as_me:7174: \$? = $ac_status" >&5
  7173   7175      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7174   7176      then
  7175   7177        # The compiler can only warn and ignore the option if not recognized
  7176   7178        # So say no if there are warnings
  7177   7179        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7178   7180        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7179   7181        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  9541   9543   else
  9542   9544     	  if test "$cross_compiling" = yes; then :
  9543   9545     lt_cv_dlopen_self=cross
  9544   9546   else
  9545   9547     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9546   9548     lt_status=$lt_dlunknown
  9547   9549     cat > conftest.$ac_ext <<_LT_EOF
  9548         -#line 9548 "configure"
         9550  +#line 9550 "configure"
  9549   9551   #include "confdefs.h"
  9550   9552   
  9551   9553   #if HAVE_DLFCN_H
  9552   9554   #include <dlfcn.h>
  9553   9555   #endif
  9554   9556   
  9555   9557   #include <stdio.h>
................................................................................
  9637   9639   else
  9638   9640     	  if test "$cross_compiling" = yes; then :
  9639   9641     lt_cv_dlopen_self_static=cross
  9640   9642   else
  9641   9643     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9642   9644     lt_status=$lt_dlunknown
  9643   9645     cat > conftest.$ac_ext <<_LT_EOF
  9644         -#line 9644 "configure"
         9646  +#line 9646 "configure"
  9645   9647   #include "confdefs.h"
  9646   9648   
  9647   9649   #if HAVE_DLFCN_H
  9648   9650   #include <dlfcn.h>
  9649   9651   #endif
  9650   9652   
  9651   9653   #include <stdio.h>
................................................................................
 10298  10300   USE_AMALGAMATION=1
 10299  10301   
 10300  10302   #########
 10301  10303   # See whether we can run specific tclsh versions known to work well;
 10302  10304   # if not, then we fall back to plain tclsh.
 10303  10305   # TODO: try other versions before falling back?
 10304  10306   #
 10305         -for ac_prog in tclsh8.6 tclsh8.5 tclsh
        10307  +for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
 10306  10308   do
 10307  10309     # Extract the first word of "$ac_prog", so it can be a program name with args.
 10308  10310   set dummy $ac_prog; ac_word=$2
 10309  10311   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 10310  10312   $as_echo_n "checking for $ac_word... " >&6; }
 10311  10313   if ${ac_cv_prog_TCLSH_CMD+:} false; then :
 10312  10314     $as_echo_n "(cached) " >&6
................................................................................
 11248  11250   if test "${enable_debug+set}" = set; then :
 11249  11251     enableval=$enable_debug; use_debug=$enableval
 11250  11252   else
 11251  11253     use_debug=no
 11252  11254   fi
 11253  11255   
 11254  11256   if test "${use_debug}" = "yes" ; then
 11255         -  TARGET_DEBUG="-DSQLITE_DEBUG=1"
        11257  +  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 11256  11258   else
 11257  11259     TARGET_DEBUG="-DNDEBUG"
 11258  11260   fi
 11259  11261   
 11260  11262   
 11261  11263   #########
 11262  11264   # See whether we should use the amalgamation to build
................................................................................
 11352  11354   else
 11353  11355     enable_memsys5=no
 11354  11356   fi
 11355  11357   
 11356  11358   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
 11357  11359   $as_echo_n "checking whether to support MEMSYS5... " >&6; }
 11358  11360   if test "${enable_memsys5}" = "yes"; then
 11359         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
        11361  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
 11360  11362     { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 11361  11363   $as_echo "yes" >&6; }
 11362  11364   else
 11363  11365     { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 11364  11366   $as_echo "no" >&6; }
 11365  11367   fi
 11366  11368   # Check whether --enable-memsys3 was given.
................................................................................
 11369  11371   else
 11370  11372     enable_memsys3=no
 11371  11373   fi
 11372  11374   
 11373  11375   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
 11374  11376   $as_echo_n "checking whether to support MEMSYS3... " >&6; }
 11375  11377   if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
 11376         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
        11378  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
 11377  11379     { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 11378  11380   $as_echo "yes" >&6; }
 11379  11381   else
 11380  11382     { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 11381  11383   $as_echo "no" >&6; }
 11382  11384   fi
 11383  11385   
................................................................................
 11387  11389   if test "${enable_fts3+set}" = set; then :
 11388  11390     enableval=$enable_fts3; enable_fts3=yes
 11389  11391   else
 11390  11392     enable_fts3=no
 11391  11393   fi
 11392  11394   
 11393  11395   if test "${enable_fts3}" = "yes" ; then
 11394         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
        11396  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 11395  11397   fi
 11396  11398   # Check whether --enable-fts4 was given.
 11397  11399   if test "${enable_fts4+set}" = set; then :
 11398  11400     enableval=$enable_fts4; enable_fts4=yes
 11399  11401   else
 11400  11402     enable_fts4=no
 11401  11403   fi
 11402  11404   
 11403  11405   if test "${enable_fts4}" = "yes" ; then
 11404         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
        11406  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
 11405  11407     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 11406  11408   $as_echo_n "checking for library containing log... " >&6; }
 11407  11409   if ${ac_cv_search_log+:} false; then :
 11408  11410     $as_echo_n "(cached) " >&6
 11409  11411   else
 11410  11412     ac_func_search_save_LIBS=$LIBS
 11411  11413   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
................................................................................
 11463  11465   if test "${enable_fts5+set}" = set; then :
 11464  11466     enableval=$enable_fts5; enable_fts5=yes
 11465  11467   else
 11466  11468     enable_fts5=no
 11467  11469   fi
 11468  11470   
 11469  11471   if test "${enable_fts5}" = "yes" ; then
 11470         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
        11472  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
 11471  11473     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 11472  11474   $as_echo_n "checking for library containing log... " >&6; }
 11473  11475   if ${ac_cv_search_log+:} false; then :
 11474  11476     $as_echo_n "(cached) " >&6
 11475  11477   else
 11476  11478     ac_func_search_save_LIBS=$LIBS
 11477  11479   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
................................................................................
 11532  11534   if test "${enable_json1+set}" = set; then :
 11533  11535     enableval=$enable_json1; enable_json1=yes
 11534  11536   else
 11535  11537     enable_json1=no
 11536  11538   fi
 11537  11539   
 11538  11540   if test "${enable_json1}" = "yes" ; then
 11539         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
        11541  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
        11542  +fi
        11543  +
        11544  +#########
        11545  +# See whether we should enable the LIMIT clause on UPDATE and DELETE
        11546  +# statements.
        11547  +# Check whether --enable-update-limit was given.
        11548  +if test "${enable_update_limit+set}" = set; then :
        11549  +  enableval=$enable_update_limit; enable_udlimit=yes
        11550  +else
        11551  +  enable_udlimit=no
        11552  +fi
        11553  +
        11554  +if test "${enable_udlimit}" = "yes" ; then
        11555  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 11540  11556   fi
 11541  11557   
 11542  11558   #########
 11543  11559   # See whether we should enable RTREE
 11544  11560   # Check whether --enable-rtree was given.
 11545  11561   if test "${enable_rtree+set}" = set; then :
 11546  11562     enableval=$enable_rtree; enable_rtree=yes
 11547  11563   else
 11548  11564     enable_rtree=no
 11549  11565   fi
 11550  11566   
 11551  11567   if test "${enable_rtree}" = "yes" ; then
 11552         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
        11568  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 11553  11569   fi
 11554  11570   
 11555  11571   #########
 11556  11572   # See whether we should enable the SESSION extension
 11557  11573   # Check whether --enable-session was given.
 11558  11574   if test "${enable_session+set}" = set; then :
 11559  11575     enableval=$enable_session; enable_session=yes
 11560  11576   else
 11561  11577     enable_session=no
 11562  11578   fi
 11563  11579   
 11564  11580   if test "${enable_session}" = "yes" ; then
 11565         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
 11566         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
        11581  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
        11582  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
 11567  11583   fi
 11568  11584   
 11569  11585   #########
 11570         -# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
        11586  +# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
 11571  11587   for option in $CFLAGS $CPPFLAGS
 11572  11588   do
 11573  11589     case $option in
 11574  11590       -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
 11575  11591       -DSQLITE_ENABLE*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
 11576  11592     esac
 11577  11593   done
................................................................................
 12147  12163   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12148  12164   
 12149  12165   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12150  12166   # Save the log message, to keep $0 and so on meaningful, and to
 12151  12167   # report actual input values of CONFIG_FILES etc. instead of their
 12152  12168   # values after options handling.
 12153  12169   ac_log="
 12154         -This file was extended by sqlite $as_me 3.16.0, which was
        12170  +This file was extended by sqlite $as_me 3.21.0, which was
 12155  12171   generated by GNU Autoconf 2.69.  Invocation command line was
 12156  12172   
 12157  12173     CONFIG_FILES    = $CONFIG_FILES
 12158  12174     CONFIG_HEADERS  = $CONFIG_HEADERS
 12159  12175     CONFIG_LINKS    = $CONFIG_LINKS
 12160  12176     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12161  12177     $ $0 $@
................................................................................
 12213  12229   
 12214  12230   Report bugs to the package provider."
 12215  12231   
 12216  12232   _ACEOF
 12217  12233   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12218  12234   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12219  12235   ac_cs_version="\\
 12220         -sqlite config.status 3.16.0
        12236  +sqlite config.status 3.21.0
 12221  12237   configured by $0, generated by GNU Autoconf 2.69,
 12222  12238     with options \\"\$ac_cs_config\\"
 12223  12239   
 12224  12240   Copyright (C) 2012 Free Software Foundation, Inc.
 12225  12241   This config.status script is free software; the Free Software Foundation
 12226  12242   gives unlimited permission to copy, distribute and modify it."
 12227  12243   

Changes to configure.ac.

   116    116   USE_AMALGAMATION=1
   117    117   
   118    118   #########
   119    119   # See whether we can run specific tclsh versions known to work well;
   120    120   # if not, then we fall back to plain tclsh.
   121    121   # TODO: try other versions before falling back?
   122    122   # 
   123         -AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh8.5 tclsh], none)
          123  +AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
   124    124   if test "$TCLSH_CMD" = "none"; then
   125    125     # If we can't find a local tclsh, then building the amalgamation will fail.
   126    126     # We act as though --disable-amalgamation has been used.
   127    127     echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
   128    128     USE_AMALGAMATION=0
   129    129     TCLSH_CMD="tclsh"
   130    130   fi
................................................................................
   556    556   AC_SEARCH_LIBS(fdatasync, [rt])
   557    557   
   558    558   #########
   559    559   # check for debug enabled
   560    560   AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
   561    561         [use_debug=$enableval],[use_debug=no])
   562    562   if test "${use_debug}" = "yes" ; then
   563         -  TARGET_DEBUG="-DSQLITE_DEBUG=1"
          563  +  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
   564    564   else
   565    565     TARGET_DEBUG="-DNDEBUG"
   566    566   fi
   567    567   AC_SUBST(TARGET_DEBUG)
   568    568   
   569    569   #########
   570    570   # See whether we should use the amalgamation to build
................................................................................
   592    592   # Do we want to support memsys3 and/or memsys5
   593    593   #
   594    594   AC_ARG_ENABLE(memsys5, 
   595    595     AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
   596    596     [enable_memsys5=yes],[enable_memsys5=no])
   597    597   AC_MSG_CHECKING([whether to support MEMSYS5])
   598    598   if test "${enable_memsys5}" = "yes"; then
   599         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5"
          599  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
   600    600     AC_MSG_RESULT([yes])
   601    601   else
   602    602     AC_MSG_RESULT([no])
   603    603   fi
   604    604   AC_ARG_ENABLE(memsys3, 
   605    605     AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
   606    606     [enable_memsys3=yes],[enable_memsys3=no])
   607    607   AC_MSG_CHECKING([whether to support MEMSYS3])
   608    608   if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
   609         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3"
          609  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
   610    610     AC_MSG_RESULT([yes])
   611    611   else
   612    612     AC_MSG_RESULT([no])
   613    613   fi
   614    614   
   615    615   #########
   616    616   # See whether we should enable Full Text Search extensions
   617    617   AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
   618    618         [Enable the FTS3 extension]),
   619    619         [enable_fts3=yes],[enable_fts3=no])
   620    620   if test "${enable_fts3}" = "yes" ; then
   621         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
          621  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
   622    622   fi
   623    623   AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
   624    624         [Enable the FTS4 extension]),
   625    625         [enable_fts4=yes],[enable_fts4=no])
   626    626   if test "${enable_fts4}" = "yes" ; then
   627         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
          627  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
   628    628     AC_SEARCH_LIBS([log],[m])
   629    629   fi
   630    630   AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
   631    631         [Enable the FTS5 extension]),
   632    632         [enable_fts5=yes],[enable_fts5=no])
   633    633   if test "${enable_fts5}" = "yes" ; then
   634         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
          634  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
   635    635     AC_SEARCH_LIBS([log],[m])
   636    636   fi
   637    637   
   638    638   #########
   639    639   # See whether we should enable JSON1
   640    640   AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
   641    641         [Enable the JSON1 extension]),
   642    642         [enable_json1=yes],[enable_json1=no])
   643    643   if test "${enable_json1}" = "yes" ; then
   644         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
          644  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
          645  +fi
          646  +
          647  +#########
          648  +# See whether we should enable the LIMIT clause on UPDATE and DELETE
          649  +# statements.
          650  +AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
          651  +      [Enable the UPDATE/DELETE LIMIT clause]),
          652  +      [enable_udlimit=yes],[enable_udlimit=no])
          653  +if test "${enable_udlimit}" = "yes" ; then
          654  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
   645    655   fi
   646    656   
   647    657   #########
   648    658   # See whether we should enable RTREE
   649    659   AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
   650    660         [Enable the RTREE extension]),
   651    661         [enable_rtree=yes],[enable_rtree=no])
   652    662   if test "${enable_rtree}" = "yes" ; then
   653         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
          663  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
   654    664   fi
   655    665   
   656    666   #########
   657    667   # See whether we should enable the SESSION extension
   658    668   AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
   659    669         [Enable the SESSION extension]),
   660    670         [enable_session=yes],[enable_session=no])
   661    671   if test "${enable_session}" = "yes" ; then
   662         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION"
   663         -  OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK"
          672  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
          673  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
   664    674   fi
   665    675   
   666    676   #########
   667         -# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
          677  +# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
   668    678   for option in $CFLAGS $CPPFLAGS
   669    679   do
   670    680     case $option in
   671    681       -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
   672    682       -DSQLITE_ENABLE*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
   673    683     esac
   674    684   done

Changes to doc/lemon.html.

     1      1   <html>
     2      2   <head>
     3      3   <title>The Lemon Parser Generator</title>
     4      4   </head>
     5         -<body bgcolor=white>
     6         -<h1 align=center>The Lemon Parser Generator</h1>
            5  +<body bgcolor='white'>
            6  +<h1 align='center'>The Lemon Parser Generator</h1>
     7      7   
     8      8   <p>Lemon is an LALR(1) parser generator for C.
     9      9   It does the same job as "bison" and "yacc".
    10         -But lemon is not a bison or yacc clone.  Lemon
           10  +But Lemon is not a bison or yacc clone.  Lemon
    11     11   uses a different grammar syntax which is designed to
    12     12   reduce the number of coding errors.  Lemon also uses a
    13     13   parsing engine that is faster than yacc and
    14     14   bison and which is both reentrant and threadsafe.
    15     15   (Update: Since the previous sentence was written, bison
    16     16   has also been updated so that it too can generate a
    17     17   reentrant and threadsafe parser.)
    18     18   Lemon also implements features that can be used
    19         -to eliminate resource leaks, making is suitable for use
           19  +to eliminate resource leaks, making it suitable for use
    20     20   in long-running programs such as graphical user interfaces
    21     21   or embedded controllers.</p>
    22     22   
    23     23   <p>This document is an introduction to the Lemon
    24     24   parser generator.</p>
           25  +
           26  +<h2>Security Note</h2>
           27  +
           28  +<p>The language parser code created by Lemon is very robust and
           29  +is well-suited for use in internet-facing applications that need to
           30  +safely process maliciously crafted inputs.
           31  +
           32  +<p>The "lemon.exe" command-line tool itself works great when given a valid
           33  +input grammar file and almost always gives helpful
           34  +error messages for malformed inputs.  However,  it is possible for
           35  +a malicious user to craft a grammar file that will cause 
           36  +lemon.exe to crash.
           37  +We do not see this as a problem, as lemon.exe is not intended to be used
           38  +with hostile inputs.
           39  +To summarize:</p>
           40  +
           41  +<ul>
           42  +<li>Parser code generated by lemon &rarr; Robust and secure
           43  +<li>The "lemon.exe" command line tool itself &rarr; Not so much
           44  +</ul>
    25     45   
    26     46   <h2>Theory of Operation</h2>
    27     47   
    28     48   <p>The main goal of Lemon is to translate a context free grammar (CFG)
    29     49   for a particular language into C code that implements a parser for
    30     50   that language.
    31     51   The program has two inputs:
................................................................................
    34     54   <li>A parser template file.
    35     55   </ul>
    36     56   Typically, only the grammar specification is supplied by the programmer.
    37     57   Lemon comes with a default parser template which works fine for most
    38     58   applications.  But the user is free to substitute a different parser
    39     59   template if desired.</p>
    40     60   
    41         -<p>Depending on command-line options, Lemon will generate between
    42         -one and three files of outputs.
           61  +<p>Depending on command-line options, Lemon will generate up to
           62  +three output files.
    43     63   <ul>
    44     64   <li>C code to implement the parser.
    45     65   <li>A header file defining an integer ID for each terminal symbol.
    46     66   <li>An information file that describes the states of the generated parser
    47     67       automaton.
    48     68   </ul>
    49     69   By default, all three of these output files are generated.
................................................................................
    66     86   
    67     87   <h3>Command Line Options</h3>
    68     88   
    69     89   <p>The behavior of Lemon can be modified using command-line options.
    70     90   You can obtain a list of the available command-line options together
    71     91   with a brief explanation of what each does by typing
    72     92   <pre>
    73         -   lemon -?
           93  +   lemon "-?"
    74     94   </pre>
    75     95   As of this writing, the following command-line options are supported:
    76     96   <ul>
    77     97   <li><b>-b</b>
    78     98   Show only the basis for each parser state in the report file.
    79     99   <li><b>-c</b>
    80         -Do not compress the generated action tables.
          100  +Do not compress the generated action tables.  The parser will be a
          101  +little larger and slower, but it will detect syntax errors sooner.
    81    102   <li><b>-D<i>name</i></b>
    82         -Define C preprocessor macro <i>name</i>.  This macro is useable by
    83         -"%ifdef" lines in the grammar file.
          103  +Define C preprocessor macro <i>name</i>.  This macro is usable by
          104  +"<tt><a href='#pifdef'>%ifdef</a></tt>" and
          105  +"<tt><a href='#pifdef'>%ifndef</a></tt>" lines
          106  +in the grammar file.
    84    107   <li><b>-g</b>
    85    108   Do not generate a parser.  Instead write the input grammar to standard
    86    109   output with all comments, actions, and other extraneous text removed.
    87    110   <li><b>-l</b>
    88    111   Omit "#line" directives in the generated parser C code.
    89    112   <li><b>-m</b>
    90    113   Cause the output C source code to be compatible with the "makeheaders"
    91         -program. 
          114  +program.
    92    115   <li><b>-p</b>
    93         -Display all conflicts that are resolved by 
          116  +Display all conflicts that are resolved by
    94    117   <a href='#precrules'>precedence rules</a>.
    95    118   <li><b>-q</b>
    96    119   Suppress generation of the report file.
    97    120   <li><b>-r</b>
    98    121   Do not sort or renumber the parser states as part of optimization.
    99    122   <li><b>-s</b>
   100    123   Show parser statistics before existing.
................................................................................
   141    164   be parsed.  This is accomplished by calling the following function
   142    165   once for each token:
   143    166   <pre>
   144    167      Parse(pParser, hTokenID, sTokenData, pArg);
   145    168   </pre>
   146    169   The first argument to the Parse() routine is the pointer returned by
   147    170   ParseAlloc().
   148         -The second argument is a small positive integer that tells the parse the
          171  +The second argument is a small positive integer that tells the parser the
   149    172   type of the next token in the data stream.
   150    173   There is one token type for each terminal symbol in the grammar.
   151    174   The gram.h file generated by Lemon contains #define statements that
   152    175   map symbolic terminal symbol names into appropriate integer values.
   153    176   A value of 0 for the second argument is a special flag to the
   154    177   parser to indicate that the end of input has been reached.
   155    178   The third argument is the value of the given token.  By default,
   156         -the type of the third argument is integer, but the grammar will
          179  +the type of the third argument is "void*", but the grammar will
   157    180   usually redefine this type to be some kind of structure.
   158    181   Typically the second argument will be a broad category of tokens
   159    182   such as "identifier" or "number" and the third argument will
   160    183   be the name of the identifier or the value of the number.</p>
   161    184   
   162    185   <p>The Parse() function may have either three or four arguments,
   163    186   depending on the grammar.  If the grammar specification file requests
   164         -it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
          187  +it (via the <tt><a href='#extraarg'>%extra_argument</a></tt> directive),
   165    188   the Parse() function will have a fourth parameter that can be
   166    189   of any type chosen by the programmer.  The parser doesn't do anything
   167    190   with this argument except to pass it through to action routines.
   168    191   This is a convenient mechanism for passing state information down
   169    192   to the action routines without having to use global variables.</p>
   170    193   
   171    194   <p>A typical use of a Lemon parser might look something like the
   172    195   following:
   173    196   <pre>
   174         -   01 ParseTree *ParseFile(const char *zFilename){
   175         -   02    Tokenizer *pTokenizer;
   176         -   03    void *pParser;
   177         -   04    Token sToken;
   178         -   05    int hTokenId;
   179         -   06    ParserState sState;
   180         -   07
   181         -   08    pTokenizer = TokenizerCreate(zFilename);
   182         -   09    pParser = ParseAlloc( malloc );
   183         -   10    InitParserState(&sState);
   184         -   11    while( GetNextToken(pTokenizer, &hTokenId, &sToken) ){
   185         -   12       Parse(pParser, hTokenId, sToken, &sState);
          197  +    1 ParseTree *ParseFile(const char *zFilename){
          198  +    2    Tokenizer *pTokenizer;
          199  +    3    void *pParser;
          200  +    4    Token sToken;
          201  +    5    int hTokenId;
          202  +    6    ParserState sState;
          203  +    7
          204  +    8    pTokenizer = TokenizerCreate(zFilename);
          205  +    9    pParser = ParseAlloc( malloc );
          206  +   10    InitParserState(&amp;sState);
          207  +   11    while( GetNextToken(pTokenizer, &amp;hTokenId, &amp;sToken) ){
          208  +   12       Parse(pParser, hTokenId, sToken, &amp;sState);
   186    209      13    }
   187         -   14    Parse(pParser, 0, sToken, &sState);
          210  +   14    Parse(pParser, 0, sToken, &amp;sState);
   188    211      15    ParseFree(pParser, free );
   189    212      16    TokenizerFree(pTokenizer);
   190    213      17    return sState.treeRoot;
   191    214      18 }
   192    215   </pre>
   193    216   This example shows a user-written routine that parses a file of
   194    217   text and returns a pointer to the parse tree.
   195    218   (All error-handling code is omitted from this example to keep it
   196    219   simple.)
   197    220   We assume the existence of some kind of tokenizer which is created
   198    221   using TokenizerCreate() on line 8 and deleted by TokenizerFree()
   199    222   on line 16.  The GetNextToken() function on line 11 retrieves the
   200         -next token from the input file and puts its type in the 
          223  +next token from the input file and puts its type in the
   201    224   integer variable hTokenId.  The sToken variable is assumed to be
   202    225   some kind of structure that contains details about each token,
   203         -such as its complete text, what line it occurs on, etc. </p>
          226  +such as its complete text, what line it occurs on, etc.</p>
   204    227   
   205    228   <p>This example also assumes the existence of structure of type
   206    229   ParserState that holds state information about a particular parse.
   207    230   An instance of such a structure is created on line 6 and initialized
   208    231   on line 10.  A pointer to this structure is passed into the Parse()
   209    232   routine as the optional 4th argument.
   210    233   The action routine specified by the grammar for the parser can use
................................................................................
   213    236   the ParserState structure is left pointing to the root of the parse
   214    237   tree.</p>
   215    238   
   216    239   <p>The core of this example as it relates to Lemon is as follows:
   217    240   <pre>
   218    241      ParseFile(){
   219    242         pParser = ParseAlloc( malloc );
   220         -      while( GetNextToken(pTokenizer,&hTokenId, &sToken) ){
          243  +      while( GetNextToken(pTokenizer,&amp;hTokenId, &amp;sToken) ){
   221    244            Parse(pParser, hTokenId, sToken);
   222    245         }
   223    246         Parse(pParser, 0, sToken);
   224    247         ParseFree(pParser, free );
   225    248      }
   226    249   </pre>
   227    250   Basically, what a program has to do to use a Lemon-generated parser
................................................................................
   273    296   
   274    297   <p>The main purpose of the grammar specification file for Lemon is
   275    298   to define the grammar for the parser.  But the input file also
   276    299   specifies additional information Lemon requires to do its job.
   277    300   Most of the work in using Lemon is in writing an appropriate
   278    301   grammar file.</p>
   279    302   
   280         -<p>The grammar file for lemon is, for the most part, free format.
          303  +<p>The grammar file for Lemon is, for the most part, free format.
   281    304   It does not have sections or divisions like yacc or bison.  Any
   282    305   declaration can occur at any point in the file.
   283    306   Lemon ignores whitespace (except where it is needed to separate
   284         -tokens) and it honors the same commenting conventions as C and C++.</p>
          307  +tokens), and it honors the same commenting conventions as C and C++.</p>
   285    308   
   286    309   <h3>Terminals and Nonterminals</h3>
   287    310   
   288    311   <p>A terminal symbol (token) is any string of alphanumeric
   289    312   and/or underscore characters
   290         -that begins with an upper case letter.
          313  +that begins with an uppercase letter.
   291    314   A terminal can contain lowercase letters after the first character,
   292         -but the usual convention is to make terminals all upper case.
          315  +but the usual convention is to make terminals all uppercase.
   293    316   A nonterminal, on the other hand, is any string of alphanumeric
   294         -and underscore characters than begins with a lower case letter.
   295         -Again, the usual convention is to make nonterminals use all lower
   296         -case letters.</p>
          317  +and underscore characters than begins with a lowercase letter.
          318  +Again, the usual convention is to make nonterminals use all lowercase
          319  +letters.</p>
   297    320   
   298         -<p>In Lemon, terminal and nonterminal symbols do not need to 
          321  +<p>In Lemon, terminal and nonterminal symbols do not need to
   299    322   be declared or identified in a separate section of the grammar file.
   300    323   Lemon is able to generate a list of all terminals and nonterminals
   301    324   by examining the grammar rules, and it can always distinguish a
   302    325   terminal from a nonterminal by checking the case of the first
   303    326   character of the name.</p>
   304    327   
   305    328   <p>Yacc and bison allow terminal symbols to have either alphanumeric
................................................................................
   315    338   Each grammar rule consists of a nonterminal symbol followed by
   316    339   the special symbol "::=" and then a list of terminals and/or nonterminals.
   317    340   The rule is terminated by a period.
   318    341   The list of terminals and nonterminals on the right-hand side of the
   319    342   rule can be empty.
   320    343   Rules can occur in any order, except that the left-hand side of the
   321    344   first rule is assumed to be the start symbol for the grammar (unless
   322         -specified otherwise using the <tt>%start</tt> directive described below.)
          345  +specified otherwise using the <tt><a href='#start_symbol'>%start_symbol</a></tt>
          346  +directive described below.)
   323    347   A typical sequence of grammar rules might look something like this:
   324    348   <pre>
   325    349     expr ::= expr PLUS expr.
   326    350     expr ::= expr TIMES expr.
   327    351     expr ::= LPAREN expr RPAREN.
   328    352     expr ::= VALUE.
   329    353   </pre>
................................................................................
   358    382   rule and say "$7" when you really mean "$8".</p>
   359    383   
   360    384   <p>Lemon avoids the need to count grammar symbols by assigning symbolic
   361    385   names to each symbol in a grammar rule and then using those symbolic
   362    386   names in the action.
   363    387   In yacc or bison, one would write this:
   364    388   <pre>
   365         -  expr -> expr PLUS expr  { $$ = $1 + $3; };
          389  +  expr -&gt; expr PLUS expr  { $$ = $1 + $3; };
   366    390   </pre>
   367    391   But in Lemon, the same rule becomes the following:
   368    392   <pre>
   369    393     expr(A) ::= expr(B) PLUS expr(C).  { A = B+C; }
   370    394   </pre>
   371    395   In the Lemon rule, any symbol in parentheses after a grammar rule
   372    396   symbol becomes a place holder for that symbol in the grammar rule.
................................................................................
   398    422   
   399    423   <p>Lemon resolves parsing ambiguities in exactly the same way as
   400    424   yacc and bison.  A shift-reduce conflict is resolved in favor
   401    425   of the shift, and a reduce-reduce conflict is resolved by reducing
   402    426   whichever rule comes first in the grammar file.</p>
   403    427   
   404    428   <p>Just like in
   405         -yacc and bison, Lemon allows a measure of control 
   406         -over the resolution of paring conflicts using precedence rules.
          429  +yacc and bison, Lemon allows a measure of control
          430  +over the resolution of parsing conflicts using precedence rules.
   407    431   A precedence value can be assigned to any terminal symbol
   408         -using the 
   409         -<a href='#pleft'>%left</a>,
   410         -<a href='#pright'>%right</a> or
   411         -<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
   412         -mentioned in earlier directives have a lower precedence that
          432  +using the
          433  +<tt><a href='#pleft'>%left</a></tt>,
          434  +<tt><a href='#pright'>%right</a></tt> or
          435  +<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives.  Terminal symbols
          436  +mentioned in earlier directives have a lower precedence than
   413    437   terminal symbols mentioned in later directives.  For example:</p>
   414    438   
   415    439   <p><pre>
   416    440      %left AND.
   417    441      %left OR.
   418    442      %nonassoc EQ NE GT GE LT LE.
   419    443      %left PLUS MINUS.
................................................................................
   481    505   <ul>
   482    506   <li> If either the token to be shifted or the rule to be reduced
   483    507        lacks precedence information, then resolve in favor of the
   484    508        shift, but report a parsing conflict.
   485    509   <li> If the precedence of the token to be shifted is greater than
   486    510        the precedence of the rule to reduce, then resolve in favor
   487    511        of the shift.  No parsing conflict is reported.
   488         -<li> If the precedence of the token it be shifted is less than the
          512  +<li> If the precedence of the token to be shifted is less than the
   489    513        precedence of the rule to reduce, then resolve in favor of the
   490    514        reduce action.  No parsing conflict is reported.
   491    515   <li> If the precedences are the same and the shift token is
   492    516        right-associative, then resolve in favor of the shift.
   493    517        No parsing conflict is reported.
   494         -<li> If the precedences are the same the shift token is
          518  +<li> If the precedences are the same and the shift token is
   495    519        left-associative, then resolve in favor of the reduce.
   496    520        No parsing conflict is reported.
   497         -<li> Otherwise, resolve the conflict by doing the shift and
   498         -     report the parsing conflict.
          521  +<li> Otherwise, resolve the conflict by doing the shift, and
          522  +     report a parsing conflict.
   499    523   </ul>
   500    524   Reduce-reduce conflicts are resolved this way:
   501    525   <ul>
   502         -<li> If either reduce rule 
          526  +<li> If either reduce rule
   503    527        lacks precedence information, then resolve in favor of the
   504         -     rule that appears first in the grammar and report a parsing
          528  +     rule that appears first in the grammar, and report a parsing
   505    529        conflict.
   506         -<li> If both rules have precedence and the precedence is different
          530  +<li> If both rules have precedence and the precedence is different,
   507    531        then resolve the dispute in favor of the rule with the highest
   508         -     precedence and do not report a conflict.
          532  +     precedence, and do not report a conflict.
   509    533   <li> Otherwise, resolve the conflict by reducing by the rule that
   510         -     appears first in the grammar and report a parsing conflict.
          534  +     appears first in the grammar, and report a parsing conflict.
   511    535   </ul>
   512    536   
   513    537   <h3>Special Directives</h3>
   514    538   
   515    539   <p>The input grammar to Lemon consists of grammar rules and special
   516    540   directives.  We've described all the grammar rules, so now we'll
   517    541   talk about the special directives.</p>
   518    542   
   519         -<p>Directives in lemon can occur in any order.  You can put them before
   520         -the grammar rules, or after the grammar rules, or in the mist of the
          543  +<p>Directives in Lemon can occur in any order.  You can put them before
          544  +the grammar rules, or after the grammar rules, or in the midst of the
   521    545   grammar rules.  It doesn't matter.  The relative order of
   522    546   directives used to assign precedence to terminals is important, but
   523    547   other than that, the order of directives in Lemon is arbitrary.</p>
   524    548   
   525    549   <p>Lemon supports the following special directives:
   526    550   <ul>
   527         -<li><tt>%code</tt>
   528         -<li><tt>%default_destructor</tt>
   529         -<li><tt>%default_type</tt>
   530         -<li><tt>%destructor</tt>
   531         -<li><tt>%endif</tt>
   532         -<li><tt>%extra_argument</tt>
   533         -<li><tt>%fallback</tt>
   534         -<li><tt>%ifdef</tt>
   535         -<li><tt>%ifndef</tt>
   536         -<li><tt>%include</tt>
   537         -<li><tt>%left</tt>
   538         -<li><tt>%name</tt>
   539         -<li><tt>%nonassoc</tt>
   540         -<li><tt>%parse_accept</tt>
   541         -<li><tt>%parse_failure </tt>
   542         -<li><tt>%right</tt>
   543         -<li><tt>%stack_overflow</tt>
   544         -<li><tt>%stack_size</tt>
   545         -<li><tt>%start_symbol</tt>
   546         -<li><tt>%syntax_error</tt>
   547         -<li><tt>%token_class</tt>
   548         -<li><tt>%token_destructor</tt>
   549         -<li><tt>%token_prefix</tt>
   550         -<li><tt>%token_type</tt>
   551         -<li><tt>%type</tt>
   552         -<li><tt>%wildcard</tt>
          551  +<li><tt><a href='#pcode'>%code</a></tt>
          552  +<li><tt><a href='#default_destructor'>%default_destructor</a></tt>
          553  +<li><tt><a href='#default_type'>%default_type</a></tt>
          554  +<li><tt><a href='#destructor'>%destructor</a></tt>
          555  +<li><tt><a href='#pifdef'>%endif</a></tt>
          556  +<li><tt><a href='#extraarg'>%extra_argument</a></tt>
          557  +<li><tt><a href='#pfallback'>%fallback</a></tt>
          558  +<li><tt><a href='#pifdef'>%ifdef</a></tt>
          559  +<li><tt><a href='#pifdef'>%ifndef</a></tt>
          560  +<li><tt><a href='#pinclude'>%include</a></tt>
          561  +<li><tt><a href='#pleft'>%left</a></tt>
          562  +<li><tt><a href='#pname'>%name</a></tt>
          563  +<li><tt><a href='#pnonassoc'>%nonassoc</a></tt>
          564  +<li><tt><a href='#parse_accept'>%parse_accept</a></tt>
          565  +<li><tt><a href='#parse_failure'>%parse_failure</a></tt>
          566  +<li><tt><a href='#pright'>%right</a></tt>
          567  +<li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
          568  +<li><tt><a href='#stack_size'>%stack_size</a></tt>
          569  +<li><tt><a href='#start_symbol'>%start_symbol</a></tt>
          570  +<li><tt><a href='#syntax_error'>%syntax_error</a></tt>
          571  +<li><tt><a href='#token_class'>%token_class</a></tt>
          572  +<li><tt><a href='#token_destructor'>%token_destructor</a></tt>
          573  +<li><tt><a href='#token_prefix'>%token_prefix</a></tt>
          574  +<li><tt><a href='#token_type'>%token_type</a></tt>
          575  +<li><tt><a href='#ptype'>%type</a></tt>
          576  +<li><tt><a href='#pwildcard'>%wildcard</a></tt>
   553    577   </ul>
   554    578   Each of these directives will be described separately in the
   555    579   following sections:</p>
   556    580   
   557    581   <a name='pcode'></a>
   558    582   <h4>The <tt>%code</tt> directive</h4>
   559    583   
   560         -<p>The %code directive is used to specify addition C code that
          584  +<p>The <tt>%code</tt> directive is used to specify additional C code that
   561    585   is added to the end of the main output file.  This is similar to
   562         -the <a href='#pinclude'>%include</a> directive except that %include
   563         -is inserted at the beginning of the main output file.</p>
          586  +the <tt><a href='#pinclude'>%include</a></tt> directive except that
          587  +<tt>%include</tt> is inserted at the beginning of the main output file.</p>
   564    588   
   565         -<p>%code is typically used to include some action routines or perhaps
   566         -a tokenizer or even the "main()" function 
          589  +<p><tt>%code</tt> is typically used to include some action routines or perhaps
          590  +a tokenizer or even the "main()" function
   567    591   as part of the output file.</p>
   568    592   
   569    593   <a name='default_destructor'></a>
   570    594   <h4>The <tt>%default_destructor</tt> directive</h4>
   571    595   
   572         -<p>The %default_destructor directive specifies a destructor to 
          596  +<p>The <tt>%default_destructor</tt> directive specifies a destructor to
   573    597   use for non-terminals that do not have their own destructor
   574         -specified by a separate %destructor directive.  See the documentation
   575         -on the <a name='#destructor'>%destructor</a> directive below for
          598  +specified by a separate <tt>%destructor</tt> directive.  See the documentation
          599  +on the <tt><a name='#destructor'>%destructor</a></tt> directive below for
   576    600   additional information.</p>
   577    601   
   578         -<p>In some grammers, many different non-terminal symbols have the
   579         -same datatype and hence the same destructor.  This directive is
   580         -a convenience way to specify the same destructor for all those
          602  +<p>In some grammars, many different non-terminal symbols have the
          603  +same data type and hence the same destructor.  This directive is
          604  +a convenient way to specify the same destructor for all those
   581    605   non-terminals using a single statement.</p>
   582    606   
   583    607   <a name='default_type'></a>
   584    608   <h4>The <tt>%default_type</tt> directive</h4>
   585    609   
   586         -<p>The %default_type directive specifies the datatype of non-terminal
   587         -symbols that do no have their own datatype defined using a separate
   588         -<a href='#ptype'>%type</a> directive.  
   589         -</p>
          610  +<p>The <tt>%default_type</tt> directive specifies the data type of non-terminal
          611  +symbols that do not have their own data type defined using a separate
          612  +<tt><a href='#ptype'>%type</a></tt> directive.</p>
   590    613   
   591    614   <a name='destructor'></a>
   592    615   <h4>The <tt>%destructor</tt> directive</h4>
   593    616   
   594         -<p>The %destructor directive is used to specify a destructor for
          617  +<p>The <tt>%destructor</tt> directive is used to specify a destructor for
   595    618   a non-terminal symbol.
   596         -(See also the <a href='#token_destructor'>%token_destructor</a>
          619  +(See also the <tt><a href='#token_destructor'>%token_destructor</a></tt>
   597    620   directive which is used to specify a destructor for terminal symbols.)</p>
   598    621   
   599    622   <p>A non-terminal's destructor is called to dispose of the
   600    623   non-terminal's value whenever the non-terminal is popped from
   601    624   the stack.  This includes all of the following circumstances:
   602    625   <ul>
   603    626   <li> When a rule reduces and the value of a non-terminal on
................................................................................
   611    634   
   612    635   <p>Consider an example:
   613    636   <pre>
   614    637      %type nt {void*}
   615    638      %destructor nt { free($$); }
   616    639      nt(A) ::= ID NUM.   { A = malloc( 100 ); }
   617    640   </pre>
   618         -This example is a bit contrived but it serves to illustrate how
          641  +This example is a bit contrived, but it serves to illustrate how
   619    642   destructors work.  The example shows a non-terminal named
   620    643   "nt" that holds values of type "void*".  When the rule for
   621    644   an "nt" reduces, it sets the value of the non-terminal to
   622    645   space obtained from malloc().  Later, when the nt non-terminal
   623    646   is popped from the stack, the destructor will fire and call
   624    647   free() on this malloced space, thus avoiding a memory leak.
   625    648   (Note that the symbol "$$" in the destructor code is replaced
................................................................................
   627    650   
   628    651   <p>It is important to note that the value of a non-terminal is passed
   629    652   to the destructor whenever the non-terminal is removed from the
   630    653   stack, unless the non-terminal is used in a C-code action.  If
   631    654   the non-terminal is used by C-code, then it is assumed that the
   632    655   C-code will take care of destroying it.
   633    656   More commonly, the value is used to build some
   634         -larger structure and we don't want to destroy it, which is why
          657  +larger structure, and we don't want to destroy it, which is why
   635    658   the destructor is not called in this circumstance.</p>
   636    659   
   637    660   <p>Destructors help avoid memory leaks by automatically freeing
   638    661   allocated objects when they go out of scope.
   639    662   To do the same using yacc or bison is much more difficult.</p>
   640    663   
   641         -<a name="extraarg"></a>
          664  +<a name='extraarg'></a>
   642    665   <h4>The <tt>%extra_argument</tt> directive</h4>
   643    666   
   644         -The %extra_argument directive instructs Lemon to add a 4th parameter
          667  +The <tt>%extra_argument</tt> directive instructs Lemon to add a 4th parameter
   645    668   to the parameter list of the Parse() function it generates.  Lemon
   646    669   doesn't do anything itself with this extra argument, but it does
   647    670   make the argument available to C-code action routines, destructors,
   648    671   and so forth.  For example, if the grammar file contains:</p>
   649    672   
   650    673   <p><pre>
   651    674       %extra_argument { MyStruct *pAbc }
................................................................................
   655    678   of type "MyStruct*" and all action routines will have access to
   656    679   a variable named "pAbc" that is the value of the 4th parameter
   657    680   in the most recent call to Parse().</p>
   658    681   
   659    682   <a name='pfallback'></a>
   660    683   <h4>The <tt>%fallback</tt> directive</h4>
   661    684   
   662         -<p>The %fallback directive specifies an alternative meaning for one
          685  +<p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
   663    686   or more tokens.  The alternative meaning is tried if the original token
   664         -would have generated a syntax error.
          687  +would have generated a syntax error.</p>
   665    688   
   666         -<p>The %fallback directive was added to support robust parsing of SQL
   667         -syntax in <a href="https://www.sqlite.org/">SQLite</a>.
          689  +<p>The <tt>%fallback</tt> directive was added to support robust parsing of SQL
          690  +syntax in <a href='https://www.sqlite.org/'>SQLite</a>.
   668    691   The SQL language contains a large assortment of keywords, each of which
   669    692   appears as a different token to the language parser.  SQL contains so
   670         -many keywords, that it can be difficult for programmers to keep up with
          693  +many keywords that it can be difficult for programmers to keep up with
   671    694   them all.  Programmers will, therefore, sometimes mistakenly use an
   672         -obscure language keyword for an identifier.  The %fallback directive
          695  +obscure language keyword for an identifier.  The <tt>%fallback</tt> directive
   673    696   provides a mechanism to tell the parser:  "If you are unable to parse
   674         -this keyword, try treating it as an identifier instead."
          697  +this keyword, try treating it as an identifier instead."</p>
   675    698   
   676         -<p>The syntax of %fallback is as follows:
          699  +<p>The syntax of <tt>%fallback</tt> is as follows:
   677    700   
   678    701   <blockquote>
   679         -<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
   680         -</blockquote>
          702  +<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b>
          703  +</blockquote></p>
   681    704   
   682         -<p>In words, the %fallback directive is followed by a list of token names
   683         -terminated by a period.  The first token name is the fallback token - the
          705  +<p>In words, the <tt>%fallback</tt> directive is followed by a list of token
          706  +names terminated by a period.
          707  +The first token name is the fallback token &mdash; the
   684    708   token to which all the other tokens fall back to.  The second and subsequent
   685    709   arguments are tokens which fall back to the token identified by the first
   686         -argument.
          710  +argument.</p>
   687    711   
   688    712   <a name='pifdef'></a>
   689         -<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
          713  +<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives</h4>
   690    714   
   691         -<p>The %ifdef, %ifndef, and %endif directives are similar to
   692         -#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
          715  +<p>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives
          716  +are similar to #ifdef, #ifndef, and #endif in the C-preprocessor,
          717  +just not as general.
   693    718   Each of these directives must begin at the left margin.  No whitespace
   694         -is allowed between the "%" and the directive name.
          719  +is allowed between the "%" and the directive name.</p>
   695    720   
   696         -<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
          721  +<p>Grammar text in between "<tt>%ifdef MACRO</tt>" and the next nested
          722  +"<tt>%endif</tt>" is
   697    723   ignored unless the "-DMACRO" command-line option is used.  Grammar text
   698         -betwen "%ifndef MACRO" and the next nested "%endif" is included except when
   699         -the "-DMACRO" command-line option is used.
          724  +betwen "<tt>%ifndef MACRO</tt>" and the next nested "<tt>%endif</tt>" is
          725  +included except when the "-DMACRO" command-line option is used.</p>
   700    726   
   701         -<p>Note that the argument to %ifdef and %ifndef must be a single 
   702         -preprocessor symbol name, not a general expression.  There is no "%else"
   703         -directive.
          727  +<p>Note that the argument to <tt>%ifdef</tt> and <tt>%ifndef</tt> must
          728  +be a single preprocessor symbol name, not a general expression.
          729  +There is no "<tt>%else</tt>" directive.</p>
   704    730   
   705    731   
   706    732   <a name='pinclude'></a>
   707    733   <h4>The <tt>%include</tt> directive</h4>
   708    734   
   709         -<p>The %include directive specifies C code that is included at the
   710         -top of the generated parser.  You can include any text you want --
          735  +<p>The <tt>%include</tt> directive specifies C code that is included at the
          736  +top of the generated parser.  You can include any text you want &mdash;
   711    737   the Lemon parser generator copies it blindly.  If you have multiple
   712         -%include directives in your grammar file, their values are concatenated
   713         -so that all %include code ultimately appears near the top of the
   714         -generated parser, in the same order as it appeared in the grammer.</p>
          738  +<tt>%include</tt> directives in your grammar file, their values are concatenated
          739  +so that all <tt>%include</tt> code ultimately appears near the top of the
          740  +generated parser, in the same order as it appeared in the grammar.</p>
   715    741   
   716         -<p>The %include directive is very handy for getting some extra #include
          742  +<p>The <tt>%include</tt> directive is very handy for getting some extra #include
   717    743   preprocessor statements at the beginning of the generated parser.
   718    744   For example:</p>
   719    745   
   720    746   <p><pre>
   721    747      %include {#include &lt;unistd.h&gt;}
   722    748   </pre></p>
   723    749   
   724    750   <p>This might be needed, for example, if some of the C actions in the
   725         -grammar call functions that are prototyed in unistd.h.</p>
          751  +grammar call functions that are prototyped in unistd.h.</p>
   726    752   
   727    753   <a name='pleft'></a>
   728    754   <h4>The <tt>%left</tt> directive</h4>
   729    755   
   730         -The %left directive is used (along with the <a href='#pright'>%right</a> and
   731         -<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
   732         -terminal symbols.  Every terminal symbol whose name appears after
   733         -a %left directive but before the next period (".") is
          756  +The <tt>%left</tt> directive is used (along with the
          757  +<tt><a href='#pright'>%right</a></tt> and
          758  +<tt><a href='#pnonassoc'>%nonassoc</a></tt> directives) to declare
          759  +precedences of terminal symbols.
          760  +Every terminal symbol whose name appears after
          761  +a <tt>%left</tt> directive but before the next period (".") is
   734    762   given the same left-associative precedence value.  Subsequent
   735         -%left directives have higher precedence.  For example:</p>
          763  +<tt>%left</tt> directives have higher precedence.  For example:</p>
   736    764   
   737    765   <p><pre>
   738    766      %left AND.
   739    767      %left OR.
   740    768      %nonassoc EQ NE GT GE LT LE.
   741    769      %left PLUS MINUS.
   742    770      %left TIMES DIVIDE MOD.
   743    771      %right EXP NOT.
   744    772   </pre></p>
   745    773   
   746         -<p>Note the period that terminates each %left, %right or %nonassoc
          774  +<p>Note the period that terminates each <tt>%left</tt>,
          775  +<tt>%right</tt> or <tt>%nonassoc</tt>
   747    776   directive.</p>
   748    777   
   749    778   <p>LALR(1) grammars can get into a situation where they require
   750    779   a large amount of stack space if you make heavy use or right-associative
   751         -operators.  For this reason, it is recommended that you use %left
   752         -rather than %right whenever possible.</p>
          780  +operators.  For this reason, it is recommended that you use <tt>%left</tt>
          781  +rather than <tt>%right</tt> whenever possible.</p>
   753    782   
   754    783   <a name='pname'></a>
   755    784   <h4>The <tt>%name</tt> directive</h4>
   756    785   
   757    786   <p>By default, the functions generated by Lemon all begin with the
   758    787   five-character string "Parse".  You can change this string to something
   759         -different using the %name directive.  For instance:</p>
          788  +different using the <tt>%name</tt> directive.  For instance:</p>
   760    789   
   761    790   <p><pre>
   762    791      %name Abcde
   763    792   </pre></p>
   764    793   
   765    794   <p>Putting this directive in the grammar file will cause Lemon to generate
   766    795   functions named
   767    796   <ul>
   768    797   <li> AbcdeAlloc(),
   769    798   <li> AbcdeFree(),
   770    799   <li> AbcdeTrace(), and
   771    800   <li> Abcde().
   772    801   </ul>
   773         -The %name directive allows you to generator two or more different
   774         -parsers and link them all into the same executable.
   775         -</p>
          802  +The <tt>%name</tt> directive allows you to generate two or more different
          803  +parsers and link them all into the same executable.</p>
   776    804   
   777    805   <a name='pnonassoc'></a>
   778    806   <h4>The <tt>%nonassoc</tt> directive</h4>
   779    807   
   780    808   <p>This directive is used to assign non-associative precedence to
   781         -one or more terminal symbols.  See the section on 
          809  +one or more terminal symbols.  See the section on
   782    810   <a href='#precrules'>precedence rules</a>
   783         -or on the <a href='#pleft'>%left</a> directive for additional information.</p>
          811  +or on the <tt><a href='#pleft'>%left</a></tt> directive
          812  +for additional information.</p>
   784    813   
   785    814   <a name='parse_accept'></a>
   786    815   <h4>The <tt>%parse_accept</tt> directive</h4>
   787    816   
   788         -<p>The %parse_accept directive specifies a block of C code that is
          817  +<p>The <tt>%parse_accept</tt> directive specifies a block of C code that is
   789    818   executed whenever the parser accepts its input string.  To "accept"
   790    819   an input string means that the parser was able to process all tokens
   791    820   without error.</p>
   792    821   
   793    822   <p>For example:</p>
   794    823   
   795    824   <p><pre>
................................................................................
   797    826         printf("parsing complete!\n");
   798    827      }
   799    828   </pre></p>
   800    829   
   801    830   <a name='parse_failure'></a>
   802    831   <h4>The <tt>%parse_failure</tt> directive</h4>
   803    832   
   804         -<p>The %parse_failure directive specifies a block of C code that
          833  +<p>The <tt>%parse_failure</tt> directive specifies a block of C code that
   805    834   is executed whenever the parser fails complete.  This code is not
   806    835   executed until the parser has tried and failed to resolve an input
   807    836   error using is usual error recovery strategy.  The routine is
   808    837   only invoked when parsing is unable to continue.</p>
   809    838   
   810    839   <p><pre>
   811    840      %parse_failure {
................................................................................
   813    842      }
   814    843   </pre></p>
   815    844   
   816    845   <a name='pright'></a>
   817    846   <h4>The <tt>%right</tt> directive</h4>
   818    847   
   819    848   <p>This directive is used to assign right-associative precedence to
   820         -one or more terminal symbols.  See the section on 
          849  +one or more terminal symbols.  See the section on
   821    850   <a href='#precrules'>precedence rules</a>
   822    851   or on the <a href='#pleft'>%left</a> directive for additional information.</p>
   823    852   
   824    853   <a name='stack_overflow'></a>
   825    854   <h4>The <tt>%stack_overflow</tt> directive</h4>
   826    855   
   827         -<p>The %stack_overflow directive specifies a block of C code that
          856  +<p>The <tt>%stack_overflow</tt> directive specifies a block of C code that
   828    857   is executed if the parser's internal stack ever overflows.  Typically
   829    858   this just prints an error message.  After a stack overflow, the parser
   830    859   will be unable to continue and must be reset.</p>
   831    860   
   832    861   <p><pre>
   833    862      %stack_overflow {
   834    863        fprintf(stderr,"Giving up.  Parser stack overflow\n");
   835    864      }
   836    865   </pre></p>
   837    866   
   838    867   <p>You can help prevent parser stack overflows by avoiding the use
   839    868   of right recursion and right-precedence operators in your grammar.
   840         -Use left recursion and and left-precedence operators instead, to
          869  +Use left recursion and and left-precedence operators instead to
   841    870   encourage rules to reduce sooner and keep the stack size down.
   842    871   For example, do rules like this:
   843    872   <pre>
   844    873      list ::= list element.      // left-recursion.  Good!
   845    874      list ::= .
   846    875   </pre>
   847    876   Not like this:
   848    877   <pre>
   849    878      list ::= element list.      // right-recursion.  Bad!
   850    879      list ::= .
   851         -</pre>
          880  +</pre></p>
   852    881   
   853    882   <a name='stack_size'></a>
   854    883   <h4>The <tt>%stack_size</tt> directive</h4>
   855    884   
   856    885   <p>If stack overflow is a problem and you can't resolve the trouble
   857    886   by using left-recursion, then you might want to increase the size
   858    887   of the parser's stack using this directive.  Put an positive integer
   859         -after the %stack_size directive and Lemon will generate a parse
          888  +after the <tt>%stack_size</tt> directive and Lemon will generate a parse
   860    889   with a stack of the requested size.  The default value is 100.</p>
   861    890   
   862    891   <p><pre>
   863    892      %stack_size 2000
   864    893   </pre></p>
   865    894   
   866    895   <a name='start_symbol'></a>
   867    896   <h4>The <tt>%start_symbol</tt> directive</h4>
   868    897   
   869         -<p>By default, the start-symbol for the grammar that Lemon generates
          898  +<p>By default, the start symbol for the grammar that Lemon generates
   870    899   is the first non-terminal that appears in the grammar file.  But you
   871         -can choose a different start-symbol using the %start_symbol directive.</p>
          900  +can choose a different start symbol using the
          901  +<tt>%start_symbol</tt> directive.</p>
   872    902   
   873    903   <p><pre>
   874    904      %start_symbol  prog
   875    905   </pre></p>
   876    906   
          907  +<a name='syntax_error'></a>
          908  +<h4>The <tt>%syntax_error</tt> directive</h4>
          909  +
          910  +<p>See <a href='#error_processing'>Error Processing</a>.</p>
          911  +
          912  +<a name='token_class'></a>
          913  +<h4>The <tt>%token_class</tt> directive</h4>
          914  +
          915  +<p>Undocumented.  Appears to be related to the MULTITERMINAL concept.
          916  +<a href='http://sqlite.org/src/fdiff?v1=796930d5fc2036c7&v2=624b24c5dc048e09&sbs=0'>Implementation</a>.</p>
          917  +
   877    918   <a name='token_destructor'></a>
   878    919   <h4>The <tt>%token_destructor</tt> directive</h4>
   879    920   
   880         -<p>The %destructor directive assigns a destructor to a non-terminal
   881         -symbol.  (See the description of the %destructor directive above.)
   882         -This directive does the same thing for all terminal symbols.</p>
          921  +<p>The <tt>%destructor</tt> directive assigns a destructor to a non-terminal
          922  +symbol.  (See the description of the
          923  +<tt><a href='%destructor'>%destructor</a></tt> directive above.)
          924  +The <tt>%token_destructor</tt> directive does the same thing
          925  +for all terminal symbols.</p>
   883    926   
   884    927   <p>Unlike non-terminal symbols which may each have a different data type
   885    928   for their values, terminals all use the same data type (defined by
   886         -the %token_type directive) and so they use a common destructor.  Other
   887         -than that, the token destructor works just like the non-terminal
          929  +the <tt><a href='#token_type'>%token_type</a></tt> directive)
          930  +and so they use a common destructor.
          931  +Other than that, the token destructor works just like the non-terminal
   888    932   destructors.</p>
   889    933   
   890    934   <a name='token_prefix'></a>
   891    935   <h4>The <tt>%token_prefix</tt> directive</h4>
   892    936   
   893    937   <p>Lemon generates #defines that assign small integer constants
   894    938   to each terminal symbol in the grammar.  If desired, Lemon will
   895    939   add a prefix specified by this directive
   896         -to each of the #defines it generates.
   897         -So if the default output of Lemon looked like this:
          940  +to each of the #defines it generates.</p>
          941  +
          942  +<p>So if the default output of Lemon looked like this:
   898    943   <pre>
   899    944       #define AND              1
   900    945       #define MINUS            2
   901    946       #define OR               3
   902    947       #define PLUS             4
   903    948   </pre>
   904    949   You can insert a statement into the grammar like this:
................................................................................
   907    952   </pre>
   908    953   to cause Lemon to produce these symbols instead:
   909    954   <pre>
   910    955       #define TOKEN_AND        1
   911    956       #define TOKEN_MINUS      2
   912    957       #define TOKEN_OR         3
   913    958       #define TOKEN_PLUS       4
   914         -</pre>
          959  +</pre></p>
   915    960   
   916    961   <a name='token_type'></a><a name='ptype'></a>
   917    962   <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
   918    963   
   919    964   <p>These directives are used to specify the data types for values
   920    965   on the parser's stack associated with terminal and non-terminal
   921    966   symbols.  The values of all terminal symbols must be of the same
................................................................................
   928    973      %token_type    {Token*}
   929    974   </pre></p>
   930    975   
   931    976   <p>If the data type of terminals is not specified, the default value
   932    977   is "void*".</p>
   933    978   
   934    979   <p>Non-terminal symbols can each have their own data types.  Typically
   935         -the data type  of a non-terminal is a pointer to the root of a parse-tree
          980  +the data type of a non-terminal is a pointer to the root of a parse tree
   936    981   structure that contains all information about that non-terminal.
   937    982   For example:</p>
   938    983   
   939    984   <p><pre>
   940    985      %type   expr  {Expr*}
   941    986   </pre></p>
   942    987   
................................................................................
   949    994   non-terminal whose data type requires 1K of storage, then your 100
   950    995   entry parser stack will require 100K of heap space.  If you are willing
   951    996   and able to pay that price, fine.  You just need to know.</p>
   952    997   
   953    998   <a name='pwildcard'></a>
   954    999   <h4>The <tt>%wildcard</tt> directive</h4>
   955   1000   
   956         -<p>The %wildcard directive is followed by a single token name and a
   957         -period.  This directive specifies that the identified token should 
   958         -match any input token.
         1001  +<p>The <tt>%wildcard</tt> directive is followed by a single token name and a
         1002  +period.  This directive specifies that the identified token should
         1003  +match any input token.</p>
   959   1004   
   960   1005   <p>When the generated parser has the choice of matching an input against
   961   1006   the wildcard token and some other token, the other token is always used.
   962         -The wildcard token is only matched if there are no other alternatives.
         1007  +The wildcard token is only matched if there are no alternatives.</p>
   963   1008   
         1009  +<a name='error_processing'></a>
   964   1010   <h3>Error Processing</h3>
   965   1011   
   966   1012   <p>After extensive experimentation over several years, it has been
   967   1013   discovered that the error recovery strategy used by yacc is about
   968   1014   as good as it gets.  And so that is what Lemon uses.</p>
   969   1015   
   970   1016   <p>When a Lemon-generated parser encounters a syntax error, it
   971         -first invokes the code specified by the %syntax_error directive, if
         1017  +first invokes the code specified by the <tt>%syntax_error</tt> directive, if
   972   1018   any.  It then enters its error recovery strategy.  The error recovery
   973   1019   strategy is to begin popping the parsers stack until it enters a
   974   1020   state where it is permitted to shift a special non-terminal symbol
   975   1021   named "error".  It then shifts this non-terminal and continues
   976         -parsing.  But the %syntax_error routine will not be called again
         1022  +parsing.  The <tt>%syntax_error</tt> routine will not be called again
   977   1023   until at least three new tokens have been successfully shifted.</p>
   978   1024   
   979   1025   <p>If the parser pops its stack until the stack is empty, and it still
   980         -is unable to shift the error symbol, then the %parse_failed routine
         1026  +is unable to shift the error symbol, then the
         1027  +<tt><a href='#parse_failure'>%parse_failure</a></tt> routine
   981   1028   is invoked and the parser resets itself to its start state, ready
   982   1029   to begin parsing a new file.  This is what will happen at the very
   983         -first syntax error, of course, if there are no instances of the 
         1030  +first syntax error, of course, if there are no instances of the
   984   1031   "error" non-terminal in your grammar.</p>
   985   1032   
   986   1033   </body>
   987   1034   </html>

Added ext/README.md.

            1  +## Loadable Extensions
            2  +
            3  +Various [loadable extensions](https://www.sqlite.org/loadext.html) for
            4  +SQLite are found in subfolders.
            5  +
            6  +Most subfolders are dedicated to a single loadable extension (for
            7  +example FTS5, or RTREE).  But the misc/ subfolder contains a collection
            8  +of smaller single-file extensions.

Deleted ext/README.txt.

     1         -Version loadable extensions to SQLite are found in subfolders
     2         -of this folder.

Changes to ext/fts3/fts3.c.

   345    345     if( (v & mask2)==0 ){ var = v; return ret; }
   346    346   
   347    347   /* 
   348    348   ** Read a 64-bit variable-length integer from memory starting at p[0].
   349    349   ** Return the number of bytes read, or 0 on error.
   350    350   ** The value is stored in *v.
   351    351   */
   352         -int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
   353         -  const char *pStart = p;
          352  +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
          353  +  const unsigned char *p = (const unsigned char*)pBuf;
          354  +  const unsigned char *pStart = p;
   354    355     u32 a;
   355    356     u64 b;
   356    357     int shift;
   357    358   
   358    359     GETVARINT_INIT(a, p, 0,  0x00,     0x80, *v, 1);
   359    360     GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *v, 2);
   360    361     GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *v, 3);
................................................................................
   367    368       if( (c & 0x80)==0 ) break;
   368    369     }
   369    370     *v = b;
   370    371     return (int)(p - pStart);
   371    372   }
   372    373   
   373    374   /*
   374         -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
   375         -** 32-bit integer before it is returned.
          375  +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to 
          376  +** a non-negative 32-bit integer before it is returned.
   376    377   */
   377    378   int sqlite3Fts3GetVarint32(const char *p, int *pi){
   378    379     u32 a;
   379    380   
   380    381   #ifndef fts3GetVarint32
   381    382     GETVARINT_INIT(a, p, 0,  0x00,     0x80, *pi, 1);
   382    383   #else
................................................................................
   384    385     assert( a & 0x80 );
   385    386   #endif
   386    387   
   387    388     GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *pi, 2);
   388    389     GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
   389    390     GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
   390    391     a = (a & 0x0FFFFFFF );
   391         -  *pi = (int)(a | ((u32)(*p & 0x0F) << 28));
          392  +  *pi = (int)(a | ((u32)(*p & 0x07) << 28));
          393  +  assert( 0==(a & 0x80000000) );
          394  +  assert( *pi>=0 );
   392    395     return 5;
   393    396   }
   394    397   
   395    398   /*
   396    399   ** Return the number of bytes required to encode v as a varint
   397    400   */
   398    401   int sqlite3Fts3VarintLen(sqlite3_uint64 v){
................................................................................
   488    491     Fts3Table *p = (Fts3Table *)pVtab;
   489    492     int i;
   490    493   
   491    494     assert( p->nPendingData==0 );
   492    495     assert( p->pSegments==0 );
   493    496   
   494    497     /* Free any prepared statements held */
          498  +  sqlite3_finalize(p->pSeekStmt);
   495    499     for(i=0; i<SizeofArray(p->aStmt); i++){
   496    500       sqlite3_finalize(p->aStmt[i]);
   497    501     }
   498    502     sqlite3_free(p->zSegmentsTbl);
   499    503     sqlite3_free(p->zReadExprlist);
   500    504     sqlite3_free(p->zWriteExprlist);
   501    505     sqlite3_free(p->zContentTbl);
................................................................................
  1213   1217         }else{
  1214   1218           for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
  1215   1219             struct Fts4Option *pOp = &aFts4Opt[iOpt];
  1216   1220             if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
  1217   1221               break;
  1218   1222             }
  1219   1223           }
  1220         -        if( iOpt==SizeofArray(aFts4Opt) ){
  1221         -          sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
  1222         -          rc = SQLITE_ERROR;
  1223         -        }else{
  1224         -          switch( iOpt ){
  1225         -            case 0:               /* MATCHINFO */
  1226         -              if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
  1227         -                sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
  1228         -                rc = SQLITE_ERROR;
  1229         -              }
  1230         -              bNoDocsize = 1;
  1231         -              break;
  1232         -
  1233         -            case 1:               /* PREFIX */
  1234         -              sqlite3_free(zPrefix);
  1235         -              zPrefix = zVal;
  1236         -              zVal = 0;
  1237         -              break;
  1238         -
  1239         -            case 2:               /* COMPRESS */
  1240         -              sqlite3_free(zCompress);
  1241         -              zCompress = zVal;
  1242         -              zVal = 0;
  1243         -              break;
  1244         -
  1245         -            case 3:               /* UNCOMPRESS */
  1246         -              sqlite3_free(zUncompress);
  1247         -              zUncompress = zVal;
  1248         -              zVal = 0;
  1249         -              break;
  1250         -
  1251         -            case 4:               /* ORDER */
  1252         -              if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
  1253         -               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
  1254         -              ){
  1255         -                sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
  1256         -                rc = SQLITE_ERROR;
  1257         -              }
  1258         -              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
  1259         -              break;
  1260         -
  1261         -            case 5:              /* CONTENT */
  1262         -              sqlite3_free(zContent);
  1263         -              zContent = zVal;
  1264         -              zVal = 0;
  1265         -              break;
  1266         -
  1267         -            case 6:              /* LANGUAGEID */
  1268         -              assert( iOpt==6 );
  1269         -              sqlite3_free(zLanguageid);
  1270         -              zLanguageid = zVal;
  1271         -              zVal = 0;
  1272         -              break;
  1273         -
  1274         -            case 7:              /* NOTINDEXED */
  1275         -              azNotindexed[nNotindexed++] = zVal;
  1276         -              zVal = 0;
  1277         -              break;
  1278         -          }
         1224  +        switch( iOpt ){
         1225  +          case 0:               /* MATCHINFO */
         1226  +            if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
         1227  +              sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
         1228  +              rc = SQLITE_ERROR;
         1229  +            }
         1230  +            bNoDocsize = 1;
         1231  +            break;
         1232  +
         1233  +          case 1:               /* PREFIX */
         1234  +            sqlite3_free(zPrefix);
         1235  +            zPrefix = zVal;
         1236  +            zVal = 0;
         1237  +            break;
         1238  +
         1239  +          case 2:               /* COMPRESS */
         1240  +            sqlite3_free(zCompress);
         1241  +            zCompress = zVal;
         1242  +            zVal = 0;
         1243  +            break;
         1244  +
         1245  +          case 3:               /* UNCOMPRESS */
         1246  +            sqlite3_free(zUncompress);
         1247  +            zUncompress = zVal;
         1248  +            zVal = 0;
         1249  +            break;
         1250  +
         1251  +          case 4:               /* ORDER */
         1252  +            if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
         1253  +             && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
         1254  +            ){
         1255  +              sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
         1256  +              rc = SQLITE_ERROR;
         1257  +            }
         1258  +            bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
         1259  +            break;
         1260  +
         1261  +          case 5:              /* CONTENT */
         1262  +            sqlite3_free(zContent);
         1263  +            zContent = zVal;
         1264  +            zVal = 0;
         1265  +            break;
         1266  +
         1267  +          case 6:              /* LANGUAGEID */
         1268  +            assert( iOpt==6 );
         1269  +            sqlite3_free(zLanguageid);
         1270  +            zLanguageid = zVal;
         1271  +            zVal = 0;
         1272  +            break;
         1273  +
         1274  +          case 7:              /* NOTINDEXED */
         1275  +            azNotindexed[nNotindexed++] = zVal;
         1276  +            zVal = 0;
         1277  +            break;
         1278  +
         1279  +          default:
         1280  +            assert( iOpt==SizeofArray(aFts4Opt) );
         1281  +            sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
         1282  +            rc = SQLITE_ERROR;
         1283  +            break;
  1279   1284           }
  1280   1285           sqlite3_free(zVal);
  1281   1286         }
  1282   1287       }
  1283   1288   
  1284   1289       /* Otherwise, the argument is a column name. */
  1285   1290       else {
................................................................................
  1359   1364     p->db = db;
  1360   1365     p->nColumn = nCol;
  1361   1366     p->nPendingData = 0;
  1362   1367     p->azColumn = (char **)&p[1];
  1363   1368     p->pTokenizer = pTokenizer;
  1364   1369     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  1365   1370     p->bHasDocsize = (isFts4 && bNoDocsize==0);
  1366         -  p->bHasStat = isFts4;
  1367         -  p->bFts4 = isFts4;
  1368         -  p->bDescIdx = bDescIdx;
         1371  +  p->bHasStat = (u8)isFts4;
         1372  +  p->bFts4 = (u8)isFts4;
         1373  +  p->bDescIdx = (u8)bDescIdx;
  1369   1374     p->nAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  1370   1375     p->zContentTbl = zContent;
  1371   1376     p->zLanguageid = zLanguageid;
  1372   1377     zContent = 0;
  1373   1378     zLanguageid = 0;
  1374   1379     TESTONLY( p->inTransaction = -1 );
  1375   1380     TESTONLY( p->mxSavepoint = -1 );
................................................................................
  1392   1397     zCsr += nDb;
  1393   1398   
  1394   1399     /* Fill in the azColumn array */
  1395   1400     for(iCol=0; iCol<nCol; iCol++){
  1396   1401       char *z; 
  1397   1402       int n = 0;
  1398   1403       z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
  1399         -    memcpy(zCsr, z, n);
         1404  +    if( n>0 ){
         1405  +      memcpy(zCsr, z, n);
         1406  +    }
  1400   1407       zCsr[n] = '\0';
  1401   1408       sqlite3Fts3Dequote(zCsr);
  1402   1409       p->azColumn[iCol] = zCsr;
  1403   1410       zCsr += n+1;
  1404   1411       assert( zCsr <= &((char *)p)[nByte] );
  1405   1412     }
  1406   1413   
................................................................................
  1675   1682     *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor));
  1676   1683     if( !pCsr ){
  1677   1684       return SQLITE_NOMEM;
  1678   1685     }
  1679   1686     memset(pCsr, 0, sizeof(Fts3Cursor));
  1680   1687     return SQLITE_OK;
  1681   1688   }
         1689  +
         1690  +/*
         1691  +** Finalize the statement handle at pCsr->pStmt.
         1692  +**
         1693  +** Or, if that statement handle is one created by fts3CursorSeekStmt(),
         1694  +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement
         1695  +** pointer there instead of finalizing it.
         1696  +*/
         1697  +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){
         1698  +  if( pCsr->bSeekStmt ){
         1699  +    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         1700  +    if( p->pSeekStmt==0 ){
         1701  +      p->pSeekStmt = pCsr->pStmt;
         1702  +      sqlite3_reset(pCsr->pStmt);
         1703  +      pCsr->pStmt = 0;
         1704  +    }
         1705  +    pCsr->bSeekStmt = 0;
         1706  +  }
         1707  +  sqlite3_finalize(pCsr->pStmt);
         1708  +}
         1709  +
         1710  +/*
         1711  +** Free all resources currently held by the cursor passed as the only
         1712  +** argument.
         1713  +*/
         1714  +static void fts3ClearCursor(Fts3Cursor *pCsr){
         1715  +  fts3CursorFinalizeStmt(pCsr);
         1716  +  sqlite3Fts3FreeDeferredTokens(pCsr);
         1717  +  sqlite3_free(pCsr->aDoclist);
         1718  +  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
         1719  +  sqlite3Fts3ExprFree(pCsr->pExpr);
         1720  +  memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
         1721  +}
  1682   1722   
  1683   1723   /*
  1684   1724   ** Close the cursor.  For additional information see the documentation
  1685   1725   ** on the xClose method of the virtual table interface.
  1686   1726   */
  1687   1727   static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
  1688   1728     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  1689   1729     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1690         -  sqlite3_finalize(pCsr->pStmt);
  1691         -  sqlite3Fts3ExprFree(pCsr->pExpr);
  1692         -  sqlite3Fts3FreeDeferredTokens(pCsr);
  1693         -  sqlite3_free(pCsr->aDoclist);
  1694         -  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
         1730  +  fts3ClearCursor(pCsr);
  1695   1731     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1696   1732     sqlite3_free(pCsr);
  1697   1733     return SQLITE_OK;
  1698   1734   }
  1699   1735   
  1700   1736   /*
  1701   1737   ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
  1702   1738   ** compose and prepare an SQL statement of the form:
  1703   1739   **
  1704   1740   **    "SELECT <columns> FROM %_content WHERE rowid = ?"
  1705   1741   **
  1706   1742   ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
  1707   1743   ** it. If an error occurs, return an SQLite error code.
  1708         -**
  1709         -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
  1710   1744   */
  1711         -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
         1745  +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){
  1712   1746     int rc = SQLITE_OK;
  1713   1747     if( pCsr->pStmt==0 ){
  1714   1748       Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  1715   1749       char *zSql;
  1716         -    zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
  1717         -    if( !zSql ) return SQLITE_NOMEM;
  1718         -    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  1719         -    sqlite3_free(zSql);
         1750  +    if( p->pSeekStmt ){
         1751  +      pCsr->pStmt = p->pSeekStmt;
         1752  +      p->pSeekStmt = 0;
         1753  +    }else{
         1754  +      zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
         1755  +      if( !zSql ) return SQLITE_NOMEM;
         1756  +      rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
         1757  +      sqlite3_free(zSql);
         1758  +    }
         1759  +    if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
  1720   1760     }
  1721         -  *ppStmt = pCsr->pStmt;
  1722   1761     return rc;
  1723   1762   }
  1724   1763   
  1725   1764   /*
  1726   1765   ** Position the pCsr->pStmt statement so that it is on the row
  1727   1766   ** of the %_content table that contains the last match.  Return
  1728   1767   ** SQLITE_OK on success.  
  1729   1768   */
  1730   1769   static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
  1731   1770     int rc = SQLITE_OK;
  1732   1771     if( pCsr->isRequireSeek ){
  1733         -    sqlite3_stmt *pStmt = 0;
  1734         -
  1735         -    rc = fts3CursorSeekStmt(pCsr, &pStmt);
         1772  +    rc = fts3CursorSeekStmt(pCsr);
  1736   1773       if( rc==SQLITE_OK ){
  1737   1774         sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
  1738   1775         pCsr->isRequireSeek = 0;
  1739   1776         if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
  1740   1777           return SQLITE_OK;
  1741   1778         }else{
  1742   1779           rc = sqlite3_reset(pCsr->pStmt);
................................................................................
  1817   1854       ** the size of zBuffer if required.  */
  1818   1855       if( !isFirstTerm ){
  1819   1856         zCsr += fts3GetVarint32(zCsr, &nPrefix);
  1820   1857       }
  1821   1858       isFirstTerm = 0;
  1822   1859       zCsr += fts3GetVarint32(zCsr, &nSuffix);
  1823   1860       
  1824         -    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
         1861  +    assert( nPrefix>=0 && nSuffix>=0 );
         1862  +    if( &zCsr[nSuffix]>zEnd ){
  1825   1863         rc = FTS_CORRUPT_VTAB;
  1826   1864         goto finish_scan;
  1827   1865       }
  1828   1866       if( nPrefix+nSuffix>nAlloc ){
  1829   1867         char *zNew;
  1830   1868         nAlloc = (nPrefix+nSuffix) * 2;
  1831   1869         zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
................................................................................
  2627   2665         nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
  2628   2666         pOut[nOut++] = 0x02;
  2629   2667         bWritten = 1;
  2630   2668       }
  2631   2669       fts3ColumnlistCopy(0, &p);
  2632   2670     }
  2633   2671   
  2634         -  while( p<pEnd && *p==0x01 ){
         2672  +  while( p<pEnd ){
  2635   2673       sqlite3_int64 iCol;
  2636   2674       p++;
  2637   2675       p += sqlite3Fts3GetVarint(p, &iCol);
  2638   2676       if( *p==0x02 ){
  2639   2677         if( bWritten==0 ){
  2640   2678           nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
  2641   2679           bWritten = 1;
................................................................................
  3186   3224     if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++];
  3187   3225     if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++];
  3188   3226     if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
  3189   3227     if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
  3190   3228     assert( iIdx==nVal );
  3191   3229   
  3192   3230     /* In case the cursor has been used before, clear it now. */
  3193         -  sqlite3_finalize(pCsr->pStmt);
  3194         -  sqlite3_free(pCsr->aDoclist);
  3195         -  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
  3196         -  sqlite3Fts3ExprFree(pCsr->pExpr);
  3197         -  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
         3231  +  fts3ClearCursor(pCsr);
  3198   3232   
  3199   3233     /* Set the lower and upper bounds on docids to return */
  3200   3234     pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
  3201   3235     pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64);
  3202   3236   
  3203   3237     if( idxStr ){
  3204   3238       pCsr->bDesc = (idxStr[0]=='D');
................................................................................
  3248   3282         );
  3249   3283       }else{
  3250   3284         zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", 
  3251   3285             p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC")
  3252   3286         );
  3253   3287       }
  3254   3288       if( zSql ){
  3255         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         3289  +      rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
  3256   3290         sqlite3_free(zSql);
  3257   3291       }else{
  3258   3292         rc = SQLITE_NOMEM;
  3259   3293       }
  3260   3294     }else if( eSearch==FTS3_DOCID_SEARCH ){
  3261         -    rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
         3295  +    rc = fts3CursorSeekStmt(pCsr);
  3262   3296       if( rc==SQLITE_OK ){
  3263   3297         rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons);
  3264   3298       }
  3265   3299     }
  3266   3300     if( rc!=SQLITE_OK ) return rc;
  3267   3301   
  3268   3302     return fts3NextMethod(pCursor);
................................................................................
  3269   3303   }
  3270   3304   
  3271   3305   /* 
  3272   3306   ** This is the xEof method of the virtual table. SQLite calls this 
  3273   3307   ** routine to find out if it has reached the end of a result set.
  3274   3308   */
  3275   3309   static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
  3276         -  return ((Fts3Cursor *)pCursor)->isEof;
         3310  +  Fts3Cursor *pCsr = (Fts3Cursor*)pCursor;
         3311  +  if( pCsr->isEof ){
         3312  +    fts3ClearCursor(pCsr);
         3313  +    pCsr->isEof = 1;
         3314  +  }
         3315  +  return pCsr->isEof;
  3277   3316   }
  3278   3317   
  3279   3318   /* 
  3280   3319   ** This is the xRowid method. The SQLite core calls this routine to
  3281   3320   ** retrieve the rowid for the current row of the result set. fts3
  3282   3321   ** exposes %_content.docid as the rowid for the virtual table. The
  3283   3322   ** rowid should be written to *pRowid.
................................................................................
  3307   3346     int rc = SQLITE_OK;             /* Return Code */
  3308   3347     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  3309   3348     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  3310   3349   
  3311   3350     /* The column value supplied by SQLite must be in range. */
  3312   3351     assert( iCol>=0 && iCol<=p->nColumn+2 );
  3313   3352   
  3314         -  if( iCol==p->nColumn+1 ){
  3315         -    /* This call is a request for the "docid" column. Since "docid" is an 
  3316         -    ** alias for "rowid", use the xRowid() method to obtain the value.
  3317         -    */
  3318         -    sqlite3_result_int64(pCtx, pCsr->iPrevId);
  3319         -  }else if( iCol==p->nColumn ){
  3320         -    /* The extra column whose name is the same as the table.
  3321         -    ** Return a blob which is a pointer to the cursor.  */
  3322         -    sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  3323         -  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
  3324         -    sqlite3_result_int64(pCtx, pCsr->iLangid);
  3325         -  }else{
  3326         -    /* The requested column is either a user column (one that contains 
  3327         -    ** indexed data), or the language-id column.  */
  3328         -    rc = fts3CursorSeek(0, pCsr);
  3329         -
  3330         -    if( rc==SQLITE_OK ){
  3331         -      if( iCol==p->nColumn+2 ){
  3332         -        int iLangid = 0;
  3333         -        if( p->zLanguageid ){
  3334         -          iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1);
  3335         -        }
  3336         -        sqlite3_result_int(pCtx, iLangid);
  3337         -      }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
         3353  +  switch( iCol-p->nColumn ){
         3354  +    case 0:
         3355  +      /* The special 'table-name' column */
         3356  +      sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0);
         3357  +      break;
         3358  +
         3359  +    case 1:
         3360  +      /* The docid column */
         3361  +      sqlite3_result_int64(pCtx, pCsr->iPrevId);
         3362  +      break;
         3363  +
         3364  +    case 2:
         3365  +      if( pCsr->pExpr ){
         3366  +        sqlite3_result_int64(pCtx, pCsr->iLangid);
         3367  +        break;
         3368  +      }else if( p->zLanguageid==0 ){
         3369  +        sqlite3_result_int(pCtx, 0);
         3370  +        break;
         3371  +      }else{
         3372  +        iCol = p->nColumn;
         3373  +        /* fall-through */
         3374  +      }
         3375  +
         3376  +    default:
         3377  +      /* A user column. Or, if this is a full-table scan, possibly the
         3378  +      ** language-id column. Seek the cursor. */
         3379  +      rc = fts3CursorSeek(0, pCsr);
         3380  +      if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){
  3338   3381           sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
  3339   3382         }
  3340         -    }
         3383  +      break;
  3341   3384     }
  3342   3385   
  3343   3386     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  3344   3387     return rc;
  3345   3388   }
  3346   3389   
  3347   3390   /* 
................................................................................
  3382   3425     ** of blocks from the segments table. But this is not considered overhead
  3383   3426     ** as it would also be required by a crisis-merge that used the same input 
  3384   3427     ** segments.
  3385   3428     */
  3386   3429     const u32 nMinMerge = 64;       /* Minimum amount of incr-merge work to do */
  3387   3430   
  3388   3431     Fts3Table *p = (Fts3Table*)pVtab;
  3389         -  int rc = sqlite3Fts3PendingTermsFlush(p);
         3432  +  int rc;
         3433  +  i64 iLastRowid = sqlite3_last_insert_rowid(p->db);
  3390   3434   
         3435  +  rc = sqlite3Fts3PendingTermsFlush(p);
  3391   3436     if( rc==SQLITE_OK 
  3392   3437      && p->nLeafAdd>(nMinMerge/16) 
  3393   3438      && p->nAutoincrmerge && p->nAutoincrmerge!=0xff
  3394   3439     ){
  3395   3440       int mxLevel = 0;              /* Maximum relative level value in db */
  3396   3441       int A;                        /* Incr-merge parameter A */
  3397   3442   
................................................................................
  3398   3443       rc = sqlite3Fts3MaxLevel(p, &mxLevel);
  3399   3444       assert( rc==SQLITE_OK || mxLevel==0 );
  3400   3445       A = p->nLeafAdd * mxLevel;
  3401   3446       A += (A/2);
  3402   3447       if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
  3403   3448     }
  3404   3449     sqlite3Fts3SegmentsClose(p);
         3450  +  sqlite3_set_last_insert_rowid(p->db, iLastRowid);
  3405   3451     return rc;
  3406   3452   }
  3407   3453   
  3408   3454   /*
  3409   3455   ** If it is currently unknown whether or not the FTS table has an %_stat
  3410   3456   ** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat
  3411   3457   ** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code
  3412   3458   ** if an error occurs.
  3413   3459   */
  3414   3460   static int fts3SetHasStat(Fts3Table *p){
  3415   3461     int rc = SQLITE_OK;
  3416   3462     if( p->bHasStat==2 ){
  3417         -    const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'";
  3418         -    char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName);
  3419         -    if( zSql ){
  3420         -      sqlite3_stmt *pStmt = 0;
  3421         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  3422         -      if( rc==SQLITE_OK ){
  3423         -        int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
  3424         -        rc = sqlite3_finalize(pStmt);
  3425         -        if( rc==SQLITE_OK ) p->bHasStat = bHasStat;
  3426         -      }
  3427         -      sqlite3_free(zSql);
         3463  +    char *zTbl = sqlite3_mprintf("%s_stat", p->zName);
         3464  +    if( zTbl ){
         3465  +      int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0);
         3466  +      sqlite3_free(zTbl);
         3467  +      p->bHasStat = (res==SQLITE_OK);
  3428   3468       }else{
  3429   3469         rc = SQLITE_NOMEM;
  3430   3470       }
  3431   3471     }
  3432   3472     return rc;
  3433   3473   }
  3434   3474   
................................................................................
  3527   3567   */
  3528   3568   static int fts3FunctionArg(
  3529   3569     sqlite3_context *pContext,      /* SQL function call context */
  3530   3570     const char *zFunc,              /* Function name */
  3531   3571     sqlite3_value *pVal,            /* argv[0] passed to function */
  3532   3572     Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
  3533   3573   ){
  3534         -  Fts3Cursor *pRet;
  3535         -  if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
  3536         -   || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
  3537         -  ){
         3574  +  int rc;
         3575  +  *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor");
         3576  +  if( (*ppCsr)!=0 ){
         3577  +    rc = SQLITE_OK;
         3578  +  }else{
  3538   3579       char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
  3539   3580       sqlite3_result_error(pContext, zErr, -1);
  3540   3581       sqlite3_free(zErr);
  3541         -    return SQLITE_ERROR;
         3582  +    rc = SQLITE_ERROR;
  3542   3583     }
  3543         -  memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
  3544         -  *ppCsr = pRet;
  3545         -  return SQLITE_OK;
         3584  +  return rc;
  3546   3585   }
  3547   3586   
  3548   3587   /*
  3549   3588   ** Implementation of the snippet() function for FTS3
  3550   3589   */
  3551   3590   static void fts3SnippetFunc(
  3552   3591     sqlite3_context *pContext,      /* SQLite function call context */
................................................................................
  3925   3964   #ifdef SQLITE_TEST
  3926   3965     if( rc==SQLITE_OK ){
  3927   3966       rc = sqlite3Fts3ExprInitTestInterface(db);
  3928   3967     }
  3929   3968   #endif
  3930   3969   
  3931   3970     /* Create the virtual table wrapper around the hash-table and overload 
  3932         -  ** the two scalar functions. If this is successful, register the
         3971  +  ** the four scalar functions. If this is successful, register the
  3933   3972     ** module with sqlite.
  3934   3973     */
  3935   3974     if( SQLITE_OK==rc 
  3936   3975      && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
  3937   3976      && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
  3938   3977      && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
  3939   3978      && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
................................................................................
  4508   4547     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  4509   4548     u8 bEof = 0;
  4510   4549   
  4511   4550     /* This is only called if it is guaranteed that the phrase has at least
  4512   4551     ** one incremental token. In which case the bIncr flag is set. */
  4513   4552     assert( p->bIncr==1 );
  4514   4553   
  4515         -  if( p->nToken==1 && p->bIncr ){
         4554  +  if( p->nToken==1 ){
  4516   4555       rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
  4517   4556           &pDL->iDocid, &pDL->pList, &pDL->nList
  4518   4557       );
  4519   4558       if( pDL->pList==0 ) bEof = 1;
  4520   4559     }else{
  4521   4560       int bDescDoclist = pCsr->bDesc;
  4522   4561       struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS];
................................................................................
  4741   4780   ** The average document size in pages is calculated by first calculating 
  4742   4781   ** determining the average size in bytes, B. If B is less than the amount
  4743   4782   ** of data that will fit on a single leaf page of an intkey table in
  4744   4783   ** this database, then the average docsize is 1. Otherwise, it is 1 plus
  4745   4784   ** the number of overflow pages consumed by a record B bytes in size.
  4746   4785   */
  4747   4786   static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
         4787  +  int rc = SQLITE_OK;
  4748   4788     if( pCsr->nRowAvg==0 ){
  4749   4789       /* The average document size, which is required to calculate the cost
  4750   4790       ** of each doclist, has not yet been determined. Read the required 
  4751   4791       ** data from the %_stat table to calculate it.
  4752   4792       **
  4753   4793       ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
  4754   4794       ** varints, where nCol is the number of columns in the FTS3 table.
  4755   4795       ** The first varint is the number of documents currently stored in
  4756   4796       ** the table. The following nCol varints contain the total amount of
  4757   4797       ** data stored in all rows of each column of the table, from left
  4758   4798       ** to right.
  4759   4799       */
  4760         -    int rc;
  4761   4800       Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
  4762   4801       sqlite3_stmt *pStmt;
  4763   4802       sqlite3_int64 nDoc = 0;
  4764   4803       sqlite3_int64 nByte = 0;
  4765   4804       const char *pEnd;
  4766   4805       const char *a;
  4767   4806   
................................................................................
  4780   4819         return FTS_CORRUPT_VTAB;
  4781   4820       }
  4782   4821   
  4783   4822       pCsr->nDoc = nDoc;
  4784   4823       pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
  4785   4824       assert( pCsr->nRowAvg>0 ); 
  4786   4825       rc = sqlite3_reset(pStmt);
  4787         -    if( rc!=SQLITE_OK ) return rc;
  4788   4826     }
  4789   4827   
  4790   4828     *pnPage = pCsr->nRowAvg;
  4791         -  return SQLITE_OK;
         4829  +  return rc;
  4792   4830   }
  4793   4831   
  4794   4832   /*
  4795   4833   ** This function is called to select the tokens (if any) that will be 
  4796   4834   ** deferred. The array aTC[] has already been populated when this is
  4797   4835   ** called.
  4798   4836   **
................................................................................
  5134   5172               }else{
  5135   5173                 fts3EvalNextRow(pCsr, pRight, pRc);
  5136   5174               }
  5137   5175             }
  5138   5176             pExpr->iDocid = pLeft->iDocid;
  5139   5177             pExpr->bEof = (pLeft->bEof || pRight->bEof);
  5140   5178             if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){
  5141         -            if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){
         5179  +            assert( pRight->eType==FTSQUERY_PHRASE );
         5180  +            if( pRight->pPhrase->doclist.aAll ){
  5142   5181                 Fts3Doclist *pDl = &pRight->pPhrase->doclist;
  5143   5182                 while( *pRc==SQLITE_OK && pRight->bEof==0 ){
  5144   5183                   memset(pDl->pList, 0, pDl->nList);
  5145   5184                   fts3EvalNextRow(pCsr, pRight, pRc);
  5146   5185                 }
  5147   5186               }
  5148   5187               if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){
................................................................................
  5163   5202           sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  5164   5203   
  5165   5204           assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
  5166   5205           assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
  5167   5206   
  5168   5207           if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
  5169   5208             fts3EvalNextRow(pCsr, pLeft, pRc);
  5170         -        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
         5209  +        }else if( pLeft->bEof || iCmp>0 ){
  5171   5210             fts3EvalNextRow(pCsr, pRight, pRc);
  5172   5211           }else{
  5173   5212             fts3EvalNextRow(pCsr, pLeft, pRc);
  5174   5213             fts3EvalNextRow(pCsr, pRight, pRc);
  5175   5214           }
  5176   5215   
  5177   5216           pExpr->bEof = (pLeft->bEof && pRight->bEof);
................................................................................
  5255   5294     **
  5256   5295     ** The right-hand child of a NEAR node is always a phrase. The 
  5257   5296     ** left-hand child may be either a phrase or a NEAR node. There are
  5258   5297     ** no exceptions to this - it's the way the parser in fts3_expr.c works.
  5259   5298     */
  5260   5299     if( *pRc==SQLITE_OK 
  5261   5300      && pExpr->eType==FTSQUERY_NEAR 
  5262         -   && pExpr->bEof==0
  5263   5301      && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  5264   5302     ){
  5265   5303       Fts3Expr *p; 
  5266   5304       int nTmp = 0;                 /* Bytes of temp space */
  5267   5305       char *aTmp;                   /* Temp space for PoslistNearMerge() */
  5268   5306   
  5269   5307       /* Allocate temporary working space. */
  5270   5308       for(p=pExpr; p->pLeft; p=p->pLeft){
         5309  +      assert( p->pRight->pPhrase->doclist.nList>0 );
  5271   5310         nTmp += p->pRight->pPhrase->doclist.nList;
  5272   5311       }
  5273   5312       nTmp += p->pPhrase->doclist.nList;
  5274         -    if( nTmp==0 ){
         5313  +    aTmp = sqlite3_malloc(nTmp*2);
         5314  +    if( !aTmp ){
         5315  +      *pRc = SQLITE_NOMEM;
  5275   5316         res = 0;
  5276   5317       }else{
  5277         -      aTmp = sqlite3_malloc(nTmp*2);
  5278         -      if( !aTmp ){
  5279         -        *pRc = SQLITE_NOMEM;
  5280         -        res = 0;
  5281         -      }else{
  5282         -        char *aPoslist = p->pPhrase->doclist.pList;
  5283         -        int nToken = p->pPhrase->nToken;
  5284         -
  5285         -        for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
  5286         -          Fts3Phrase *pPhrase = p->pRight->pPhrase;
  5287         -          int nNear = p->nNear;
  5288         -          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  5289         -        }
  5290         -
  5291         -        aPoslist = pExpr->pRight->pPhrase->doclist.pList;
  5292         -        nToken = pExpr->pRight->pPhrase->nToken;
  5293         -        for(p=pExpr->pLeft; p && res; p=p->pLeft){
  5294         -          int nNear;
  5295         -          Fts3Phrase *pPhrase;
  5296         -          assert( p->pParent && p->pParent->pLeft==p );
  5297         -          nNear = p->pParent->nNear;
  5298         -          pPhrase = (
  5299         -              p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
  5300         -              );
  5301         -          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  5302         -        }
         5318  +      char *aPoslist = p->pPhrase->doclist.pList;
         5319  +      int nToken = p->pPhrase->nToken;
         5320  +
         5321  +      for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
         5322  +        Fts3Phrase *pPhrase = p->pRight->pPhrase;
         5323  +        int nNear = p->nNear;
         5324  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         5325  +      }
         5326  +
         5327  +      aPoslist = pExpr->pRight->pPhrase->doclist.pList;
         5328  +      nToken = pExpr->pRight->pPhrase->nToken;
         5329  +      for(p=pExpr->pLeft; p && res; p=p->pLeft){
         5330  +        int nNear;
         5331  +        Fts3Phrase *pPhrase;
         5332  +        assert( p->pParent && p->pParent->pLeft==p );
         5333  +        nNear = p->pParent->nNear;
         5334  +        pPhrase = (
         5335  +            p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
         5336  +        );
         5337  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  5303   5338         }
         5339  +    }
  5304   5340   
  5305         -      sqlite3_free(aTmp);
  5306         -    }
         5341  +    sqlite3_free(aTmp);
  5307   5342     }
  5308   5343   
  5309   5344     return res;
  5310   5345   }
  5311   5346   
  5312   5347   /*
  5313   5348   ** This function is a helper function for sqlite3Fts3EvalTestDeferred().

Changes to ext/fts3/fts3Int.h.

   226    226     int nAutoincrmerge;             /* Value configured by 'automerge' */
   227    227     u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
   228    228   
   229    229     /* Precompiled statements used by the implementation. Each of these 
   230    230     ** statements is run and reset within a single virtual table API call. 
   231    231     */
   232    232     sqlite3_stmt *aStmt[40];
          233  +  sqlite3_stmt *pSeekStmt;        /* Cache for fts3CursorSeekStmt() */
   233    234   
   234    235     char *zReadExprlist;
   235    236     char *zWriteExprlist;
   236    237   
   237    238     int nNodeSize;                  /* Soft limit for node size */
   238    239     u8 bFts4;                       /* True for FTS4, false for FTS3 */
   239    240     u8 bHasStat;                    /* True if %_stat table exists (2==unknown) */
................................................................................
   295    296   ** the xOpen method. Cursors are destroyed using the xClose method.
   296    297   */
   297    298   struct Fts3Cursor {
   298    299     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
   299    300     i16 eSearch;                    /* Search strategy (see below) */
   300    301     u8 isEof;                       /* True if at End Of Results */
   301    302     u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
          303  +  u8 bSeekStmt;                   /* True if pStmt is a seek */
   302    304     sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
   303    305     Fts3Expr *pExpr;                /* Parsed MATCH query string */
   304    306     int iLangid;                    /* Language being queried for */
   305    307     int nPhrase;                    /* Number of matchable phrases in query */
   306    308     Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
   307    309     sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
   308    310     char *pNextId;                  /* Pointer into the body of aDoclist */

Changes to ext/fts3/fts3_unicode.c.

   132    132     unicode_tokenizer *p,           /* Tokenizer to add exceptions to */
   133    133     int bAlnum,                     /* Replace Isalnum() return value with this */
   134    134     const char *zIn,                /* Array of characters to make exceptions */
   135    135     int nIn                         /* Length of z in bytes */
   136    136   ){
   137    137     const unsigned char *z = (const unsigned char *)zIn;
   138    138     const unsigned char *zTerm = &z[nIn];
   139         -  int iCode;
          139  +  unsigned int iCode;
   140    140     int nEntry = 0;
   141    141   
   142    142     assert( bAlnum==0 || bAlnum==1 );
   143    143   
   144    144     while( z<zTerm ){
   145    145       READ_UTF8(z, zTerm, iCode);
   146         -    assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
   147         -    if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
   148         -     && sqlite3FtsUnicodeIsdiacritic(iCode)==0 
          146  +    assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 );
          147  +    if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
          148  +     && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0 
   149    149       ){
   150    150         nEntry++;
   151    151       }
   152    152     }
   153    153   
   154    154     if( nEntry ){
   155    155       int *aNew;                    /* New aiException[] array */
................................................................................
   158    158       aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int));
   159    159       if( aNew==0 ) return SQLITE_NOMEM;
   160    160       nNew = p->nException;
   161    161   
   162    162       z = (const unsigned char *)zIn;
   163    163       while( z<zTerm ){
   164    164         READ_UTF8(z, zTerm, iCode);
   165         -      if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 
   166         -       && sqlite3FtsUnicodeIsdiacritic(iCode)==0
          165  +      if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 
          166  +       && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0
   167    167         ){
   168    168           int i, j;
   169         -        for(i=0; i<nNew && aNew[i]<iCode; i++);
          169  +        for(i=0; i<nNew && aNew[i]<(int)iCode; i++);
   170    170           for(j=nNew; j>i; j--) aNew[j] = aNew[j-1];
   171         -        aNew[i] = iCode;
          171  +        aNew[i] = (int)iCode;
   172    172           nNew++;
   173    173         }
   174    174       }
   175    175       p->aiException = aNew;
   176    176       p->nException = nNew;
   177    177     }
   178    178   
................................................................................
   314    314     int *pnToken,                   /* OUT: Number of bytes at *paToken */
   315    315     int *piStart,                   /* OUT: Starting offset of token */
   316    316     int *piEnd,                     /* OUT: Ending offset of token */
   317    317     int *piPos                      /* OUT: Position integer of token */
   318    318   ){
   319    319     unicode_cursor *pCsr = (unicode_cursor *)pC;
   320    320     unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer);
   321         -  int iCode = 0;
          321  +  unsigned int iCode = 0;
   322    322     char *zOut;
   323    323     const unsigned char *z = &pCsr->aInput[pCsr->iOff];
   324    324     const unsigned char *zStart = z;
   325    325     const unsigned char *zEnd;
   326    326     const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput];
   327    327   
   328    328     /* Scan past any delimiter characters before the start of the next token.
   329    329     ** Return SQLITE_DONE early if this takes us all the way to the end of 
   330    330     ** the input.  */
   331    331     while( z<zTerm ){
   332    332       READ_UTF8(z, zTerm, iCode);
   333         -    if( unicodeIsAlnum(p, iCode) ) break;
          333  +    if( unicodeIsAlnum(p, (int)iCode) ) break;
   334    334       zStart = z;
   335    335     }
   336    336     if( zStart>=zTerm ) return SQLITE_DONE;
   337    337   
   338    338     zOut = pCsr->zToken;
   339    339     do {
   340    340       int iOut;
................................................................................
   346    346         zOut = &zNew[zOut - pCsr->zToken];
   347    347         pCsr->zToken = zNew;
   348    348         pCsr->nAlloc += 64;
   349    349       }
   350    350   
   351    351       /* Write the folded case of the last character read to the output */
   352    352       zEnd = z;
   353         -    iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic);
          353  +    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
   354    354       if( iOut ){
   355    355         WRITE_UTF8(zOut, iOut);
   356    356       }
   357    357   
   358    358       /* If the cursor is not at EOF, read the next character */
   359    359       if( z>=zTerm ) break;
   360    360       READ_UTF8(z, zTerm, iCode);
   361         -  }while( unicodeIsAlnum(p, iCode) 
   362         -       || sqlite3FtsUnicodeIsdiacritic(iCode)
          361  +  }while( unicodeIsAlnum(p, (int)iCode) 
          362  +       || sqlite3FtsUnicodeIsdiacritic((int)iCode)
   363    363     );
   364    364   
   365    365     /* Set the output variables and return. */
   366    366     pCsr->iOff = (int)(z - pCsr->aInput);
   367    367     *paToken = pCsr->zToken;
   368    368     *pnToken = (int)(zOut - pCsr->zToken);
   369    369     *piStart = (int)(zStart - pCsr->aInput);

Changes to ext/fts3/fts3_unicode2.c.

   123    123       0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
   124    124       0x380400F0,
   125    125     };
   126    126     static const unsigned int aAscii[4] = {
   127    127       0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
   128    128     };
   129    129   
   130         -  if( c<128 ){
   131         -    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   132         -  }else if( c<(1<<22) ){
          130  +  if( (unsigned int)c<128 ){
          131  +    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
          132  +  }else if( (unsigned int)c<(1<<22) ){
   133    133       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   134    134       int iRes = 0;
   135    135       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   136    136       int iLo = 0;
   137    137       while( iHi>=iLo ){
   138    138         int iTest = (iHi + iLo) / 2;
   139    139         if( key >= aEntry[iTest] ){
................................................................................
   318    318      65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, 
   319    319      65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, 
   320    320      65514, 65521, 65527, 65528, 65529, 
   321    321     };
   322    322   
   323    323     int ret = c;
   324    324   
   325         -  assert( c>=0 );
   326    325     assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
   327    326   
   328    327     if( c<128 ){
   329    328       if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
   330    329     }else if( c<65536 ){
          330  +    const struct TableEntry *p;
   331    331       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   332    332       int iLo = 0;
   333    333       int iRes = -1;
   334    334   
          335  +    assert( c>aEntry[0].iCode );
   335    336       while( iHi>=iLo ){
   336    337         int iTest = (iHi + iLo) / 2;
   337    338         int cmp = (c - aEntry[iTest].iCode);
   338    339         if( cmp>=0 ){
   339    340           iRes = iTest;
   340    341           iLo = iTest+1;
   341    342         }else{
   342    343           iHi = iTest-1;
   343    344         }
   344    345       }
   345         -    assert( iRes<0 || c>=aEntry[iRes].iCode );
   346    346   
   347         -    if( iRes>=0 ){
   348         -      const struct TableEntry *p = &aEntry[iRes];
   349         -      if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   350         -        ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   351         -        assert( ret>0 );
   352         -      }
          347  +    assert( iRes>=0 && c>=aEntry[iRes].iCode );
          348  +    p = &aEntry[iRes];
          349  +    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
          350  +      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
          351  +      assert( ret>0 );
   353    352       }
   354    353   
   355    354       if( bRemoveDiacritic ) ret = remove_diacritic(ret);
   356    355     }
   357    356     
   358    357     else if( c>=66560 && c<66600 ){
   359    358       ret = c + 40;
   360    359     }
   361    360   
   362    361     return ret;
   363    362   }
   364    363   #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
   365    364   #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/fts3_write.c.

   403    403         zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
   404    404       }else{
   405    405         zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
   406    406       }
   407    407       if( !zSql ){
   408    408         rc = SQLITE_NOMEM;
   409    409       }else{
   410         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
          410  +      rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
          411  +                              &pStmt, NULL);
   411    412         sqlite3_free(zSql);
   412    413         assert( rc==SQLITE_OK || pStmt==0 );
   413    414         p->aStmt[eStmt] = pStmt;
   414    415       }
   415    416     }
   416    417     if( apVal ){
   417    418       int i;
................................................................................
  4952   4953     return rc;
  4953   4954   }
  4954   4955   
  4955   4956   /*
  4956   4957   ** Convert the text beginning at *pz into an integer and return
  4957   4958   ** its value.  Advance *pz to point to the first character past
  4958   4959   ** the integer.
         4960  +**
         4961  +** This function used for parameters to merge= and incrmerge=
         4962  +** commands. 
  4959   4963   */
  4960   4964   static int fts3Getint(const char **pz){
  4961   4965     const char *z = *pz;
  4962   4966     int i = 0;
  4963         -  while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
         4967  +  while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0';
  4964   4968     *pz = z;
  4965   4969     return i;
  4966   4970   }
  4967   4971   
  4968   4972   /*
  4969   4973   ** Process statements of the form:
  4970   4974   **

Added ext/fts3/tool/fts3cov.sh.

            1  +#!/bin/sh
            2  +
            3  +set -e
            4  +
            5  +srcdir=`dirname $(dirname $(dirname $(dirname $0)))`
            6  +./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt
            7  +
            8  +echo ""
            9  +
           10  +for f in `ls $srcdir/ext/fts3/*.c` 
           11  +do
           12  +  f=`basename $f`
           13  +  echo -ne "$f: "
           14  +  gcov -b $f | grep Taken | sed 's/Taken at least once://'
           15  +done
           16  +

Changes to ext/fts3/unicode/mkunicode.tcl.

   223    223     puts "** is less than zero."
   224    224     puts "*/"
   225    225     puts "int ${zFunc}\(int c)\{"
   226    226     an_print_range_array $lRange
   227    227     an_print_ascii_bitmap $lRange
   228    228     puts {
   229    229     if( (unsigned int)c<128 ){
   230         -    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
          230  +    return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
   231    231     }else if( (unsigned int)c<(1<<22) ){
   232    232       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   233    233       int iRes = 0;
   234    234       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   235    235       int iLo = 0;
   236    236       while( iHi>=iLo ){
   237    237         int iTest = (iHi + iLo) / 2;

Changes to ext/fts5/fts5Int.h.

    26     26   typedef unsigned char  u8;
    27     27   typedef unsigned int   u32;
    28     28   typedef unsigned short u16;
    29     29   typedef short i16;
    30     30   typedef sqlite3_int64 i64;
    31     31   typedef sqlite3_uint64 u64;
    32     32   
    33         -#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
           33  +#ifndef ArraySize
           34  +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
           35  +#endif
    34     36   
    35     37   #define testcase(x)
    36     38   #define ALWAYS(x) 1
    37     39   #define NEVER(x) 0
    38     40   
    39     41   #define MIN(x,y) (((x) < (y)) ? (x) : (y))
    40     42   #define MAX(x,y) (((x) > (y)) ? (x) : (y))
................................................................................
   440    442     Fts5Index *p,                   /* Index to write to */
   441    443     int bDelete,                    /* True if current operation is a delete */
   442    444     i64 iDocid                      /* Docid to add or remove data from */
   443    445   );
   444    446   
   445    447   /*
   446    448   ** Flush any data stored in the in-memory hash tables to the database.
   447         -** If the bCommit flag is true, also close any open blob handles.
          449  +** Also close any open blob handles.
   448    450   */
   449         -int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit);
          451  +int sqlite3Fts5IndexSync(Fts5Index *p);
   450    452   
   451    453   /*
   452    454   ** Discard any data stored in the in-memory hash tables. Do not write it
   453    455   ** to the database. Additionally, assume that the contents of the %_data
   454    456   ** table may have changed on disk. So any in-memory caches of %_data 
   455    457   ** records must be invalidated.
   456    458   */
................................................................................
   612    614   int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
   613    615   void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
   614    616   
   615    617   int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
   616    618   int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
   617    619   int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
   618    620   
   619         -int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
          621  +int sqlite3Fts5StorageSync(Fts5Storage *p);
   620    622   int sqlite3Fts5StorageRollback(Fts5Storage *p);
   621    623   
   622    624   int sqlite3Fts5StorageConfigValue(
   623    625       Fts5Storage *p, const char*, sqlite3_value*, int
   624    626   );
   625    627   
   626    628   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
................................................................................
   648    650     const char *p;                  /* Token text (not NULL terminated) */
   649    651     int n;                          /* Size of buffer p in bytes */
   650    652   };
   651    653   
   652    654   /* Parse a MATCH expression. */
   653    655   int sqlite3Fts5ExprNew(
   654    656     Fts5Config *pConfig, 
          657  +  int iCol,                       /* Column on LHS of MATCH operator */
   655    658     const char *zExpr,
   656    659     Fts5Expr **ppNew, 
   657    660     char **pzErr
   658    661   );
   659    662   
   660    663   /*
   661    664   ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
................................................................................
   732    735   );
   733    736   
   734    737   void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
   735    738   void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
   736    739   void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
   737    740   
   738    741   void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
   739         -void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
          742  +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
   740    743   Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
   741    744   void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
   742    745   void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
   743    746   
   744    747   /*
   745    748   ** End of interface to code in fts5_expr.c.
   746    749   **************************************************************************/

Changes to ext/fts5/fts5_buffer.c.

    63     63   void sqlite3Fts5BufferAppendBlob(
    64     64     int *pRc,
    65     65     Fts5Buffer *pBuf, 
    66     66     u32 nData, 
    67     67     const u8 *pData
    68     68   ){
    69     69     assert_nc( *pRc || nData>=0 );
    70         -  if( fts5BufferGrow(pRc, pBuf, nData) ) return;
    71         -  memcpy(&pBuf->p[pBuf->n], pData, nData);
    72         -  pBuf->n += nData;
           70  +  if( nData ){
           71  +    if( fts5BufferGrow(pRc, pBuf, nData) ) return;
           72  +    memcpy(&pBuf->p[pBuf->n], pData, nData);
           73  +    pBuf->n += nData;
           74  +  }
    73     75   }
    74     76   
    75     77   /*
    76     78   ** Append the nul-terminated string zStr to the buffer pBuf. This function
    77     79   ** ensures that the byte following the buffer data is set to 0x00, even 
    78     80   ** though this byte is not included in the pBuf->n count.
    79     81   */
................................................................................
   242    244     return SQLITE_OK;
   243    245   }
   244    246   
   245    247   void *sqlite3Fts5MallocZero(int *pRc, int nByte){
   246    248     void *pRet = 0;
   247    249     if( *pRc==SQLITE_OK ){
   248    250       pRet = sqlite3_malloc(nByte);
   249         -    if( pRet==0 && nByte>0 ){
   250         -      *pRc = SQLITE_NOMEM;
          251  +    if( pRet==0 ){
          252  +      if( nByte>0 ) *pRc = SQLITE_NOMEM;
   251    253       }else{
   252    254         memset(pRet, 0, nByte);
   253    255       }
   254    256     }
   255    257     return pRet;
   256    258   }
   257    259   

Changes to ext/fts5/fts5_expr.c.

   209    209   }
   210    210   
   211    211   static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); }
   212    212   static void fts5ParseFree(void *p){ sqlite3_free(p); }
   213    213   
   214    214   int sqlite3Fts5ExprNew(
   215    215     Fts5Config *pConfig,            /* FTS5 Configuration */
          216  +  int iCol,
   216    217     const char *zExpr,              /* Expression text */
   217    218     Fts5Expr **ppNew, 
   218    219     char **pzErr
   219    220   ){
   220    221     Fts5Parse sParse;
   221    222     Fts5Token token;
   222    223     const char *z = zExpr;
................................................................................
   232    233     sParse.pConfig = pConfig;
   233    234   
   234    235     do {
   235    236       t = fts5ExprGetToken(&sParse, &z, &token);
   236    237       sqlite3Fts5Parser(pEngine, t, token, &sParse);
   237    238     }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
   238    239     sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
          240  +
          241  +  /* If the LHS of the MATCH expression was a user column, apply the
          242  +  ** implicit column-filter.  */
          243  +  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
          244  +    int n = sizeof(Fts5Colset);
          245  +    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
          246  +    if( pColset ){
          247  +      pColset->nCol = 1;
          248  +      pColset->aiCol[0] = iCol;
          249  +      sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
          250  +    }
          251  +  }
   239    252   
   240    253     assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
   241    254     if( sParse.rc==SQLITE_OK ){
   242    255       *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
   243    256       if( pNew==0 ){
   244    257         sParse.rc = SQLITE_NOMEM;
   245    258         sqlite3Fts5ParseNodeFree(sParse.pExpr);
................................................................................
   742    755   }
   743    756   
   744    757   
   745    758   /*
   746    759   ** Initialize all term iterators in the pNear object. If any term is found
   747    760   ** to match no documents at all, return immediately without initializing any
   748    761   ** further iterators.
          762  +**
          763  +** If an error occurs, return an SQLite error code. Otherwise, return
          764  +** SQLITE_OK. It is not considered an error if some term matches zero
          765  +** documents.
   749    766   */
   750    767   static int fts5ExprNearInitAll(
   751    768     Fts5Expr *pExpr,
   752    769     Fts5ExprNode *pNode
   753    770   ){
   754    771     Fts5ExprNearset *pNear = pNode->pNear;
   755         -  int i, j;
   756         -  int rc = SQLITE_OK;
   757         -  int bEof = 1;
          772  +  int i;
   758    773   
   759    774     assert( pNode->bNomatch==0 );
   760         -  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
          775  +  for(i=0; i<pNear->nPhrase; i++){
   761    776       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   762         -    for(j=0; j<pPhrase->nTerm; j++){
   763         -      Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
   764         -      Fts5ExprTerm *p;
   765         -
   766         -      for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
   767         -        if( p->pIter ){
   768         -          sqlite3Fts5IterClose(p->pIter);
   769         -          p->pIter = 0;
   770         -        }
   771         -        rc = sqlite3Fts5IndexQuery(
   772         -            pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
   773         -            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
   774         -            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
   775         -            pNear->pColset,
   776         -            &p->pIter
   777         -        );
   778         -        assert( rc==SQLITE_OK || p->pIter==0 );
   779         -        if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
   780         -          bEof = 0;
   781         -        }
   782         -      }
   783         -
   784         -      if( bEof ) break;
   785         -    }
   786         -    if( bEof ) break;
   787         -  }
   788         -
   789         -  pNode->bEof = bEof;
   790         -  return rc;
          777  +    if( pPhrase->nTerm==0 ){
          778  +      pNode->bEof = 1;
          779  +      return SQLITE_OK;
          780  +    }else{
          781  +      int j;
          782  +      for(j=0; j<pPhrase->nTerm; j++){
          783  +        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
          784  +        Fts5ExprTerm *p;
          785  +        int bHit = 0;
          786  +
          787  +        for(p=pTerm; p; p=p->pSynonym){
          788  +          int rc;
          789  +          if( p->pIter ){
          790  +            sqlite3Fts5IterClose(p->pIter);
          791  +            p->pIter = 0;
          792  +          }
          793  +          rc = sqlite3Fts5IndexQuery(
          794  +              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
          795  +              (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
          796  +              (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
          797  +              pNear->pColset,
          798  +              &p->pIter
          799  +          );
          800  +          assert( (rc==SQLITE_OK)==(p->pIter!=0) );
          801  +          if( rc!=SQLITE_OK ) return rc;
          802  +          if( 0==sqlite3Fts5IterEof(p->pIter) ){
          803  +            bHit = 1;
          804  +          }
          805  +        }
          806  +
          807  +        if( bHit==0 ){
          808  +          pNode->bEof = 1;
          809  +          return SQLITE_OK;
          810  +        }
          811  +      }
          812  +    }
          813  +  }
          814  +
          815  +  pNode->bEof = 0;
          816  +  return SQLITE_OK;
   791    817   }
   792    818   
   793    819   /*
   794    820   ** If pExpr is an ASC iterator, this function returns a value with the
   795    821   ** same sign as:
   796    822   **
   797    823   **   (iLhs - iRhs)
................................................................................
   916    942             if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
   917    943               pNode->bNomatch = 0;
   918    944               pNode->bEof = 1;
   919    945               return rc;
   920    946             }
   921    947           }else{
   922    948             Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
   923         -          if( pIter->iRowid==iLast ) continue;
          949  +          if( pIter->iRowid==iLast || pIter->bEof ) continue;
   924    950             bMatch = 0;
   925    951             if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
   926    952               return rc;
   927    953             }
   928    954           }
   929    955         }
   930    956       }
................................................................................
  1093   1119       Fts5ExprNode *p1 = pNode->apChild[i];
  1094   1120       assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 );
  1095   1121       if( p1->bEof==0 ){
  1096   1122         if( (p1->iRowid==iLast) 
  1097   1123          || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
  1098   1124         ){
  1099   1125           int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
  1100         -        if( rc!=SQLITE_OK ) return rc;
         1126  +        if( rc!=SQLITE_OK ){
         1127  +          pNode->bNomatch = 0;
         1128  +          return rc;
         1129  +        }
  1101   1130         }
  1102   1131       }
  1103   1132     }
  1104   1133   
  1105   1134     fts5ExprNodeTest_OR(pExpr, pNode);
  1106   1135     return SQLITE_OK;
  1107   1136   }
................................................................................
  1124   1153       bMatch = 1;
  1125   1154       for(iChild=0; iChild<pAnd->nChild; iChild++){
  1126   1155         Fts5ExprNode *pChild = pAnd->apChild[iChild];
  1127   1156         int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
  1128   1157         if( cmp>0 ){
  1129   1158           /* Advance pChild until it points to iLast or laster */
  1130   1159           rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
  1131         -        if( rc!=SQLITE_OK ) return rc;
         1160  +        if( rc!=SQLITE_OK ){
         1161  +          pAnd->bNomatch = 0;
         1162  +          return rc;
         1163  +        }
  1132   1164         }
  1133   1165   
  1134   1166         /* If the child node is now at EOF, so is the parent AND node. Otherwise,
  1135   1167         ** the child node is guaranteed to have advanced at least as far as
  1136   1168         ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
  1137   1169         ** new lastest rowid seen so far.  */
  1138   1170         assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 );
................................................................................
  1163   1195     Fts5ExprNode *pNode,
  1164   1196     int bFromValid,
  1165   1197     i64 iFrom
  1166   1198   ){
  1167   1199     int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  1168   1200     if( rc==SQLITE_OK ){
  1169   1201       rc = fts5ExprNodeTest_AND(pExpr, pNode);
         1202  +  }else{
         1203  +    pNode->bNomatch = 0;
  1170   1204     }
  1171   1205     return rc;
  1172   1206   }
  1173   1207   
  1174   1208   static int fts5ExprNodeTest_NOT(
  1175   1209     Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
  1176   1210     Fts5ExprNode *pNode             /* FTS5_NOT node to advance */
................................................................................
  1205   1239     int bFromValid,
  1206   1240     i64 iFrom
  1207   1241   ){
  1208   1242     int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  1209   1243     if( rc==SQLITE_OK ){
  1210   1244       rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  1211   1245     }
         1246  +  if( rc!=SQLITE_OK ){
         1247  +    pNode->bNomatch = 0;
         1248  +  }
  1212   1249     return rc;
  1213   1250   }
  1214   1251   
  1215   1252   /*
  1216   1253   ** If pNode currently points to a match, this function returns SQLITE_OK
  1217   1254   ** without modifying it. Otherwise, pNode is advanced until it does point
  1218   1255   ** to a match or EOF is reached.
................................................................................
  1327   1364   
  1328   1365     p->pIndex = pIdx;
  1329   1366     p->bDesc = bDesc;
  1330   1367     rc = fts5ExprNodeFirst(p, pRoot);
  1331   1368   
  1332   1369     /* If not at EOF but the current rowid occurs earlier than iFirst in
  1333   1370     ** the iteration order, move to document iFirst or later. */
  1334         -  if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
         1371  +  if( rc==SQLITE_OK 
         1372  +   && 0==pRoot->bEof 
         1373  +   && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 
         1374  +  ){
  1335   1375       rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
  1336   1376     }
  1337   1377   
  1338   1378     /* If the iterator is not at a real match, skip forward until it is. */
  1339   1379     while( pRoot->bNomatch ){
  1340   1380       assert( pRoot->bEof==0 && rc==SQLITE_OK );
  1341   1381       rc = fts5ExprNodeNext(p, pRoot, 0, 0);
................................................................................
  1581   1621     char *z = 0;
  1582   1622   
  1583   1623     memset(&sCtx, 0, sizeof(TokenCtx));
  1584   1624     sCtx.pPhrase = pAppend;
  1585   1625   
  1586   1626     rc = fts5ParseStringFromToken(pToken, &z);
  1587   1627     if( rc==SQLITE_OK ){
  1588         -    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
         1628  +    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
  1589   1629       int n;
  1590   1630       sqlite3Fts5Dequote(z);
  1591   1631       n = (int)strlen(z);
  1592   1632       rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
  1593   1633     }
  1594   1634     sqlite3_free(z);
  1595   1635     if( rc || (rc = sCtx.rc) ){
................................................................................
  1855   1895       assert( pParse->rc!=SQLITE_OK );
  1856   1896       sqlite3_free(pColset);
  1857   1897     }
  1858   1898   
  1859   1899     return pRet;
  1860   1900   }
  1861   1901   
         1902  +/*
         1903  +** If argument pOrig is NULL, or if (*pRc) is set to anything other than
         1904  +** SQLITE_OK when this function is called, NULL is returned. 
         1905  +**
         1906  +** Otherwise, a copy of (*pOrig) is made into memory obtained from
         1907  +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
         1908  +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
         1909  +*/
         1910  +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
         1911  +  Fts5Colset *pRet;
         1912  +  if( pOrig ){
         1913  +    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
         1914  +    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
         1915  +    if( pRet ){ 
         1916  +      memcpy(pRet, pOrig, nByte);
         1917  +    }
         1918  +  }else{
         1919  +    pRet = 0;
         1920  +  }
         1921  +  return pRet;
         1922  +}
         1923  +
         1924  +/*
         1925  +** Remove from colset pColset any columns that are not also in colset pMerge.
         1926  +*/
         1927  +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
         1928  +  int iIn = 0;          /* Next input in pColset */
         1929  +  int iMerge = 0;       /* Next input in pMerge */
         1930  +  int iOut = 0;         /* Next output slot in pColset */
         1931  +
         1932  +  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
         1933  +    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
         1934  +    if( iDiff==0 ){
         1935  +      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
         1936  +      iMerge++;
         1937  +      iIn++;
         1938  +    }else if( iDiff>0 ){
         1939  +      iMerge++;
         1940  +    }else{
         1941  +      iIn++;
         1942  +    }
         1943  +  }
         1944  +  pColset->nCol = iOut;
         1945  +}
         1946  +
         1947  +/*
         1948  +** Recursively apply colset pColset to expression node pNode and all of
         1949  +** its decendents. If (*ppFree) is not NULL, it contains a spare copy
         1950  +** of pColset. This function may use the spare copy and set (*ppFree) to
         1951  +** zero, or it may create copies of pColset using fts5CloneColset().
         1952  +*/
         1953  +static void fts5ParseSetColset(
         1954  +  Fts5Parse *pParse, 
         1955  +  Fts5ExprNode *pNode, 
         1956  +  Fts5Colset *pColset,
         1957  +  Fts5Colset **ppFree
         1958  +){
         1959  +  if( pParse->rc==SQLITE_OK ){
         1960  +    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
         1961  +         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
         1962  +         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
         1963  +    );
         1964  +    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
         1965  +      Fts5ExprNearset *pNear = pNode->pNear;
         1966  +      if( pNear->pColset ){
         1967  +        fts5MergeColset(pNear->pColset, pColset);
         1968  +        if( pNear->pColset->nCol==0 ){
         1969  +          pNode->eType = FTS5_EOF;
         1970  +          pNode->xNext = 0;
         1971  +        }
         1972  +      }else if( *ppFree ){
         1973  +        pNear->pColset = pColset;
         1974  +        *ppFree = 0;
         1975  +      }else{
         1976  +        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
         1977  +      }
         1978  +    }else{
         1979  +      int i;
         1980  +      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
         1981  +      for(i=0; i<pNode->nChild; i++){
         1982  +        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
         1983  +      }
         1984  +    }
         1985  +  }
         1986  +}
         1987  +
         1988  +/*
         1989  +** Apply colset pColset to expression node pExpr and all of its descendents.
         1990  +*/
  1862   1991   void sqlite3Fts5ParseSetColset(
  1863   1992     Fts5Parse *pParse, 
  1864         -  Fts5ExprNearset *pNear, 
         1993  +  Fts5ExprNode *pExpr, 
  1865   1994     Fts5Colset *pColset 
  1866   1995   ){
         1996  +  Fts5Colset *pFree = pColset;
  1867   1997     if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
  1868   1998       pParse->rc = SQLITE_ERROR;
  1869   1999       pParse->zErr = sqlite3_mprintf(
  1870   2000         "fts5: column queries are not supported (detail=none)"
  1871   2001       );
  1872         -    sqlite3_free(pColset);
  1873         -    return;
  1874         -  }
  1875         -
  1876         -  if( pNear ){
  1877         -    pNear->pColset = pColset;
  1878   2002     }else{
  1879         -    sqlite3_free(pColset);
         2003  +    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
  1880   2004     }
         2005  +  sqlite3_free(pFree);
  1881   2006   }
  1882   2007   
  1883   2008   static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  1884   2009     switch( pNode->eType ){
  1885   2010       case FTS5_STRING: {
  1886   2011         Fts5ExprNearset *pNear = pNode->pNear;
  1887   2012         if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
................................................................................
  2327   2452       azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]);
  2328   2453     }
  2329   2454   
  2330   2455     zExpr = (const char*)sqlite3_value_text(apVal[0]);
  2331   2456   
  2332   2457     rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  2333   2458     if( rc==SQLITE_OK ){
  2334         -    rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
         2459  +    rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr);
  2335   2460     }
  2336   2461     if( rc==SQLITE_OK ){
  2337   2462       char *zText;
  2338   2463       if( pExpr->pRoot->xNext==0 ){
  2339   2464         zText = sqlite3_mprintf("");
  2340   2465       }else if( bTcl ){
  2341   2466         zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);

Changes to ext/fts5/fts5_hash.c.

    32     32     int nSlot;                      /* Size of aSlot[] array */
    33     33     Fts5HashEntry *pScan;           /* Current ordered scan item */
    34     34     Fts5HashEntry **aSlot;          /* Array of hash slots */
    35     35   };
    36     36   
    37     37   /*
    38     38   ** Each entry in the hash table is represented by an object of the 
    39         -** following type. Each object, its key (zKey[]) and its current data
    40         -** are stored in a single memory allocation. The position list data 
    41         -** immediately follows the key data in memory.
           39  +** following type. Each object, its key (a nul-terminated string) and 
           40  +** its current data are stored in a single memory allocation. The 
           41  +** key immediately follows the object in memory. The position list
           42  +** data immediately follows the key data in memory.
    42     43   **
    43     44   ** The data that follows the key is in a similar, but not identical format
    44     45   ** to the doclist data stored in the database. It is:
    45     46   **
    46     47   **   * Rowid, as a varint
    47     48   **   * Position list, without 0x00 terminator.
    48     49   **   * Size of previous position list and rowid, as a 4 byte
................................................................................
    58     59   struct Fts5HashEntry {
    59     60     Fts5HashEntry *pHashNext;       /* Next hash entry with same hash-key */
    60     61     Fts5HashEntry *pScanNext;       /* Next entry in sorted order */
    61     62     
    62     63     int nAlloc;                     /* Total size of allocation */
    63     64     int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
    64     65     int nData;                      /* Total bytes of data (incl. structure) */
    65         -  int nKey;                       /* Length of zKey[] in bytes */
           66  +  int nKey;                       /* Length of key in bytes */
    66     67     u8 bDel;                        /* Set delete-flag @ iSzPoslist */
    67     68     u8 bContent;                    /* Set content-flag (detail=none mode) */
    68     69     i16 iCol;                       /* Column of last value written */
    69     70     int iPos;                       /* Position of last value written */
    70     71     i64 iRowid;                     /* Rowid of last value written */
    71         -  char zKey[8];                   /* Nul-terminated entry key */
    72     72   };
    73     73   
    74     74   /*
    75         -** Size of Fts5HashEntry without the zKey[] array.
           75  +** Eqivalent to:
           76  +**
           77  +**   char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; }
    76     78   */
    77         -#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
    78         -
           79  +#define fts5EntryKey(p) ( ((char *)(&(p)[1])) )
    79     80   
    80     81   
    81     82   /*
    82     83   ** Allocate a new hash table.
    83     84   */
    84     85   int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
    85     86     int rc = SQLITE_OK;
................................................................................
   166    167   
   167    168     apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*));
   168    169     if( !apNew ) return SQLITE_NOMEM;
   169    170     memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));
   170    171   
   171    172     for(i=0; i<pHash->nSlot; i++){
   172    173       while( apOld[i] ){
   173         -      int iHash;
          174  +      unsigned int iHash;
   174    175         Fts5HashEntry *p = apOld[i];
   175    176         apOld[i] = p->pHashNext;
   176         -      iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey));
          177  +      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
          178  +                          (int)strlen(fts5EntryKey(p)));
   177    179         p->pHashNext = apNew[iHash];
   178    180         apNew[iHash] = p;
   179    181       }
   180    182     }
   181    183   
   182    184     sqlite3_free(apOld);
   183    185     pHash->nSlot = nNew;
................................................................................
   240    242     int bNew;                       /* If non-delete entry should be written */
   241    243     
   242    244     bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
   243    245   
   244    246     /* Attempt to locate an existing hash entry */
   245    247     iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   246    248     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   247         -    if( p->zKey[0]==bByte 
          249  +    char *zKey = fts5EntryKey(p);
          250  +    if( zKey[0]==bByte 
   248    251        && p->nKey==nToken
   249         -     && memcmp(&p->zKey[1], pToken, nToken)==0 
          252  +     && memcmp(&zKey[1], pToken, nToken)==0 
   250    253       ){
   251    254         break;
   252    255       }
   253    256     }
   254    257   
   255    258     /* If an existing hash entry cannot be found, create a new one. */
   256    259     if( p==0 ){
   257    260       /* Figure out how much space to allocate */
   258         -    int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
          261  +    char *zKey;
          262  +    int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
   259    263       if( nByte<128 ) nByte = 128;
   260    264   
   261    265       /* Grow the Fts5Hash.aSlot[] array if necessary. */
   262    266       if( (pHash->nEntry*2)>=pHash->nSlot ){
   263    267         int rc = fts5HashResize(pHash);
   264    268         if( rc!=SQLITE_OK ) return rc;
   265    269         iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   266    270       }
   267    271   
   268    272       /* Allocate new Fts5HashEntry and add it to the hash table. */
   269    273       p = (Fts5HashEntry*)sqlite3_malloc(nByte);
   270    274       if( !p ) return SQLITE_NOMEM;
   271         -    memset(p, 0, FTS5_HASHENTRYSIZE);
          275  +    memset(p, 0, sizeof(Fts5HashEntry));
   272    276       p->nAlloc = nByte;
   273         -    p->zKey[0] = bByte;
   274         -    memcpy(&p->zKey[1], pToken, nToken);
   275         -    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
          277  +    zKey = fts5EntryKey(p);
          278  +    zKey[0] = bByte;
          279  +    memcpy(&zKey[1], pToken, nToken);
          280  +    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
   276    281       p->nKey = nToken;
   277         -    p->zKey[nToken+1] = '\0';
   278         -    p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
          282  +    zKey[nToken+1] = '\0';
          283  +    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
   279    284       p->pHashNext = pHash->aSlot[iHash];
   280    285       pHash->aSlot[iHash] = p;
   281    286       pHash->nEntry++;
   282    287   
   283    288       /* Add the first rowid field to the hash-entry */
   284    289       p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
   285    290       p->iRowid = iRowid;
................................................................................
   389    394         *ppOut = p2;
   390    395         p2 = 0;
   391    396       }else if( p2==0 ){
   392    397         *ppOut = p1;
   393    398         p1 = 0;
   394    399       }else{
   395    400         int i = 0;
   396         -      while( p1->zKey[i]==p2->zKey[i] ) i++;
          401  +      char *zKey1 = fts5EntryKey(p1);
          402  +      char *zKey2 = fts5EntryKey(p2);
          403  +      while( zKey1[i]==zKey2[i] ) i++;
   397    404   
   398         -      if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){
          405  +      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
   399    406           /* p2 is smaller */
   400    407           *ppOut = p2;
   401    408           ppOut = &p2->pScanNext;
   402    409           p2 = p2->pScanNext;
   403    410         }else{
   404    411           /* p1 is smaller */
   405    412           *ppOut = p1;
................................................................................
   434    441     ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot);
   435    442     if( !ap ) return SQLITE_NOMEM;
   436    443     memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);
   437    444   
   438    445     for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
   439    446       Fts5HashEntry *pIter;
   440    447       for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
   441         -      if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
          448  +      if( pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm) ){
   442    449           Fts5HashEntry *pEntry = pIter;
   443    450           pEntry->pScanNext = 0;
   444    451           for(i=0; ap[i]; i++){
   445    452             pEntry = fts5HashEntryMerge(pEntry, ap[i]);
   446    453             ap[i] = 0;
   447    454           }
   448    455           ap[i] = pEntry;
................................................................................
   467    474   int sqlite3Fts5HashQuery(
   468    475     Fts5Hash *pHash,                /* Hash table to query */
   469    476     const char *pTerm, int nTerm,   /* Query term */
   470    477     const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
   471    478     int *pnDoclist                  /* OUT: Size of doclist in bytes */
   472    479   ){
   473    480     unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
          481  +  char *zKey = 0;
   474    482     Fts5HashEntry *p;
   475    483   
   476    484     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   477         -    if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
          485  +    zKey = fts5EntryKey(p);
          486  +    if( memcmp(zKey, pTerm, nTerm)==0 && zKey[nTerm]==0 ) break;
   478    487     }
   479    488   
   480    489     if( p ){
   481    490       fts5HashAddPoslistSize(pHash, p);
   482         -    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   483         -    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
          491  +    *ppDoclist = (const u8*)&zKey[nTerm+1];
          492  +    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
   484    493     }else{
   485    494       *ppDoclist = 0;
   486    495       *pnDoclist = 0;
   487    496     }
   488    497   
   489    498     return SQLITE_OK;
   490    499   }
................................................................................
   509    518     Fts5Hash *pHash,
   510    519     const char **pzTerm,            /* OUT: term (nul-terminated) */
   511    520     const u8 **ppDoclist,           /* OUT: pointer to doclist */
   512    521     int *pnDoclist                  /* OUT: size of doclist in bytes */
   513    522   ){
   514    523     Fts5HashEntry *p;
   515    524     if( (p = pHash->pScan) ){
   516         -    int nTerm = (int)strlen(p->zKey);
          525  +    char *zKey = fts5EntryKey(p);
          526  +    int nTerm = (int)strlen(zKey);
   517    527       fts5HashAddPoslistSize(pHash, p);
   518         -    *pzTerm = p->zKey;
   519         -    *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   520         -    *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
          528  +    *pzTerm = zKey;
          529  +    *ppDoclist = (const u8*)&zKey[nTerm+1];
          530  +    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
   521    531     }else{
   522    532       *pzTerm = 0;
   523    533       *ppDoclist = 0;
   524    534       *pnDoclist = 0;
   525    535     }
   526    536   }
   527    537   

Changes to ext/fts5/fts5_index.c.

   624    624     if( p->pReader ){
   625    625       sqlite3_blob *pReader = p->pReader;
   626    626       p->pReader = 0;
   627    627       sqlite3_blob_close(pReader);
   628    628     }
   629    629   }
   630    630   
   631         -
   632    631   /*
   633    632   ** Retrieve a record from the %_data table.
   634    633   **
   635    634   ** If an error occurs, NULL is returned and an error left in the 
   636    635   ** Fts5Index object.
   637    636   */
   638    637   static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
................................................................................
   725    724   static int fts5IndexPrepareStmt(
   726    725     Fts5Index *p,
   727    726     sqlite3_stmt **ppStmt,
   728    727     char *zSql
   729    728   ){
   730    729     if( p->rc==SQLITE_OK ){
   731    730       if( zSql ){
   732         -      p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
          731  +      p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1,
          732  +                                 SQLITE_PREPARE_PERSISTENT, ppStmt, 0);
   733    733       }else{
   734    734         p->rc = SQLITE_NOMEM;
   735    735       }
   736    736     }
   737    737     sqlite3_free(zSql);
   738    738     return p->rc;
   739    739   }
................................................................................
   774    774       char *zSql = sqlite3_mprintf(
   775    775           "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", 
   776    776             pConfig->zDb, pConfig->zName
   777    777       );
   778    778       if( zSql==0 ){
   779    779         rc = SQLITE_NOMEM;
   780    780       }else{
   781         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0);
          781  +      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
          782  +                              SQLITE_PREPARE_PERSISTENT, &p->pDeleter, 0);
   782    783         sqlite3_free(zSql);
   783    784       }
   784    785       if( rc!=SQLITE_OK ){
   785    786         p->rc = rc;
   786    787         return;
   787    788       }
   788    789     }
................................................................................
  2035   2036                 &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
  2036   2037             );
  2037   2038           }
  2038   2039         }
  2039   2040         else if( pLeaf->nn>pLeaf->szLeaf ){
  2040   2041           pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  2041   2042               &pLeaf->p[pLeaf->szLeaf], iOff
  2042         -            );
         2043  +        );
  2043   2044           pIter->iLeafOffset = iOff;
  2044   2045           pIter->iEndofDoclist = iOff;
  2045   2046           bNewTerm = 1;
  2046   2047         }
  2047   2048         assert_nc( iOff<pLeaf->szLeaf );
  2048   2049         if( iOff>pLeaf->szLeaf ){
  2049   2050           p->rc = FTS5_CORRUPT;
................................................................................
  2069   2070         ** code is inlined. 
  2070   2071         **
  2071   2072         ** Later: Switched back to fts5SegIterLoadNPos() because it supports
  2072   2073         ** detail=none mode. Not ideal.
  2073   2074         */
  2074   2075         int nSz;
  2075   2076         assert( p->rc==SQLITE_OK );
         2077  +      assert( pIter->iLeafOffset<=pIter->pLeaf->nn );
  2076   2078         fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
  2077   2079         pIter->bDel = (nSz & 0x0001);
  2078   2080         pIter->nPos = nSz>>1;
  2079   2081         assert_nc( pIter->nPos>=0 );
  2080   2082       }
  2081   2083     }
  2082   2084   }
................................................................................
  2836   2838   static void fts5MultiIterNext(
  2837   2839     Fts5Index *p, 
  2838   2840     Fts5Iter *pIter,
  2839   2841     int bFrom,                      /* True if argument iFrom is valid */
  2840   2842     i64 iFrom                       /* Advance at least as far as this */
  2841   2843   ){
  2842   2844     int bUseFrom = bFrom;
         2845  +  assert( pIter->base.bEof==0 );
  2843   2846     while( p->rc==SQLITE_OK ){
  2844   2847       int iFirst = pIter->aFirst[1].iFirst;
  2845   2848       int bNewTerm = 0;
  2846   2849       Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2847   2850       assert( p->rc==SQLITE_OK );
  2848   2851       if( bUseFrom && pSeg->pDlidx ){
  2849   2852         fts5SegIterNextFrom(p, pSeg, iFrom);
................................................................................
  2873   2876   static void fts5MultiIterNext2(
  2874   2877     Fts5Index *p, 
  2875   2878     Fts5Iter *pIter,
  2876   2879     int *pbNewTerm                  /* OUT: True if *might* be new term */
  2877   2880   ){
  2878   2881     assert( pIter->bSkipEmpty );
  2879   2882     if( p->rc==SQLITE_OK ){
  2880         -    do {
         2883  +    *pbNewTerm = 0;
         2884  +    do{
  2881   2885         int iFirst = pIter->aFirst[1].iFirst;
  2882   2886         Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2883   2887         int bNewTerm = 0;
  2884   2888   
  2885   2889         assert( p->rc==SQLITE_OK );
  2886   2890         pSeg->xNext(p, pSeg, &bNewTerm);
  2887   2891         if( pSeg->pLeaf==0 || bNewTerm 
  2888   2892          || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
  2889   2893         ){
  2890   2894           fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2891   2895           fts5MultiIterSetEof(pIter);
  2892   2896           *pbNewTerm = 1;
  2893         -      }else{
  2894         -        *pbNewTerm = 0;
  2895   2897         }
  2896   2898         fts5AssertMultiIterSetup(p, pIter);
  2897   2899   
  2898   2900       }while( fts5MultiIterIsEmpty(p, pIter) );
  2899   2901     }
  2900   2902   }
  2901   2903   
................................................................................
  3062   3064       xChunk(p, pCtx, pChunk, nChunk);
  3063   3065       nRem -= nChunk;
  3064   3066       fts5DataRelease(pData);
  3065   3067       if( nRem<=0 ){
  3066   3068         break;
  3067   3069       }else{
  3068   3070         pgno++;
  3069         -      pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
         3071  +      pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno));
  3070   3072         if( pData==0 ) break;
  3071   3073         pChunk = &pData->p[4];
  3072   3074         nChunk = MIN(nRem, pData->szLeaf - 4);
  3073   3075         if( pgno==pgnoSave ){
  3074   3076           assert( pSeg->pNextLeaf==0 );
  3075   3077           pSeg->pNextLeaf = pData;
  3076   3078           pData = 0;
................................................................................
  3153   3155     while( p<pEnd && *p!=0x01 ){
  3154   3156       while( *p++ & 0x80 );
  3155   3157     }
  3156   3158   
  3157   3159     return p - (*pa);
  3158   3160   }
  3159   3161   
  3160         -static int fts5IndexExtractColset (
         3162  +static void fts5IndexExtractColset(
         3163  +  int *pRc,
  3161   3164     Fts5Colset *pColset,            /* Colset to filter on */
  3162   3165     const u8 *pPos, int nPos,       /* Position list */
  3163   3166     Fts5Buffer *pBuf                /* Output buffer */
  3164   3167   ){
  3165         -  int rc = SQLITE_OK;
  3166         -  int i;
  3167         -
  3168         -  fts5BufferZero(pBuf);
  3169         -  for(i=0; i<pColset->nCol; i++){
  3170         -    const u8 *pSub = pPos;
  3171         -    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
  3172         -    if( nSub ){
  3173         -      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
         3168  +  if( *pRc==SQLITE_OK ){
         3169  +    int i;
         3170  +    fts5BufferZero(pBuf);
         3171  +    for(i=0; i<pColset->nCol; i++){
         3172  +      const u8 *pSub = pPos;
         3173  +      int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
         3174  +      if( nSub ){
         3175  +        fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
         3176  +      }
  3174   3177       }
  3175   3178     }
  3176         -  return rc;
  3177   3179   }
  3178   3180   
  3179   3181   /*
  3180   3182   ** xSetOutputs callback used by detail=none tables.
  3181   3183   */
  3182   3184   static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
  3183   3185     assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
................................................................................
  3293   3295       /* All data is stored on the current page. Populate the output 
  3294   3296       ** variables to point into the body of the page object. */
  3295   3297       const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  3296   3298       if( pColset->nCol==1 ){
  3297   3299         pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
  3298   3300         pIter->base.pData = a;
  3299   3301       }else{
         3302  +      int *pRc = &pIter->pIndex->rc;
  3300   3303         fts5BufferZero(&pIter->poslist);
  3301         -      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
         3304  +      fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
  3302   3305         pIter->base.pData = pIter->poslist.p;
  3303   3306         pIter->base.nData = pIter->poslist.n;
  3304   3307       }
  3305   3308     }else{
  3306   3309       /* The data is distributed over two or more pages. Copy it into the
  3307   3310       ** Fts5Iter.poslist buffer and then set the output pointer to point
  3308   3311       ** to this buffer.  */
................................................................................
  3839   3842   }
  3840   3843   
  3841   3844   static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
  3842   3845     static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
  3843   3846     Fts5PageWriter *pPage = &pWriter->writer;
  3844   3847     i64 iRowid;
  3845   3848   
  3846         -static int nCall = 0;
  3847         -nCall++;
  3848         -
  3849   3849     assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
  3850   3850   
  3851   3851     /* Set the szLeaf header field. */
  3852   3852     assert( 0==fts5GetU16(&pPage->buf.p[2]) );
  3853   3853     fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
  3854   3854   
  3855   3855     if( pWriter->bFirstTermInPage ){
................................................................................
  4190   4190     int nInput;                     /* Number of input segments */
  4191   4191     Fts5SegWriter writer;           /* Writer object */
  4192   4192     Fts5StructureSegment *pSeg;     /* Output segment */
  4193   4193     Fts5Buffer term;
  4194   4194     int bOldest;                    /* True if the output segment is the oldest */
  4195   4195     int eDetail = p->pConfig->eDetail;
  4196   4196     const int flags = FTS5INDEX_QUERY_NOOUTPUT;
         4197  +  int bTermWritten = 0;           /* True if current term already output */
  4197   4198   
  4198   4199     assert( iLvl<pStruct->nLevel );
  4199   4200     assert( pLvl->nMerge<=pLvl->nSeg );
  4200   4201   
  4201   4202     memset(&writer, 0, sizeof(Fts5SegWriter));
  4202   4203     memset(&term, 0, sizeof(Fts5Buffer));
  4203   4204     if( pLvl->nMerge ){
................................................................................
  4243   4244         fts5MultiIterNext(p, pIter, 0, 0)
  4244   4245     ){
  4245   4246       Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  4246   4247       int nPos;                     /* position-list size field value */
  4247   4248       int nTerm;
  4248   4249       const u8 *pTerm;
  4249   4250   
  4250         -    /* Check for key annihilation. */
  4251         -    if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue;
  4252         -
  4253   4251       pTerm = fts5MultiIterTerm(pIter, &nTerm);
  4254   4252       if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){
  4255   4253         if( pnRem && writer.nLeafWritten>nRem ){
  4256   4254           break;
  4257   4255         }
         4256  +      fts5BufferSet(&p->rc, &term, nTerm, pTerm);
         4257  +      bTermWritten =0;
         4258  +    }
  4258   4259   
         4260  +    /* Check for key annihilation. */
         4261  +    if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue;
         4262  +
         4263  +    if( p->rc==SQLITE_OK && bTermWritten==0 ){
  4259   4264         /* This is a new term. Append a term to the output segment. */
  4260   4265         fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
  4261         -      fts5BufferSet(&p->rc, &term, nTerm, pTerm);
         4266  +      bTermWritten = 1;
  4262   4267       }
  4263   4268   
  4264   4269       /* Append the rowid to the output */
  4265   4270       /* WRITEPOSLISTSIZE */
  4266   4271       fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));
  4267   4272   
  4268   4273       if( eDetail==FTS5_DETAIL_NONE ){
................................................................................
  5086   5091       }
  5087   5092       fts5MultiIterFree(p1);
  5088   5093   
  5089   5094       pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
  5090   5095       if( pData ){
  5091   5096         pData->p = (u8*)&pData[1];
  5092   5097         pData->nn = pData->szLeaf = doclist.n;
  5093         -      memcpy(pData->p, doclist.p, doclist.n);
         5098  +      if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n);
  5094   5099         fts5MultiIterNew2(p, pData, bDesc, ppIter);
  5095   5100       }
  5096   5101       fts5BufferFree(&doclist);
  5097   5102     }
  5098   5103   
  5099   5104     fts5StructureRelease(pStruct);
  5100   5105     sqlite3_free(aBuf);
................................................................................
  5125   5130     p->bDelete = bDelete;
  5126   5131     return fts5IndexReturn(p);
  5127   5132   }
  5128   5133   
  5129   5134   /*
  5130   5135   ** Commit data to disk.
  5131   5136   */
  5132         -int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
         5137  +int sqlite3Fts5IndexSync(Fts5Index *p){
  5133   5138     assert( p->rc==SQLITE_OK );
  5134   5139     fts5IndexFlush(p);
  5135         -  if( bCommit ) fts5CloseReader(p);
         5140  +  fts5CloseReader(p);
  5136   5141     return fts5IndexReturn(p);
  5137   5142   }
  5138   5143   
  5139   5144   /*
  5140   5145   ** Discard any data stored in the in-memory hash tables. Do not write it
  5141   5146   ** to the database. Additionally, assume that the contents of the %_data
  5142   5147   ** table may have changed on disk. So any in-memory caches of %_data 
................................................................................
  5325   5330     Fts5Buffer buf = {0, 0, 0};
  5326   5331   
  5327   5332     /* If the QUERY_SCAN flag is set, all other flags must be clear. */
  5328   5333     assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN );
  5329   5334   
  5330   5335     if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
  5331   5336       int iIdx = 0;                 /* Index to search */
  5332         -    memcpy(&buf.p[1], pToken, nToken);
         5337  +    if( nToken ) memcpy(&buf.p[1], pToken, nToken);
  5333   5338   
  5334   5339       /* Figure out which index to search and set iIdx accordingly. If this
  5335   5340       ** is a prefix query for which there is no prefix index, set iIdx to
  5336   5341       ** greater than pConfig->nPrefix to indicate that the query will be
  5337   5342       ** satisfied by scanning multiple terms in the main index.
  5338   5343       **
  5339   5344       ** If the QUERY_TEST_NOIDX flag was specified, then this must be a
................................................................................
  5374   5379         if( p->rc==SQLITE_OK ){
  5375   5380           Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
  5376   5381           if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
  5377   5382         }
  5378   5383       }
  5379   5384   
  5380   5385       if( p->rc ){
  5381         -      sqlite3Fts5IterClose(&pRet->base);
         5386  +      sqlite3Fts5IterClose((Fts5IndexIter*)pRet);
  5382   5387         pRet = 0;
  5383   5388         fts5CloseReader(p);
  5384   5389       }
  5385   5390   
  5386   5391       *ppIter = &pRet->base;
  5387   5392       sqlite3Fts5BufferFree(&buf);
  5388   5393     }
................................................................................
  5824   5829       int iIdxLeaf = sqlite3_column_int(pStmt, 2);
  5825   5830       int bIdxDlidx = sqlite3_column_int(pStmt, 3);
  5826   5831   
  5827   5832       /* If the leaf in question has already been trimmed from the segment, 
  5828   5833       ** ignore this b-tree entry. Otherwise, load it into memory. */
  5829   5834       if( iIdxLeaf<pSeg->pgnoFirst ) continue;
  5830   5835       iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf);
  5831         -    pLeaf = fts5DataRead(p, iRow);
         5836  +    pLeaf = fts5LeafRead(p, iRow);
  5832   5837       if( pLeaf==0 ) break;
  5833   5838   
  5834   5839       /* Check that the leaf contains at least one term, and that it is equal
  5835   5840       ** to or larger than the split-key in zIdxTerm.  Also check that if there
  5836   5841       ** is also a rowid pointer within the leaf page header, it points to a
  5837   5842       ** location before the term.  */
  5838   5843       if( pLeaf->nn<=pLeaf->szLeaf ){

Changes to ext/fts5/fts5_main.c.

   502    502   **       * An == rowid constraint:       cost=10.0
   503    503   **
   504    504   ** Costs are not modified by the ORDER BY clause.
   505    505   */
   506    506   static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   507    507     Fts5Table *pTab = (Fts5Table*)pVTab;
   508    508     Fts5Config *pConfig = pTab->pConfig;
          509  +  const int nCol = pConfig->nCol;
   509    510     int idxFlags = 0;               /* Parameter passed through to xFilter() */
   510    511     int bHasMatch;
   511    512     int iNext;
   512    513     int i;
   513    514   
   514    515     struct Constraint {
   515    516       int op;                       /* Mask against sqlite3_index_constraint.op */
................................................................................
   527    528                                       FTS5_BI_ROWID_LE, 0, 0, -1},
   528    529       {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, 
   529    530                                       FTS5_BI_ROWID_GE, 0, 0, -1},
   530    531     };
   531    532   
   532    533     int aColMap[3];
   533    534     aColMap[0] = -1;
   534         -  aColMap[1] = pConfig->nCol;
   535         -  aColMap[2] = pConfig->nCol+1;
          535  +  aColMap[1] = nCol;
          536  +  aColMap[2] = nCol+1;
   536    537   
   537    538     /* Set idxFlags flags for all WHERE clause terms that will be used. */
   538    539     for(i=0; i<pInfo->nConstraint; i++){
   539    540       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   540         -    int j;
   541         -    for(j=0; j<ArraySize(aConstraint); j++){
   542         -      struct Constraint *pC = &aConstraint[j];
   543         -      if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
   544         -        if( p->usable ){
          541  +    int iCol = p->iColumn;
          542  +
          543  +    if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
          544  +     || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol)
          545  +    ){
          546  +      /* A MATCH operator or equivalent */
          547  +      if( p->usable ){
          548  +        idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16);
          549  +        aConstraint[0].iConsIndex = i;
          550  +      }else{
          551  +        /* As there exists an unusable MATCH constraint this is an 
          552  +        ** unusable plan. Set a prohibitively high cost. */
          553  +        pInfo->estimatedCost = 1e50;
          554  +        return SQLITE_OK;
          555  +      }
          556  +    }else{
          557  +      int j;
          558  +      for(j=1; j<ArraySize(aConstraint); j++){
          559  +        struct Constraint *pC = &aConstraint[j];
          560  +        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
   545    561             pC->iConsIndex = i;
   546    562             idxFlags |= pC->fts5op;
   547         -        }else if( j==0 ){
   548         -          /* As there exists an unusable MATCH constraint this is an 
   549         -          ** unusable plan. Set a prohibitively high cost. */
   550         -          pInfo->estimatedCost = 1e50;
   551         -          return SQLITE_OK;
   552    563           }
   553    564         }
   554    565       }
   555    566     }
   556    567   
   557    568     /* Set idxFlags flags for the ORDER BY clause */
   558    569     if( pInfo->nOrderBy==1 ){
................................................................................
   868    879     va_list ap;
   869    880   
   870    881     va_start(ap, zFmt);
   871    882     zSql = sqlite3_vmprintf(zFmt, ap);
   872    883     if( zSql==0 ){
   873    884       rc = SQLITE_NOMEM; 
   874    885     }else{
   875         -    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
          886  +    rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 
          887  +                            SQLITE_PREPARE_PERSISTENT, &pRet, 0);
   876    888       if( rc!=SQLITE_OK ){
   877    889         *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
   878    890       }
   879    891       sqlite3_free(zSql);
   880    892     }
   881    893   
   882    894     va_end(ap);
................................................................................
  1004   1016     const char *zRank = pCsr->zRank;
  1005   1017     const char *zRankArgs = pCsr->zRankArgs;
  1006   1018   
  1007   1019     if( zRankArgs ){
  1008   1020       char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
  1009   1021       if( zSql ){
  1010   1022         sqlite3_stmt *pStmt = 0;
  1011         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0);
         1023  +      rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
         1024  +                              SQLITE_PREPARE_PERSISTENT, &pStmt, 0);
  1012   1025         sqlite3_free(zSql);
  1013   1026         assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
  1014   1027         if( rc==SQLITE_OK ){
  1015   1028           if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1016   1029             int nByte;
  1017   1030             pCsr->nRankArg = sqlite3_column_count(pStmt);
  1018   1031             nByte = sizeof(sqlite3_value*)*pCsr->nRankArg;
................................................................................
  1119   1132     int bDesc;                      /* True if ORDER BY [rank|rowid] DESC */
  1120   1133     int bOrderByRank;               /* True if ORDER BY rank */
  1121   1134     sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  1122   1135     sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  1123   1136     sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  1124   1137     sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  1125   1138     sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
         1139  +  int iCol;                       /* Column on LHS of MATCH operator */
  1126   1140     char **pzErrmsg = pConfig->pzErrmsg;
  1127   1141   
  1128   1142     UNUSED_PARAM(zUnused);
  1129   1143     UNUSED_PARAM(nVal);
  1130   1144   
  1131   1145     if( pCsr->ePlan ){
  1132   1146       fts5FreeCursorComponents(pCsr);
................................................................................
  1149   1163     ** order as the corresponding entries in the struct at the top of
  1150   1164     ** fts5BestIndexMethod().  */
  1151   1165     if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++];
  1152   1166     if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++];
  1153   1167     if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++];
  1154   1168     if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++];
  1155   1169     if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++];
         1170  +  iCol = (idxNum>>16);
         1171  +  assert( iCol>=0 && iCol<=pConfig->nCol );
  1156   1172     assert( iVal==nVal );
  1157   1173     bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
  1158   1174     pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
  1159   1175   
  1160   1176     /* Set the cursor upper and lower rowid limits. Only some strategies 
  1161   1177     ** actually use them. This is ok, as the xBestIndex() method leaves the
  1162   1178     ** sqlite3_index_constraint.omit flag clear for range constraints
................................................................................
  1195   1211         if( zExpr[0]=='*' ){
  1196   1212           /* The user has issued a query of the form "MATCH '*...'". This
  1197   1213           ** indicates that the MATCH expression is not a full text query,
  1198   1214           ** but a request for an internal parameter.  */
  1199   1215           rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
  1200   1216         }else{
  1201   1217           char **pzErr = &pTab->base.zErrMsg;
  1202         -        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
         1218  +        rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr);
  1203   1219           if( rc==SQLITE_OK ){
  1204   1220             if( bOrderByRank ){
  1205   1221               pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
  1206   1222               rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
  1207   1223             }else{
  1208   1224               pCsr->ePlan = FTS5_PLAN_MATCH;
  1209   1225               rc = fts5CursorFirst(pTab, pCsr, bDesc);
................................................................................
  1575   1591   */
  1576   1592   static int fts5SyncMethod(sqlite3_vtab *pVtab){
  1577   1593     int rc;
  1578   1594     Fts5Table *pTab = (Fts5Table*)pVtab;
  1579   1595     fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
  1580   1596     pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
  1581   1597     fts5TripCursors(pTab);
  1582         -  rc = sqlite3Fts5StorageSync(pTab->pStorage, 1);
         1598  +  rc = sqlite3Fts5StorageSync(pTab->pStorage);
  1583   1599     pTab->pConfig->pzErrmsg = 0;
  1584   1600     return rc;
  1585   1601   }
  1586   1602   
  1587   1603   /*
  1588   1604   ** Implementation of xBegin() method. 
  1589   1605   */
................................................................................
  2386   2402   ** Flush the contents of the pending-terms table to disk.
  2387   2403   */
  2388   2404   static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2389   2405     Fts5Table *pTab = (Fts5Table*)pVtab;
  2390   2406     UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  2391   2407     fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
  2392   2408     fts5TripCursors(pTab);
  2393         -  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
         2409  +  return sqlite3Fts5StorageSync(pTab->pStorage);
  2394   2410   }
  2395   2411   
  2396   2412   /*
  2397   2413   ** The xRelease() method.
  2398   2414   **
  2399   2415   ** This is a no-op.
  2400   2416   */
  2401   2417   static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2402   2418     Fts5Table *pTab = (Fts5Table*)pVtab;
  2403   2419     UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  2404   2420     fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
  2405   2421     fts5TripCursors(pTab);
  2406         -  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
         2422  +  return sqlite3Fts5StorageSync(pTab->pStorage);
  2407   2423   }
  2408   2424   
  2409   2425   /*
  2410   2426   ** The xRollbackTo() method.
  2411   2427   **
  2412   2428   ** Discard the contents of the pending terms table.
  2413   2429   */
................................................................................
  2589   2605   
  2590   2606     sqlite3_free(pGlobal);
  2591   2607   }
  2592   2608   
  2593   2609   static void fts5Fts5Func(
  2594   2610     sqlite3_context *pCtx,          /* Function call context */
  2595   2611     int nArg,                       /* Number of args */
  2596         -  sqlite3_value **apUnused        /* Function arguments */
         2612  +  sqlite3_value **apArg           /* Function arguments */
  2597   2613   ){
  2598   2614     Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
  2599         -  char buf[8];
  2600         -  UNUSED_PARAM2(nArg, apUnused);
  2601         -  assert( nArg==0 );
  2602         -  assert( sizeof(buf)>=sizeof(pGlobal) );
  2603         -  memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
  2604         -  sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
         2615  +  fts5_api **ppApi;
         2616  +  UNUSED_PARAM(nArg);
         2617  +  assert( nArg==1 );
         2618  +  ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr");
         2619  +  if( ppApi ) *ppApi = &pGlobal->api;
  2605   2620   }
  2606   2621   
  2607   2622   /*
  2608   2623   ** Implementation of fts5_source_id() function.
  2609   2624   */
  2610   2625   static void fts5SourceIdFunc(
  2611   2626     sqlite3_context *pCtx,          /* Function call context */
................................................................................
  2662   2677       if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
  2663   2678       if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
  2664   2679       if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
  2665   2680       if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
  2666   2681       if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
  2667   2682       if( rc==SQLITE_OK ){
  2668   2683         rc = sqlite3_create_function(
  2669         -          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
         2684  +          db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
  2670   2685         );
  2671   2686       }
  2672   2687       if( rc==SQLITE_OK ){
  2673   2688         rc = sqlite3_create_function(
  2674   2689             db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0
  2675   2690         );
  2676   2691       }

Changes to ext/fts5/fts5_storage.c.

   132    132           zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
   133    133           break;
   134    134       }
   135    135   
   136    136       if( zSql==0 ){
   137    137         rc = SQLITE_NOMEM;
   138    138       }else{
   139         -      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
          139  +      rc = sqlite3_prepare_v3(pC->db, zSql, -1,
          140  +                              SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0);
   140    141         sqlite3_free(zSql);
   141    142         if( rc!=SQLITE_OK && pzErrMsg ){
   142    143           *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
   143    144         }
   144    145       }
   145    146     }
   146    147   
................................................................................
   214    215           pConfig->zDb, pConfig->zName, zTail, zName, zTail
   215    216       );
   216    217     }
   217    218   }
   218    219   
   219    220   int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
   220    221     Fts5Config *pConfig = pStorage->pConfig;
   221         -  int rc = sqlite3Fts5StorageSync(pStorage, 1);
          222  +  int rc = sqlite3Fts5StorageSync(pStorage);
   222    223   
   223    224     fts5StorageRenameOne(pConfig, &rc, "data", zName);
   224    225     fts5StorageRenameOne(pConfig, &rc, "idx", zName);
   225    226     fts5StorageRenameOne(pConfig, &rc, "config", zName);
   226    227     if( pConfig->bColumnsize ){
   227    228       fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
   228    229     }
................................................................................
   541    542       if( rc==SQLITE_OK ){
   542    543         sqlite3_bind_int64(pDel, 1, iDel);
   543    544         sqlite3_step(pDel);
   544    545         rc = sqlite3_reset(pDel);
   545    546       }
   546    547     }
   547    548   
   548         -  /* Write the averages record */
   549         -  if( rc==SQLITE_OK ){
   550         -    rc = fts5StorageSaveTotals(p);
   551         -  }
   552         -
   553    549     return rc;
   554    550   }
   555    551   
   556    552   /*
   557    553   ** Delete all entries in the FTS5 index.
   558    554   */
   559    555   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
................................................................................
   749    745   
   750    746     /* Write the %_docsize record */
   751    747     if( rc==SQLITE_OK ){
   752    748       rc = fts5StorageInsertDocsize(p, iRowid, &buf);
   753    749     }
   754    750     sqlite3_free(buf.p);
   755    751   
   756         -  /* Write the averages record */
   757         -  if( rc==SQLITE_OK ){
   758         -    rc = fts5StorageSaveTotals(p);
   759         -  }
   760         -
   761    752     return rc;
   762    753   }
   763    754   
   764    755   static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
   765    756     Fts5Config *pConfig = p->pConfig;
   766    757     char *zSql;
   767    758     int rc;
................................................................................
  1087   1078     }
  1088   1079     return rc;
  1089   1080   }
  1090   1081   
  1091   1082   /*
  1092   1083   ** Flush any data currently held in-memory to disk.
  1093   1084   */
  1094         -int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
  1095         -  if( bCommit && p->bTotalsValid ){
  1096         -    int rc = fts5StorageSaveTotals(p);
         1085  +int sqlite3Fts5StorageSync(Fts5Storage *p){
         1086  +  int rc = SQLITE_OK;
         1087  +  i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
         1088  +  if( p->bTotalsValid ){
         1089  +    rc = fts5StorageSaveTotals(p);
  1097   1090       p->bTotalsValid = 0;
  1098         -    if( rc!=SQLITE_OK ) return rc;
         1091  +  }
         1092  +  if( rc==SQLITE_OK ){
         1093  +    rc = sqlite3Fts5IndexSync(p->pIndex);
  1099   1094     }
  1100         -  return sqlite3Fts5IndexSync(p->pIndex, bCommit);
         1095  +  sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
         1096  +  return rc;
  1101   1097   }
  1102   1098   
  1103   1099   int sqlite3Fts5StorageRollback(Fts5Storage *p){
  1104   1100     p->bTotalsValid = 0;
  1105   1101     return sqlite3Fts5IndexRollback(p->pIndex);
  1106   1102   }
  1107   1103   

Changes to ext/fts5/fts5_tcl.c.

    95     95     int rc = f5tDbPointer(interp, pObj, &db);
    96     96     if( rc!=TCL_OK ){
    97     97       return TCL_ERROR;
    98     98     }else{
    99     99       sqlite3_stmt *pStmt = 0;
   100    100       fts5_api *pApi = 0;
   101    101   
   102         -    rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0);
          102  +    rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
   103    103       if( rc!=SQLITE_OK ){
   104    104         Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
   105    105         return TCL_ERROR;
   106    106       }
   107         -
   108         -    if( SQLITE_ROW==sqlite3_step(pStmt) ){
   109         -      const void *pPtr = sqlite3_column_blob(pStmt, 0);
   110         -      memcpy((void*)&pApi, pPtr, sizeof(pApi));
   111         -    }
          107  +    sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
          108  +    sqlite3_step(pStmt);
   112    109   
   113    110       if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
   114    111         Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
   115    112         return TCL_ERROR;
   116    113       }
   117    114   
   118    115       *ppDb = db;

Changes to ext/fts5/fts5_test_mi.c.

    69     69   ** handle (accessible using sqlite3_errcode()/errmsg()).
    70     70   */
    71     71   static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
    72     72     sqlite3_stmt *pStmt = 0;
    73     73     int rc;
    74     74   
    75     75     *ppApi = 0;
    76         -  rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
           76  +  rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
    77     77     if( rc==SQLITE_OK ){
    78         -    if( SQLITE_ROW==sqlite3_step(pStmt) 
    79         -        && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
    80         -      ){
    81         -      memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
    82         -    }
           78  +    sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
           79  +    (void)sqlite3_step(pStmt);
    83     80       rc = sqlite3_finalize(pStmt);
    84     81     }
    85     82   
    86     83     return rc;
    87     84   }
    88     85   
    89     86   
................................................................................
   418    415     /* Register the implementation of matchinfo() */
   419    416     rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
   420    417   
   421    418     return rc;
   422    419   }
   423    420   
   424    421   #endif /* SQLITE_ENABLE_FTS5 */
   425         -

Changes to ext/fts5/fts5_test_tok.c.

    36     36   **   end:     Byte offset of the byte immediately following the end of the
    37     37   **            token within the input string.
    38     38   **   pos:     Token offset of token within input.
    39     39   **
    40     40   */
    41     41   #if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)
    42     42   
    43         -#include <fts5.h>
           43  +#include "fts5.h"
    44     44   #include <string.h>
    45     45   #include <assert.h>
    46     46   
    47     47   typedef struct Fts5tokTable Fts5tokTable;
    48     48   typedef struct Fts5tokCursor Fts5tokCursor;
    49     49   typedef struct Fts5tokRow Fts5tokRow;
    50     50   
................................................................................
   178    178     sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
   179    179     char **pzErr                    /* OUT: sqlite3_malloc'd error message */
   180    180   ){
   181    181     fts5_api *pApi = (fts5_api*)pCtx;
   182    182     Fts5tokTable *pTab = 0;
   183    183     int rc;
   184    184     char **azDequote = 0;
   185         -  int nDequote;
          185  +  int nDequote = 0;
   186    186   
   187    187     rc = sqlite3_declare_vtab(db, 
   188    188          "CREATE TABLE x(input HIDDEN, token, start, end, position)"
   189    189     );
   190    190   
   191    191     if( rc==SQLITE_OK ){
   192    192       nDequote = argc-3;

Changes to ext/fts5/fts5_vocab.c.

    25     25   ** row:
    26     26   **     CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
    27     27   **
    28     28   **   One row for each term in the database. The value of $doc is set to
    29     29   **   the number of fts5 rows that contain at least one instance of term
    30     30   **   $term. Field $cnt is set to the total number of instances of term 
    31     31   **   $term in the database.
           32  +**
           33  +** instance:
           34  +**     CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
           35  +**
           36  +**   One row for each term instance in the database. 
    32     37   */
    33     38   
    34     39   
    35     40   #include "fts5Int.h"
    36     41   
    37     42   
    38     43   typedef struct Fts5VocabTable Fts5VocabTable;
................................................................................
    40     45   
    41     46   struct Fts5VocabTable {
    42     47     sqlite3_vtab base;
    43     48     char *zFts5Tbl;                 /* Name of fts5 table */
    44     49     char *zFts5Db;                  /* Db containing fts5 table */
    45     50     sqlite3 *db;                    /* Database handle */
    46     51     Fts5Global *pGlobal;            /* FTS5 global object for this database */
    47         -  int eType;                      /* FTS5_VOCAB_COL or ROW */
           52  +  int eType;                      /* FTS5_VOCAB_COL, ROW or INSTANCE */
    48     53   };
    49     54   
    50     55   struct Fts5VocabCursor {
    51     56     sqlite3_vtab_cursor base;
    52     57     sqlite3_stmt *pStmt;            /* Statement holding lock on pIndex */
    53     58     Fts5Index *pIndex;              /* Associated FTS5 index */
    54     59   
................................................................................
    60     65   
    61     66     /* These are used by 'col' tables only */
    62     67     Fts5Config *pConfig;            /* Fts5 table configuration */
    63     68     int iCol;
    64     69     i64 *aCnt;
    65     70     i64 *aDoc;
    66     71   
    67         -  /* Output values used by 'row' and 'col' tables */
           72  +  /* Output values used by all tables. */
    68     73     i64 rowid;                      /* This table's current rowid value */
    69     74     Fts5Buffer term;                /* Current value of 'term' column */
           75  +
           76  +  /* Output values Used by 'instance' tables only */
           77  +  i64 iInstPos;
           78  +  int iInstOff;
    70     79   };
    71     80   
    72         -#define FTS5_VOCAB_COL    0
    73         -#define FTS5_VOCAB_ROW    1
           81  +#define FTS5_VOCAB_COL      0
           82  +#define FTS5_VOCAB_ROW      1
           83  +#define FTS5_VOCAB_INSTANCE 2
    74     84   
    75     85   #define FTS5_VOCAB_COL_SCHEMA  "term, col, doc, cnt"
    76     86   #define FTS5_VOCAB_ROW_SCHEMA  "term, doc, cnt"
           87  +#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset"
    77     88   
    78     89   /*
    79     90   ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
    80     91   */
    81     92   #define FTS5_VOCAB_TERM_EQ 0x01
    82     93   #define FTS5_VOCAB_TERM_GE 0x02
    83     94   #define FTS5_VOCAB_TERM_LE 0x04
................................................................................
    96    107       sqlite3Fts5Dequote(zCopy);
    97    108       if( sqlite3_stricmp(zCopy, "col")==0 ){
    98    109         *peType = FTS5_VOCAB_COL;
    99    110       }else
   100    111   
   101    112       if( sqlite3_stricmp(zCopy, "row")==0 ){
   102    113         *peType = FTS5_VOCAB_ROW;
          114  +    }else
          115  +    if( sqlite3_stricmp(zCopy, "instance")==0 ){
          116  +      *peType = FTS5_VOCAB_INSTANCE;
   103    117       }else
   104    118       {
   105    119         *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
   106    120         rc = SQLITE_ERROR;
   107    121       }
   108    122       sqlite3_free(zCopy);
   109    123     }
................................................................................
   157    171     int argc,                       /* Number of elements in argv array */
   158    172     const char * const *argv,       /* xCreate/xConnect argument array */
   159    173     sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
   160    174     char **pzErr                    /* Write any error message here */
   161    175   ){
   162    176     const char *azSchema[] = { 
   163    177       "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA  ")", 
   164         -    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")"
          178  +    "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA  ")",
          179  +    "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")"
   165    180     };
   166    181   
   167    182     Fts5VocabTable *pRet = 0;
   168    183     int rc = SQLITE_OK;             /* Return code */
   169    184     int bDb;
   170    185   
   171    186     bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
................................................................................
   231    246     char **pzErr                    /* OUT: sqlite3_malloc'd error message */
   232    247   ){
   233    248     return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
   234    249   }
   235    250   
   236    251   /* 
   237    252   ** Implementation of the xBestIndex method.
          253  +**
          254  +** Only constraints of the form:
          255  +**
          256  +**     term <= ?
          257  +**     term == ?
          258  +**     term >= ?
          259  +**
          260  +** are interpreted. Less-than and less-than-or-equal are treated 
          261  +** identically, as are greater-than and greater-than-or-equal.
   238    262   */
   239    263   static int fts5VocabBestIndexMethod(
   240    264     sqlite3_vtab *pUnused,
   241    265     sqlite3_index_info *pInfo
   242    266   ){
   243    267     int i;
   244    268     int iTermEq = -1;
................................................................................
   374    398     fts5VocabResetCursor(pCsr);
   375    399     sqlite3Fts5BufferFree(&pCsr->term);
   376    400     sqlite3_finalize(pCsr->pStmt);
   377    401     sqlite3_free(pCsr);
   378    402     return SQLITE_OK;
   379    403   }
   380    404   
          405  +static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
          406  +  int rc = SQLITE_OK;
          407  +  
          408  +  if( sqlite3Fts5IterEof(pCsr->pIter) ){
          409  +    pCsr->bEof = 1;
          410  +  }else{
          411  +    const char *zTerm;
          412  +    int nTerm;
          413  +    zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
          414  +    if( pCsr->nLeTerm>=0 ){
          415  +      int nCmp = MIN(nTerm, pCsr->nLeTerm);
          416  +      int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
          417  +      if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
          418  +        pCsr->bEof = 1;
          419  +      }
          420  +    }
          421  +
          422  +    sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
          423  +  }
          424  +  return rc;
          425  +}
          426  +
          427  +static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
          428  +  int eDetail = pCsr->pConfig->eDetail;
          429  +  int rc = SQLITE_OK;
          430  +  Fts5IndexIter *pIter = pCsr->pIter;
          431  +  i64 *pp = &pCsr->iInstPos;
          432  +  int *po = &pCsr->iInstOff;
          433  +  
          434  +  while( eDetail==FTS5_DETAIL_NONE
          435  +      || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
          436  +  ){
          437  +    pCsr->iInstPos = 0;
          438  +    pCsr->iInstOff = 0;
          439  +
          440  +    rc = sqlite3Fts5IterNextScan(pCsr->pIter);
          441  +    if( rc==SQLITE_OK ){
          442  +      rc = fts5VocabInstanceNewTerm(pCsr);
          443  +      if( eDetail==FTS5_DETAIL_NONE ) break;
          444  +    }
          445  +    if( rc ){
          446  +      pCsr->bEof = 1;
          447  +      break;
          448  +    }
          449  +  }
          450  +
          451  +  return rc;
          452  +}
   381    453   
   382    454   /*
   383    455   ** Advance the cursor to the next row in the table.
   384    456   */
   385    457   static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
   386    458     Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
   387    459     Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
   388    460     int rc = SQLITE_OK;
   389    461     int nCol = pCsr->pConfig->nCol;
   390    462   
   391    463     pCsr->rowid++;
          464  +
          465  +  if( pTab->eType==FTS5_VOCAB_INSTANCE ){
          466  +    return fts5VocabInstanceNext(pCsr);
          467  +  }
   392    468   
   393    469     if( pTab->eType==FTS5_VOCAB_COL ){
   394    470       for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
   395    471         if( pCsr->aDoc[pCsr->iCol] ) break;
   396    472       }
   397    473     }
   398    474   
   399         -  if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
          475  +  if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){
   400    476       if( sqlite3Fts5IterEof(pCsr->pIter) ){
   401    477         pCsr->bEof = 1;
   402    478       }else{
   403    479         const char *zTerm;
   404    480         int nTerm;
   405    481   
   406    482         zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
................................................................................
   416    492         sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
   417    493         memset(pCsr->aCnt, 0, nCol * sizeof(i64));
   418    494         memset(pCsr->aDoc, 0, nCol * sizeof(i64));
   419    495         pCsr->iCol = 0;
   420    496   
   421    497         assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
   422    498         while( rc==SQLITE_OK ){
          499  +        int eDetail = pCsr->pConfig->eDetail;
   423    500           const u8 *pPos; int nPos;   /* Position list */
   424    501           i64 iPos = 0;               /* 64-bit position read from poslist */
   425    502           int iOff = 0;               /* Current offset within position list */
   426    503   
   427    504           pPos = pCsr->pIter->pData;
   428    505           nPos = pCsr->pIter->nData;
   429         -        switch( pCsr->pConfig->eDetail ){
   430         -          case FTS5_DETAIL_FULL:
   431         -            pPos = pCsr->pIter->pData;
   432         -            nPos = pCsr->pIter->nData;
   433         -            if( pTab->eType==FTS5_VOCAB_ROW ){
          506  +
          507  +        switch( pTab->eType ){
          508  +          case FTS5_VOCAB_ROW:
          509  +            if( eDetail==FTS5_DETAIL_FULL ){
   434    510                 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   435    511                   pCsr->aCnt[0]++;
   436    512                 }
   437         -              pCsr->aDoc[0]++;
   438         -            }else{
          513  +            }
          514  +            pCsr->aDoc[0]++;
          515  +            break;
          516  +
          517  +          case FTS5_VOCAB_COL:
          518  +            if( eDetail==FTS5_DETAIL_FULL ){
   439    519                 int iCol = -1;
   440    520                 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   441    521                   int ii = FTS5_POS2COLUMN(iPos);
   442    522                   pCsr->aCnt[ii]++;
   443    523                   if( iCol!=ii ){
   444    524                     if( ii>=nCol ){
   445    525                       rc = FTS5_CORRUPT;
   446    526                       break;
   447    527                     }
   448    528                     pCsr->aDoc[ii]++;
   449    529                     iCol = ii;
   450    530                   }
   451    531                 }
   452         -            }
   453         -            break;
   454         -
   455         -          case FTS5_DETAIL_COLUMNS:
   456         -            if( pTab->eType==FTS5_VOCAB_ROW ){
   457         -              pCsr->aDoc[0]++;
   458         -            }else{
          532  +            }else if( eDetail==FTS5_DETAIL_COLUMNS ){
   459    533                 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
   460    534                   assert_nc( iPos>=0 && iPos<nCol );
   461    535                   if( iPos>=nCol ){
   462    536                     rc = FTS5_CORRUPT;
   463    537                     break;
   464    538                   }
   465    539                   pCsr->aDoc[iPos]++;
   466    540                 }
          541  +            }else{
          542  +              assert( eDetail==FTS5_DETAIL_NONE );
          543  +              pCsr->aDoc[0]++;
   467    544               }
   468    545               break;
   469    546   
   470         -          default: 
   471         -            assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
   472         -            pCsr->aDoc[0]++;
          547  +          default:
          548  +            assert( pTab->eType==FTS5_VOCAB_INSTANCE );
   473    549               break;
   474    550           }
   475    551   
   476    552           if( rc==SQLITE_OK ){
   477    553             rc = sqlite3Fts5IterNextScan(pCsr->pIter);
   478    554           }
          555  +        if( pTab->eType==FTS5_VOCAB_INSTANCE ) break;
   479    556   
   480    557           if( rc==SQLITE_OK ){
   481    558             zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
   482    559             if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
   483    560               break;
   484    561             }
   485    562             if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
................................................................................
   501    578   static int fts5VocabFilterMethod(
   502    579     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
   503    580     int idxNum,                     /* Strategy index */
   504    581     const char *zUnused,            /* Unused */
   505    582     int nUnused,                    /* Number of elements in apVal */
   506    583     sqlite3_value **apVal           /* Arguments for the indexing scheme */
   507    584   ){
          585  +  Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
   508    586     Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
          587  +  int eType = pTab->eType;
   509    588     int rc = SQLITE_OK;
   510    589   
   511    590     int iVal = 0;
   512    591     int f = FTS5INDEX_QUERY_SCAN;
   513    592     const char *zTerm = 0;
   514    593     int nTerm = 0;
   515    594   
................................................................................
   541    620           rc = SQLITE_NOMEM;
   542    621         }else{
   543    622           memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
   544    623         }
   545    624       }
   546    625     }
   547    626   
   548         -
   549    627     if( rc==SQLITE_OK ){
   550    628       rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
   551    629     }
   552         -  if( rc==SQLITE_OK ){
          630  +  if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){
          631  +    rc = fts5VocabInstanceNewTerm(pCsr);
          632  +  }
          633  +  if( rc==SQLITE_OK 
          634  +   && !pCsr->bEof 
          635  +   && (eType!=FTS5_VOCAB_INSTANCE || pCsr->pConfig->eDetail!=FTS5_DETAIL_NONE)
          636  +  ){
   553    637       rc = fts5VocabNextMethod(pCursor);
   554    638     }
   555    639   
   556    640     return rc;
   557    641   }
   558    642   
   559    643   /* 
................................................................................
   587    671           sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
   588    672         }
   589    673       }else if( iCol==2 ){
   590    674         iVal = pCsr->aDoc[pCsr->iCol];
   591    675       }else{
   592    676         iVal = pCsr->aCnt[pCsr->iCol];
   593    677       }
   594         -  }else{
          678  +  }else if( eType==FTS5_VOCAB_ROW ){
   595    679       assert( iCol==1 || iCol==2 );
   596    680       if( iCol==1 ){
   597    681         iVal = pCsr->aDoc[0];
   598    682       }else{
   599    683         iVal = pCsr->aCnt[0];
          684  +    }
          685  +  }else{
          686  +    int eDetail = pCsr->pConfig->eDetail;
          687  +    assert( eType==FTS5_VOCAB_INSTANCE );
          688  +    switch( iCol ){
          689  +      case 1:
          690  +        sqlite3_result_int64(pCtx, pCsr->pIter->iRowid);
          691  +        break;
          692  +      case 2: {
          693  +        int ii = -1;
          694  +        if( eDetail==FTS5_DETAIL_FULL ){
          695  +          ii = FTS5_POS2COLUMN(pCsr->iInstPos);
          696  +        }else if( eDetail==FTS5_DETAIL_COLUMNS ){
          697  +          ii = pCsr->iInstPos;
          698  +        }
          699  +        if( ii>=0 && ii<pCsr->pConfig->nCol ){
          700  +          const char *z = pCsr->pConfig->azCol[ii];
          701  +          sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
          702  +        }
          703  +        break;
          704  +      }
          705  +      default: {
          706  +        assert( iCol==3 );
          707  +        if( eDetail==FTS5_DETAIL_FULL ){
          708  +          int ii = FTS5_POS2OFFSET(pCsr->iInstPos);
          709  +          sqlite3_result_int(pCtx, ii);
          710  +        }
          711  +        break;
          712  +      }
   600    713       }
   601    714     }
   602    715   
   603    716     if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
   604    717     return SQLITE_OK;
   605    718   }
   606    719   

Changes to ext/fts5/fts5parse.y.

    85     85   %type cnearset    {Fts5ExprNode*}
    86     86   %type expr        {Fts5ExprNode*}
    87     87   %type exprlist    {Fts5ExprNode*}
    88     88   %destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
    89     89   %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
    90     90   %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
    91     91   
    92         -expr(A) ::= expr(X) AND expr(Y). {
    93         -  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
    94         -}
    95         -expr(A) ::= expr(X) OR expr(Y). {
    96         -  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
    97         -}
    98         -expr(A) ::= expr(X) NOT expr(Y). {
    99         -  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
   100         -}
   101         -
   102         -expr(A) ::= LP expr(X) RP. {A = X;}
   103         -expr(A) ::= exprlist(X).   {A = X;}
   104         -
   105         -exprlist(A) ::= cnearset(X). {A = X;}
   106         -exprlist(A) ::= exprlist(X) cnearset(Y). {
   107         -  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
   108         -}
   109         -
   110         -cnearset(A) ::= nearset(X). { 
   111         -  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
   112         -}
   113         -cnearset(A) ::= colset(X) COLON nearset(Y). { 
   114         -  sqlite3Fts5ParseSetColset(pParse, Y, X);
   115         -  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
   116         -}
   117         -
   118     92   %type colset {Fts5Colset*}
   119     93   %destructor colset { sqlite3_free($$); }
   120     94   %type colsetlist {Fts5Colset*}
   121     95   %destructor colsetlist { sqlite3_free($$); }
   122     96   
   123     97   colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
   124     98       A = sqlite3Fts5ParseColsetInvert(pParse, X);
................................................................................
   133    107   }
   134    108   
   135    109   colsetlist(A) ::= colsetlist(Y) STRING(X). { 
   136    110     A = sqlite3Fts5ParseColset(pParse, Y, &X); }
   137    111   colsetlist(A) ::= STRING(X). { 
   138    112     A = sqlite3Fts5ParseColset(pParse, 0, &X); 
   139    113   }
          114  +
          115  +expr(A) ::= expr(X) AND expr(Y). {
          116  +  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
          117  +}
          118  +expr(A) ::= expr(X) OR expr(Y). {
          119  +  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
          120  +}
          121  +expr(A) ::= expr(X) NOT expr(Y). {
          122  +  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
          123  +}
          124  +
          125  +expr(A) ::= colset(X) COLON LP expr(Y) RP. {
          126  +  sqlite3Fts5ParseSetColset(pParse, Y, X);
          127  +  A = Y;
          128  +}
          129  +expr(A) ::= LP expr(X) RP. {A = X;}
          130  +expr(A) ::= exprlist(X).   {A = X;}
          131  +
          132  +exprlist(A) ::= cnearset(X). {A = X;}
          133  +exprlist(A) ::= exprlist(X) cnearset(Y). {
          134  +  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
          135  +}
          136  +
          137  +cnearset(A) ::= nearset(X). { 
          138  +  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
          139  +}
          140  +cnearset(A) ::= colset(X) COLON nearset(Y). { 
          141  +  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
          142  +  sqlite3Fts5ParseSetColset(pParse, A, X);
          143  +}
          144  +
   140    145   
   141    146   %type nearset     {Fts5ExprNearset*}
   142    147   %type nearphrases {Fts5ExprNearset*}
   143    148   %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
   144    149   %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
   145    150   
   146    151   nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }

Changes to ext/fts5/test/fts5aa.test.

   437    437   # exception. But since bm25() can now used the cached structure record,
   438    438   # it never sees the corruption introduced by funk() and so the following 
   439    439   # statement no longer fails.
   440    440   #
   441    441   do_catchsql_test 16.2 {
   442    442     SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
   443    443   } {0 {{} -1e-06 {}}}
   444         -# {1 {SQL logic error or missing database}}
          444  +# {1 {SQL logic error}}
   445    445   
   446    446   #-------------------------------------------------------------------------
   447    447   #
   448    448   reset_db
   449    449   do_execsql_test 17.1 {
   450    450     CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
   451    451     INSERT INTO b2 VALUES('a');
................................................................................
   557    557   do_test 20.1 {
   558    558     foreach id $::ids {
   559    559       execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') }
   560    560     }
   561    561     execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
   562    562   } $::ids
   563    563   
          564  +#--------------------------------------------------------------------
          565  +# Test that a DROP TABLE may be executed within a transaction that
          566  +# writes to an FTS5 table.
          567  +#
          568  +do_execsql_test 21.0 {
          569  +  CREATE TEMP TABLE t8(a, b);
          570  +  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
          571  +}
          572  +
          573  +do_execsql_test 21.1 {
          574  +  BEGIN;
          575  +    INSERT INTO ft VALUES('a b c');
          576  +    DROP TABLE t8;
          577  +  COMMIT;
          578  +}
          579  +
          580  +do_execsql_test 22.0 {
          581  +  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
          582  +  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
          583  +  BEGIN;
          584  +    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
          585  +    DELETE FROM t9 WHERE rowid = 2;
          586  +    INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
          587  +  COMMIT;
          588  +}
          589  +
          590  +do_execsql_test 22.1 {
          591  +  SELECT rowid FROM t9('a*')
          592  +} {1}
          593  +
   564    594   }
   565    595   
   566    596   
   567    597   finish_test
   568         -
   569         -

Changes to ext/fts5/test/fts5ab.test.

   290    290     INSERT INTO x1 VALUES($doc);
   291    291   }
   292    292   
   293    293   } ;# foreach_detail_mode...
   294    294   
   295    295   
   296    296   finish_test
   297         -

Changes to ext/fts5/test/fts5ac.test.

   272    272   } {
   273    273     do_execsql_test 2.3.$tn {
   274    274       SELECT fts5_expr_tcl($expr, 'N $x')
   275    275     } [list $tclexpr]
   276    276   }
   277    277   
   278    278   finish_test
   279         -

Changes to ext/fts5/test/fts5ad.test.

   227    227         28 {a f*} 29 {a* f*} 30 {a* fghij*}
   228    228       } {
   229    229         set res [prefix_query $prefix]
   230    230         if {$bAsc} {
   231    231           set res [lsort -integer -increasing $res]
   232    232         }
   233    233         set n [llength $res]
   234         -      if {$T==5} breakpoint 
   235    234         do_execsql_test $T.$bAsc.$tn.$n $sql $res
   236    235       }
   237    236     }
   238    237   
   239    238     catchsql COMMIT
   240    239   }
   241    240   
   242    241   }
   243    242   
   244    243   finish_test
   245         -

Changes to ext/fts5/test/fts5ae.test.

   305    305       SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
   306    306     } $cnt
   307    307   }
   308    308   
   309    309   }
   310    310   
   311    311   finish_test
   312         -

Changes to ext/fts5/test/fts5af.test.

   174    174   do_execsql_test 5.1 {
   175    175     SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
   176    176   } {{[x] a a a a a...}}
   177    177   
   178    178   } ;# foreach_detail_mode 
   179    179   
   180    180   finish_test
   181         -

Changes to ext/fts5/test/fts5ag.test.

   138    138     }
   139    139   }
   140    140   
   141    141   } ;# foreach_detail_mode
   142    142   
   143    143   
   144    144   finish_test
   145         -

Changes to ext/fts5/test/fts5ah.test.

   163    163   } {10000}
   164    164   
   165    165   } ;# foreach_detail_mode
   166    166   
   167    167   #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
   168    168   
   169    169   finish_test
   170         -

Changes to ext/fts5/test/fts5ai.test.

    51     51   do_execsql_test 1.2 {
    52     52     INSERT INTO t1(t1) VALUES('integrity-check');
    53     53   }
    54     54   }
    55     55   
    56     56   
    57     57   finish_test
    58         -

Changes to ext/fts5/test/fts5aj.test.

    62     62     }
    63     63   }
    64     64   
    65     65   do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }
    66     66   
    67     67   
    68     68   finish_test
    69         -

Changes to ext/fts5/test/fts5ak.test.

   143    143     {[a b c] [c d e]}
   144    144     {[a b c d e]}
   145    145   }
   146    146   
   147    147   }
   148    148   
   149    149   finish_test
   150         -

Changes to ext/fts5/test/fts5al.test.

    73     73     1 ""
    74     74     2 "fname"
    75     75     3 "fname(X'234ab')"
    76     76     4 "myfunc(-1.,'abc')"
    77     77   } {
    78     78     do_test 2.2.$tn {
    79     79       catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
    80         -  } {1 {SQL logic error or missing database}}
           80  +  } {1 {SQL logic error}}
    81     81   }
    82     82   
    83     83   #-------------------------------------------------------------------------
    84     84   # Assorted tests of the tcl interface for creating extension functions.
    85     85   #
    86     86   
    87     87   do_execsql_test 3.1 {
................................................................................
   293    293     SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
   294    294   } {1 {parse error in rank function: }}
   295    295   
   296    296   } ;# foreach_detail_mode
   297    297   
   298    298   
   299    299   finish_test
   300         -

Changes to ext/fts5/test/fts5alter.test.

    85     85   
    86     86   do_execsql_test 3.1 {
    87     87     CREATE VIRTUAL TABLE abc USING fts5(a);
    88     88     INSERT INTO abc(rowid, a) VALUES(1, 'a');
    89     89     BEGIN;
    90     90       INSERT INTO abc(rowid, a) VALUES(2, 'a');
    91     91   }
    92         -breakpoint
    93     92   do_execsql_test 3.2 {
    94     93       SELECT rowid FROM abc WHERE abc MATCH 'a';
    95     94   } {1 2}
    96     95   
    97     96   do_execsql_test 3.3 {
    98     97     COMMIT;
    99     98     SELECT rowid FROM abc WHERE abc MATCH 'a';
   100     99   } {1 2}
   101    100   
   102    101   finish_test
   103         -

Changes to ext/fts5/test/fts5auto.test.

   338    338   } {
   339    339     do_auto_test 4.$tn yy $expr
   340    340   }
   341    341   
   342    342   
   343    343   
   344    344   finish_test
   345         -

Changes to ext/fts5/test/fts5aux.test.

   236    236     4  {"a a a" "b" "a d"} {"[a] [a] [a]" "[a] d"}
   237    237     1  {"b d" "a b"}       {"[b] [d]" "[a] b"}
   238    238     2  {"d b" "a d"}       {"[d] [b]" "[a] d"}
   239    239     3  {"a a d"}           {"[a] [a] d"}
   240    240   } {
   241    241     execsql { DELETE FROM x1 }
   242    242     foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } }
   243         -  breakpoint
   244    243     do_execsql_test 8.$tn {
   245    244       SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)';
   246    245     } $res
   247    246   }
   248    247   
   249    248   #-------------------------------------------------------------------------
   250    249   # Test the built-in bm25() demo.
................................................................................
   275    274   } {
   276    275     9 10
   277    276   }
   278    277   
   279    278   
   280    279   
   281    280   finish_test
   282         -

Changes to ext/fts5/test/fts5auxdata.test.

   108    108   db eval { 
   109    109     SELECT aux_function_2(f1, 2, 'A'), aux_function_2(f1, 2, 'B') 
   110    110     FROM f1 WHERE f1 MATCH 'a'
   111    111     ORDER BY rowid ASC
   112    112   }
   113    113   
   114    114   finish_test
   115         -

Changes to ext/fts5/test/fts5bigpl.test.

    57     57       set doc [string repeat "$t " 150000000]
    58     58       execsql { INSERT INTO t1 VALUES($doc) }
    59     59     }
    60     60     execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    61     61   } {}
    62     62   
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5bigtok.test.

    60     60       do_execsql_test 2.[string range $v 0 0] {
    61     61         SELECT rowid FROM t1($v) ORDER BY rowid DESC
    62     62       } [lsort -integer -decr $res]
    63     63     }
    64     64   }
    65     65   
    66     66   finish_test
    67         -
    68         -

Changes to ext/fts5/test/fts5colset.test.

    40     40       5 " - {d d c} : a" {1 2}
    41     41       6 "- {d c b a} : a" {}
    42     42       7 "-{\"a\"} : b" {1 2 3}
    43     43       8 "- c : a" {1 2 4}
    44     44       9 "-c : a"  {1 2 4}
    45     45       10 "-\"c\" : a"  {1 2 4}
    46     46     } {
    47         -  breakpoint
    48     47       do_execsql_test 1.$tn {
    49     48         SELECT rowid FROM t1($q)
    50     49       } $res
    51     50     }
    52     51   
           52  +  foreach {tn q res} {
           53  +    0 {{a} : (a AND ":")}     {}
           54  +    1 "{a b c} : (a AND d)"   {2 3}
           55  +    2 "{a b c} : (a AND b:d)" {3}
           56  +    3 "{a b c} : (a AND d:d)" {}
           57  +    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
           58  +    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
           59  +    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
           60  +    7 "{a b c} : (b:a AND c:b)" {2}
           61  +  } {
           62  +    do_execsql_test 2.$tn {
           63  +      SELECT rowid FROM t1($q)
           64  +    } $res
           65  +  }
           66  +
           67  +  foreach {tn w res} {
           68  +    0 "a MATCH 'a'" {1}
           69  +    1 "b MATCH 'a'" {2}
           70  +    2 "b MATCH '{a b c} : a'" {2}
           71  +    3 "b MATCH 'a OR b'"      {1 2}
           72  +    4 "b MATCH 'a OR a:b'"    {2}
           73  +    5 "b MATCH 'a OR b:b'"    {1 2}
           74  +  } {
           75  +    do_execsql_test 3.$tn "
           76  +      SELECT rowid FROM t1 WHERE $w
           77  +    " $res
           78  +  }
    53     79   
           80  +  do_catchsql_test 4.1 {
           81  +    SELECT * FROM t1 WHERE rowid MATCH 'a'
           82  +  } {1 {unable to use function MATCH in the requested context}}
    54     83   }
    55     84   
    56     85   
    57     86   finish_test
    58         -
    59         -

Changes to ext/fts5/test/fts5columnsize.test.

   139    139   #
   140    140   do_execsql_test 4.1.1 {
   141    141     CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0);
   142    142     INSERT INTO t5 VALUES('1 2 3 4');
   143    143     INSERT INTO t5 VALUES('2 4 6 8');
   144    144   }
   145    145   
   146         -breakpoint
   147    146   do_execsql_test 4.1.2 {
   148    147     INSERT INTO t5(t5) VALUES('integrity-check');
   149    148   }
   150    149   
   151    150   finish_test

Changes to ext/fts5/test/fts5config.test.

    62     62     5 "f1(x':;')"
    63     63     6 "f1(x'[]')"
    64     64     7 "f1(x'{}')"
    65     65     8 "f1('abc)"
    66     66   } {
    67     67     do_catchsql_test 3.$tn {
    68     68       INSERT INTO t1(t1, rank) VALUES('rank', $val);
    69         -  } {1 {SQL logic error or missing database}}
           69  +  } {1 {SQL logic error}}
    70     70   }
    71     71   
    72     72   #-------------------------------------------------------------------------
    73     73   # The parsing of SQL literals specified as part of 'rank' options.
    74     74   #
    75     75   do_execsql_test 4.0 {
    76     76     CREATE VIRTUAL TABLE zzz USING fts5(one);
................................................................................
   106    106   #-------------------------------------------------------------------------
   107    107   # Misquoting in tokenize= and other options. 
   108    108   #
   109    109   do_catchsql_test 5.1 {
   110    110     CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii");
   111    111   } {1 {parse error in tokenize directive}} 
   112    112   
   113         -breakpoint
   114    113   do_catchsql_test 5.2 {
   115    114     CREATE VIRTUAL TABLE xx USING fts5(x, [y[]);
   116    115   } {0 {}}
   117    116   
   118    117   do_catchsql_test 5.3 {
   119    118     CREATE VIRTUAL TABLE yy USING fts5(x, [y]]);
   120    119   } {1 {unrecognized token: "]"}}
................................................................................
   165    164   #   9.5.* 'hashsize' options.
   166    165   #
   167    166   do_execsql_test 9.0 {
   168    167     CREATE VIRTUAL TABLE abc USING fts5(a, b);
   169    168   } {}
   170    169   do_catchsql_test 9.1.1 {
   171    170     INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
   172         -} {1 {SQL logic error or missing database}}
          171  +} {1 {SQL logic error}}
   173    172   do_catchsql_test 9.1.2 {
   174    173     INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
   175         -} {1 {SQL logic error or missing database}}
          174  +} {1 {SQL logic error}}
   176    175   do_catchsql_test 9.1.3 {
   177    176     INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
   178         -} {1 {SQL logic error or missing database}}
          177  +} {1 {SQL logic error}}
   179    178   
   180    179   do_catchsql_test 9.2.1 {
   181    180     INSERT INTO abc(abc, rank) VALUES('automerge', -5);
   182         -} {1 {SQL logic error or missing database}}
          181  +} {1 {SQL logic error}}
   183    182   do_catchsql_test 9.2.2 {
   184    183     INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
   185         -} {1 {SQL logic error or missing database}}
          184  +} {1 {SQL logic error}}
   186    185   do_catchsql_test 9.2.3 {
   187    186     INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
   188         -} {1 {SQL logic error or missing database}}
          187  +} {1 {SQL logic error}}
   189    188   do_execsql_test 9.2.4 {
   190    189     INSERT INTO abc(abc, rank) VALUES('automerge', 1);
   191    190   } {}
   192    191   
   193    192   do_catchsql_test 9.3.1 {
   194    193     INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
   195         -} {1 {SQL logic error or missing database}}
          194  +} {1 {SQL logic error}}
   196    195   do_catchsql_test 9.3.2 {
   197    196     INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
   198         -} {1 {SQL logic error or missing database}}
          197  +} {1 {SQL logic error}}
   199    198   do_execsql_test 9.3.3 {
   200    199     INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
   201    200   } {}
   202    201   do_execsql_test 9.3.4 {
   203    202     INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
   204    203   } {}
   205    204   
   206    205   do_catchsql_test 9.4.1 {
   207    206     INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
   208         -} {1 {SQL logic error or missing database}}
          207  +} {1 {SQL logic error}}
   209    208   
   210    209   do_catchsql_test 9.5.1 {
   211    210     INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
   212         -} {1 {SQL logic error or missing database}}
          211  +} {1 {SQL logic error}}
   213    212   do_catchsql_test 9.5.2 {
   214    213     INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
   215         -} {1 {SQL logic error or missing database}}
          214  +} {1 {SQL logic error}}
   216    215   do_catchsql_test 9.5.3 {
   217    216     INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
   218    217   } {0 {}}
   219    218   
   220    219   #-------------------------------------------------------------------------
   221    220   # Too many prefix indexes. Maximum allowed is 31.
   222    221   #
................................................................................
   241    240   } {
   242    241     set res [list 1 {malformed detail=... directive}]
   243    242     do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
   244    243   }
   245    244   
   246    245   do_catchsql_test 12.1 {
   247    246     INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
   248         -} {1 {SQL logic error or missing database}}
          247  +} {1 {SQL logic error}}
   249    248   
   250    249   #-------------------------------------------------------------------------
   251    250   # errors in the 'usermerge' option
   252    251   #
   253    252   do_execsql_test 13.0 {
   254    253     CREATE VIRTUAL TABLE tt USING fts5(ttt);
   255    254   }
................................................................................
   256    255   foreach {tn val} {
   257    256     1     -1
   258    257     2     4.2
   259    258     3     17
   260    259     4     1
   261    260   } {
   262    261     set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
   263         -  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
          262  +  do_catchsql_test 13.$tn $sql {1 {SQL logic error}}
   264    263   }
   265    264   
   266    265   finish_test
   267         -

Changes to ext/fts5/test/fts5conflict.test.

    62     62     REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1');
    63     63     DELETE FROM tbl WHERE a=100;
    64     64   
    65     65     INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
    66     66   }
    67     67   
    68     68   finish_test
    69         -
    70         -

Added ext/fts5/test/fts5connect.test.

            1  +# 2017 August 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5connect
           17  +
           18  +ifcapable !fts5 {
           19  +  finish_test
           20  +  return
           21  +}
           22  +
           23  +#-------------------------------------------------------------------------
           24  +# The tests in this file test the outcome of a schema-reset happening 
           25  +# within the xConnect() method of an FTS5 table. At one point this
           26  +# was causing a problem in SQLite. Each test proceeds as follows:
           27  +#
           28  +#   1. Connection [db] opens the db and reads from some unrelated, non-FTS5
           29  +#      table causing SQLite to load the db schema into memory.
           30  +#
           31  +#   2. Connection [db2] opens the db and modifies the db schema.
           32  +#
           33  +#   3. Connection [db] reads or writes an existing fts5 table. That the
           34  +#      schema has been modified is detected inside the fts5 xConnect() 
           35  +#      callback that is invoked by sqlite3_prepare(). 
           36  +#
           37  +#   4. Verify that the statement in 3 has worked. SQLite should detect
           38  +#      that the schema has changed and successfully prepare the 
           39  +#      statement against the new schema.
           40  +#
           41  +# Test plan:
           42  +#
           43  +#   1.*: Trigger the xConnect()/schema-reset using statements executed
           44  +#        directly against an FTS5 table.
           45  +#
           46  +#   2.*: Using various statements executed by various BEFORE triggers.
           47  +#
           48  +#   3.*: Using various statements executed by various AFTER triggers.
           49  +#
           50  +#   4.*: Using various statements executed by various INSTEAD OF triggers.
           51  +#
           52  +
           53  +
           54  +
           55  +do_execsql_test 1.0 {
           56  +  CREATE VIRTUAL TABLE ft1 USING fts5(a, b);
           57  +  CREATE TABLE abc(x INTEGER PRIMARY KEY);
           58  +  CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b);
           59  +
           60  +  INSERT INTO ft1 VALUES('one', 'two');
           61  +  INSERT INTO ft1 VALUES('three', 'four');
           62  +}
           63  +
           64  +foreach {tn sql res} {
           65  +  1 "SELECT * FROM ft1" {one two three four}
           66  +  2 "REPLACE INTO ft1(rowid, a, b) VALUES(1, 'five', 'six')" {}
           67  +  3 "SELECT * FROM ft1" {five six three four}
           68  +  4 "INSERT INTO ft1 VALUES('seven', 'eight')" {}
           69  +  5 "SELECT * FROM ft1" {five six three four seven eight}
           70  +  6 "DELETE FROM ft1 WHERE rowid=2" {}
           71  +  7 "UPDATE ft1 SET b='nine' WHERE rowid=1" {}
           72  +  8 "SELECT * FROM ft1" {five nine seven eight}
           73  +} {
           74  +
           75  +  catch { db close }
           76  +  catch { db2 close }
           77  +  sqlite3 db  test.db
           78  +  sqlite3 db2 test.db
           79  +
           80  +  do_test 1.$tn.1 {
           81  +    db eval { INSERT INTO abc DEFAULT VALUES }
           82  +    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
           83  +  } {}
           84  +
           85  +  do_execsql_test 1.$tn.2 $sql $res
           86  +
           87  +  do_execsql_test 1.$tn.3 {
           88  +    INSERT INTO ft1(ft1) VALUES('integrity-check');
           89  +  }
           90  +}
           91  +
           92  +do_execsql_test 2.0 {
           93  +  CREATE VIRTUAL TABLE ft2 USING fts5(a, b);
           94  +  CREATE TABLE t2(a, b);
           95  +  CREATE TABLE log(txt);
           96  +
           97  +  CREATE TRIGGER t2_ai AFTER INSERT ON t2 BEGIN
           98  +    INSERT INTO ft2(rowid, a, b) VALUES(new.rowid, new.a, new.b);
           99  +    INSERT INTO log VALUES('insert');
          100  +  END;
          101  +
          102  +  CREATE TRIGGER t2_ad AFTER DELETE ON t2 BEGIN
          103  +    DELETE FROM ft2 WHERE rowid = old.rowid;
          104  +    INSERT INTO log VALUES('delete');
          105  +  END;
          106  +
          107  +  CREATE TRIGGER t2_au AFTER UPDATE ON t2 BEGIN
          108  +    UPDATE ft2 SET a=new.a, b=new.b WHERE rowid=new.rowid;
          109  +    INSERT INTO log VALUES('update');
          110  +  END;
          111  +
          112  +  INSERT INTO t2 VALUES('one', 'two');
          113  +  INSERT INTO t2 VALUES('three', 'four');
          114  +}
          115  +
          116  +foreach {tn sql res} {
          117  +  1 "SELECT * FROM t2" {one two three four}
          118  +  2 "REPLACE INTO t2(rowid, a, b) VALUES(1, 'five', 'six')" {}
          119  +  3 "SELECT * FROM ft2" {five six three four}
          120  +  4 "INSERT INTO t2 VALUES('seven', 'eight')" {}
          121  +  5 "SELECT * FROM ft2" {five six three four seven eight}
          122  +  6 "DELETE FROM t2 WHERE rowid=2" {}
          123  +  7 "UPDATE t2 SET b='nine' WHERE rowid=1" {}
          124  +  8 "SELECT * FROM ft2" {five nine seven eight}
          125  +} {
          126  +
          127  +  catch { db close }
          128  +  catch { db2 close }
          129  +  sqlite3 db  test.db
          130  +  sqlite3 db2 test.db
          131  +
          132  +  do_test 2.$tn.1 {
          133  +    db eval { INSERT INTO abc DEFAULT VALUES }
          134  +    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
          135  +  } {}
          136  +
          137  +  do_execsql_test 2.$tn.2 $sql $res
          138  +
          139  +  do_execsql_test 2.$tn.3 {
          140  +    INSERT INTO ft2(ft2) VALUES('integrity-check');
          141  +  }
          142  +}
          143  +
          144  +do_execsql_test 3.0 {
          145  +  CREATE VIRTUAL TABLE ft3 USING fts5(a, b);
          146  +  CREATE TABLE t3(a, b);
          147  +
          148  +  CREATE TRIGGER t3_ai BEFORE INSERT ON t3 BEGIN
          149  +    INSERT INTO ft3(rowid, a, b) VALUES(new.rowid, new.a, new.b);
          150  +    INSERT INTO log VALUES('insert');
          151  +  END;
          152  +
          153  +  CREATE TRIGGER t3_ad BEFORE DELETE ON t3 BEGIN
          154  +    DELETE FROM ft3 WHERE rowid = old.rowid;
          155  +    INSERT INTO log VALUES('delete');
          156  +  END;
          157  +
          158  +  CREATE TRIGGER t3_au BEFORE UPDATE ON t3 BEGIN
          159  +    UPDATE ft3 SET a=new.a, b=new.b WHERE rowid=new.rowid;
          160  +    INSERT INTO log VALUES('update');
          161  +  END;
          162  +
          163  +  INSERT INTO t3(rowid, a, b) VALUES(1, 'one', 'two');
          164  +  INSERT INTO t3(rowid, a, b) VALUES(2, 'three', 'four');
          165  +}
          166  +
          167  +foreach {tn sql res} {
          168  +  1 "SELECT * FROM t3" {one two three four}
          169  +  2 "REPLACE INTO t3(rowid, a, b) VALUES(1, 'five', 'six')" {}
          170  +  3 "SELECT * FROM ft3" {five six three four}
          171  +  4 "INSERT INTO t3(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
          172  +  5 "SELECT * FROM ft3" {five six three four seven eight}
          173  +  6 "DELETE FROM t3 WHERE rowid=2" {}
          174  +  7 "UPDATE t3 SET b='nine' WHERE rowid=1" {}
          175  +  8 "SELECT * FROM ft3" {five nine seven eight}
          176  +} {
          177  +
          178  +  catch { db close }
          179  +  catch { db2 close }
          180  +  sqlite3 db  test.db
          181  +  sqlite3 db2 test.db
          182  +
          183  +  do_test 3.$tn.1 {
          184  +    db eval { INSERT INTO abc DEFAULT VALUES }
          185  +    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
          186  +  } {}
          187  +
          188  +  do_execsql_test 3.$tn.2 $sql $res
          189  +
          190  +  do_execsql_test 3.$tn.3 {
          191  +    INSERT INTO ft3(ft3) VALUES('integrity-check');
          192  +  }
          193  +}
          194  +
          195  +do_execsql_test 4.0 {
          196  +  CREATE VIRTUAL TABLE ft4 USING fts5(a, b);
          197  +  CREATE VIEW v4 AS SELECT rowid, * FROM ft4;
          198  +
          199  +  CREATE TRIGGER t4_ai INSTEAD OF INSERT ON v4 BEGIN
          200  +    INSERT INTO ft4(rowid, a, b) VALUES(new.rowid, new.a, new.b);
          201  +    INSERT INTO log VALUES('insert');
          202  +  END;
          203  +
          204  +  CREATE TRIGGER t4_ad INSTEAD OF DELETE ON v4 BEGIN
          205  +    DELETE FROM ft4 WHERE rowid = old.rowid;
          206  +    INSERT INTO log VALUES('delete');
          207  +  END;
          208  +
          209  +  CREATE TRIGGER t4_au INSTEAD OF UPDATE ON v4 BEGIN
          210  +    UPDATE ft4 SET a=new.a, b=new.b WHERE rowid=new.rowid;
          211  +    INSERT INTO log VALUES('update');
          212  +  END;
          213  +
          214  +  INSERT INTO ft4(rowid, a, b) VALUES(1, 'one', 'two');
          215  +  INSERT INTO ft4(rowid, a, b) VALUES(2, 'three', 'four');
          216  +}
          217  +
          218  +foreach {tn sql res} {
          219  +  1 "SELECT * FROM ft4" {one two three four}
          220  +  2 "REPLACE INTO v4(rowid, a, b) VALUES(1, 'five', 'six')" {}
          221  +  3 "SELECT * FROM ft4" {five six three four}
          222  +  4 "INSERT INTO v4(rowid, a, b) VALUES(3, 'seven', 'eight')" {}
          223  +  5 "SELECT * FROM ft4" {five six three four seven eight}
          224  +  6 "DELETE FROM v4 WHERE rowid=2" {}
          225  +  7 "UPDATE v4 SET b='nine' WHERE rowid=1" {}
          226  +  8 "SELECT * FROM ft4" {five nine seven eight}
          227  +} {
          228  +
          229  +  catch { db close }
          230  +  catch { db2 close }
          231  +  sqlite3 db  test.db
          232  +  sqlite3 db2 test.db
          233  +
          234  +  do_test 4.$tn.1 {
          235  +    db eval { INSERT INTO abc DEFAULT VALUES }
          236  +    db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable }
          237  +  } {}
          238  +
          239  +  do_execsql_test 4.$tn.2 $sql $res
          240  +
          241  +  do_execsql_test 4.$tn.3 {
          242  +    INSERT INTO ft3(ft3) VALUES('integrity-check');
          243  +  }
          244  +}
          245  +
          246  +finish_test
          247  +

Changes to ext/fts5/test/fts5content.test.

   251    251   do_execsql_test 6.2 {
   252    252     DROP TABLE xx;
   253    253     SELECT name FROM sqlite_master;
   254    254   } {}
   255    255   
   256    256   
   257    257   finish_test
   258         -

Changes to ext/fts5/test/fts5corrupt.test.

    92     92   
    93     93   do_catchsql_test 3.1 {
    94     94     DELETE FROM t3_content WHERE rowid = 3;
    95     95     SELECT * FROM t3 WHERE t3 MATCH 'o';
    96     96   } {1 {database disk image is malformed}}
    97     97   
    98     98   finish_test
    99         -

Changes to ext/fts5/test/fts5corrupt2.test.

   265    265   do_catchsql_test 6.2 {
   266    266     SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a'
   267    267   } {1 SQLITE_CORRUPT_VTAB}
   268    268   
   269    269   
   270    270   sqlite3_fts5_may_be_corrupt 0
   271    271   finish_test
   272         -

Changes to ext/fts5/test/fts5corrupt3.test.

   405    405   } {}
   406    406   do_catchsql_test 9.2.2 {
   407    407     SELECT * FROM t1('one AND two');
   408    408   } {1 {database disk image is malformed}}
   409    409   
   410    410   sqlite3_fts5_may_be_corrupt 0
   411    411   finish_test
   412         -

Added ext/fts5/test/fts5delete.test.

            1  +# 2017 May 12
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5delete
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +fts5_aux_test_functions db
           24  +
           25  +do_execsql_test 1.0 {
           26  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           27  +  WITH s(i) AS (
           28  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
           29  +  )
           30  +  INSERT INTO t1(rowid, x) SELECT i, (i/2)*2 FROM s;
           31  +}
           32  +
           33  +do_test 1.1 {
           34  +  execsql BEGIN
           35  +  for {set i 1} {$i<=5000} {incr i} {
           36  +    if {$i % 2} {
           37  +      execsql { INSERT INTO t1 VALUES($i) }
           38  +    } else {
           39  +      execsql { DELETE FROM t1 WHERE rowid = $i }
           40  +    }
           41  +  }
           42  +  execsql COMMIT
           43  +} {}
           44  +
           45  +do_test 1.2 {
           46  +  execsql { INSERT INTO t1(t1, rank) VALUES('usermerge', 2); }
           47  +  for {set i 0} {$i < 5} {incr i} {
           48  +    execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
           49  +    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
           50  +  }
           51  +} {}
           52  +
           53  +finish_test

Changes to ext/fts5/test/fts5detail.test.

   237    237       (SELECT sum(length(block)) from t2_data) <
   238    238       (SELECT sum(length(block)) from t3_data)
   239    239   } {1}
   240    240   
   241    241   
   242    242   
   243    243   finish_test
   244         -

Changes to ext/fts5/test/fts5determin.test.

    59     59     } {}
    60     60   
    61     61     do_determin_test 1.4
    62     62   }
    63     63   
    64     64   
    65     65   finish_test
    66         -
    67         -

Changes to ext/fts5/test/fts5dlidx.test.

    62     62           append doc " y" 
    63     63         }
    64     64       }
    65     65       execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) }
    66     66     }
    67     67     execsql COMMIT
    68     68   
    69         -  breakpoint
    70     69     do_test $tn.1 {
    71     70       execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    72     71     } {}
    73     72     
    74     73     do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
    75     74     do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
    76     75     
................................................................................
   120    119       INSERT INTO t1(rowid,x) SELECT i, $str FROM iii;
   121    120       COMMIT;
   122    121     }
   123    122   
   124    123     do_execsql_test $tn.1 {
   125    124       SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a'
   126    125     } {1}
   127         -  breakpoint
   128    126     do_execsql_test $tn.2 {
   129    127       SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
   130    128     } {1}
   131    129   }
   132    130   
   133    131   do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
   134    132   
................................................................................
   193    191   }
   194    192   
   195    193   } ;# foreach_detail_mode
   196    194   
   197    195   
   198    196   
   199    197   finish_test
   200         -

Changes to ext/fts5/test/fts5doclist.test.

    40     40   
    41     41   do_execsql_test 1.2 {
    42     42     INSERT INTO ccc(ccc) VALUES('integrity-check');
    43     43   }
    44     44   
    45     45   
    46     46   finish_test
    47         -

Changes to ext/fts5/test/fts5eb.test.

    77     77   do_execsql_test 3.3 {
    78     78     SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
    79     79   } {1 -1e-06}
    80     80   
    81     81   
    82     82   
    83     83   finish_test
    84         -
    85         -
    86         -

Changes to ext/fts5/test/fts5fault1.test.

   347    347       if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" }
   348    348     }
   349    349   }
   350    350   
   351    351   
   352    352   
   353    353   finish_test
   354         -

Changes to ext/fts5/test/fts5fault2.test.

   133    133       );
   134    134     }
   135    135   } -test {
   136    136     faultsim_test_result {0 {}}
   137    137   }
   138    138   
   139    139   finish_test
   140         -

Changes to ext/fts5/test/fts5fault3.test.

   106    106   } -test {
   107    107     faultsim_test_result [list 0 {}]
   108    108   }
   109    109   
   110    110   
   111    111   
   112    112   finish_test
   113         -

Changes to ext/fts5/test/fts5fault4.test.

   391    391   } -body {
   392    392     db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" }
   393    393   } -test {
   394    394     faultsim_test_result {0 {}}
   395    395   }
   396    396   
   397    397   finish_test
   398         -

Changes to ext/fts5/test/fts5fault5.test.

   101    101     db eval {
   102    102       SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
   103    103     }
   104    104   } -test {
   105    105     faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
   106    106   }
   107    107   
   108         -breakpoint
   109    108   do_execsql_test 3.3.0 {
   110    109     SELECT * FROM tv2;
   111    110   } {
   112    111     0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        
   113    112     14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     
   114    113     2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   
   115    114     9 x 1 {}
................................................................................
   126    125         9 x 1 {}
   127    126     ]]
   128    127   }
   129    128   
   130    129   
   131    130   
   132    131   finish_test
   133         -

Changes to ext/fts5/test/fts5fault6.test.

   276    276     }
   277    277   } -test {
   278    278     faultsim_test_result {0 1}
   279    279   }
   280    280   
   281    281   #-------------------------------------------------------------------------
   282    282   catch { db close }
   283         -breakpoint
   284    283   do_faultsim_test 6 -faults oom* -prep {
   285    284     sqlite_orig db test.db
   286    285     sqlite3_db_config_lookaside db 0 0 0
   287    286   } -test {
   288    287     faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }}
   289    288     if {$testrc==0} {
   290    289       db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) }
   291    290     }
   292    291     db close
   293    292   }
   294    293   finish_test
   295         -

Changes to ext/fts5/test/fts5fault7.test.

   112    112   do_faultsim_test 2.2 -faults oom-* -body {
   113    113     db eval { SELECT * FROM xy('""') }
   114    114   } -test {
   115    115     faultsim_test_result {0 {}}
   116    116   }
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5fault8.test.

    78     78     execsql { INSERT INTO x2(x2) VALUES('optimize') }
    79     79   } -test {
    80     80     faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
    81     81   }
    82     82   
    83     83   
    84     84   finish_test
    85         -

Changes to ext/fts5/test/fts5fault9.test.

   149    149     faultsim_test_result [list 0 {1 3}]
   150    150   }
   151    151   
   152    152   
   153    153   } ;# foreach_detail_mode...
   154    154   
   155    155   finish_test
   156         -

Changes to ext/fts5/test/fts5faultA.test.

    57     57     sqlite3 db test.db
    58     58   } -body {
    59     59     execsql { SELECT rowid FROM o2('a+b+c NOT xyz') }
    60     60   } -test {
    61     61     faultsim_test_result {0 {1 2}}
    62     62   }
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5faultB.test.

    74     74   
    75     75   do_faultsim_test 2.4 -faults oom* -body {
    76     76     execsql { SELECT mit(matchinfo(t1, 's')) FROM t1('a b c') }
    77     77   } -test {
    78     78     faultsim_test_result {0 {{3 2} {2 3}}} 
    79     79   }
    80     80   
           81  +#-------------------------------------------------------------------------
           82  +#
           83  +reset_db 
           84  +do_execsql_test 3.0 {
           85  +  CREATE VIRTUAL TABLE x1 USING fts5(z);
           86  +}
           87  +
           88  +do_faultsim_test 3.1 -faults oom* -body {
           89  +  execsql {
           90  +    SELECT rowid FROM x1('c') WHERE rowid>1;
           91  +  }
           92  +} -test {
           93  +  faultsim_test_result {0 {}}
           94  +}
           95  +
           96  +do_execsql_test 3.2 {
           97  +  INSERT INTO x1 VALUES('a b c');
           98  +  INSERT INTO x1 VALUES('b c d');
           99  +  INSERT INTO x1 VALUES('c d e');
          100  +  INSERT INTO x1 VALUES('d e f');
          101  +}
          102  +do_faultsim_test 3.3 -faults oom* -body {
          103  +  execsql {
          104  +    SELECT rowid FROM x1('c') WHERE rowid>1;
          105  +  }
          106  +} -test {
          107  +  faultsim_test_result {0 {2 3}}
          108  +}
          109  +
          110  +#-------------------------------------------------------------------------
          111  +# Test OOM injection with nested colsets.
          112  +#
          113  +reset_db
          114  +do_execsql_test 4.0 {
          115  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
          116  +  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
          117  +  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
          118  +  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
          119  +  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
          120  +}
          121  +do_faultsim_test 4.1 -faults oom* -body {
          122  +  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
          123  +} -test {
          124  +  faultsim_test_result {0 2}
          125  +}
          126  +
          127  +do_faultsim_test 4.2 -faults oom* -body {
          128  +  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
          129  +} -test {
          130  +  faultsim_test_result {0 {2 3}}
          131  +}
          132  +
    81    133   
    82    134   finish_test
    83         -

Added ext/fts5/test/fts5faultD.test.

            1  +# 2016 February 2
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +# This file is focused on OOM errors.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +source $testdir/malloc_common.tcl
           17  +set testprefix fts5faultA
           18  +
           19  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           20  +ifcapable !fts5 {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +foreach_detail_mode $testprefix {
           26  +  if {"%DETAIL%"=="none"} continue
           27  +
           28  +  do_execsql_test 1.0 {
           29  +    CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%);
           30  +    INSERT INTO o1(o1, rank) VALUES('pgsz', 32);
           31  +
           32  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
           33  +    INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s;
           34  +
           35  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
           36  +    INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s;
           37  +
           38  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 )
           39  +    INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s;
           40  +  }
           41  +  
           42  +  do_faultsim_test 1 -faults int* -prep {
           43  +    sqlite3 db test.db
           44  +  } -body {
           45  +    execsql { SELECT count(*) FROM o1('a') }
           46  +  } -test {
           47  +    faultsim_test_result {0 180} {1 {vtable constructor failed: o1}}
           48  +  }
           49  +
           50  +  do_faultsim_test 2 -faults int* -prep {
           51  +    sqlite3 db test.db
           52  +  } -body {
           53  +    execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank }
           54  +    expr 1
           55  +  } -test {
           56  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           57  +  }
           58  +
           59  +  do_faultsim_test 3 -faults int* -prep {
           60  +    sqlite3 db test.db
           61  +  } -body {
           62  +    execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank }
           63  +    expr 1
           64  +  } -test {
           65  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           66  +  }
           67  +
           68  +  do_faultsim_test 4 -faults int* -prep {
           69  +    sqlite3 db test.db
           70  +  } -body {
           71  +    execsql { SELECT * FROM o1('b:b OR a:a') }
           72  +    expr 1
           73  +  } -test {
           74  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           75  +  }
           76  +
           77  +  do_faultsim_test 5 -faults int* -prep {
           78  +    sqlite3 db test.db
           79  +  } -body {
           80  +    execsql { SELECT count(*) FROM o1('c:b') }
           81  +    expr 1
           82  +  } -test {
           83  +    faultsim_test_result {0 1} {1 {vtable constructor failed: o1}}
           84  +  }
           85  +}
           86  +
           87  +finish_test

Changes to ext/fts5/test/fts5full.test.

    36     36         execsql { INSERT INTO x8 VALUES( rnddoc(5) ); }
    37     37       }
    38     38     } msg] $msg
    39     39   } {1 {database or disk is full}}
    40     40   
    41     41   
    42     42   finish_test
    43         -

Changes to ext/fts5/test/fts5fuzz1.test.

    86     86   reset_db
    87     87   do_catchsql_test 4.1 {
    88     88     CREATE VIRTUAL TABLE f2 USING fts5(o, t);
    89     89     SELECT * FROM f2('(8 AND 9)`AND 10');
    90     90   } {1 {fts5: syntax error near "`"}}
    91     91   
    92     92   finish_test
    93         -

Changes to ext/fts5/test/fts5hash.test.

   117    117       set hash [sqlite3_fts5_token_hash 1024 $big]
   118    118       while {1} {
   119    119         set small [random_token]
   120    120         if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break
   121    121       }
   122    122   
   123    123       execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) }
   124         -breakpoint
   125    124       execsql {
   126    125         INSERT INTO t2 VALUES($small || ' ' || $big);
   127    126       }
   128    127     } {}
   129    128   
   130    129   } ;# foreach_detail_mode
   131    130   
   132    131   finish_test
   133         -

Changes to ext/fts5/test/fts5integrity.test.

   206    206         if {$res == [lsort -integer $res2]} { incr ok }
   207    207       }
   208    208       set ok
   209    209     } {1000}
   210    210   }
   211    211   
   212    212   finish_test
   213         -

Added ext/fts5/test/fts5lastrowid.test.

            1  +# 2017 Feb 27
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Tests of the last_insert_rowid functionality with fts5.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5lastrowid
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(str);
           26  +}
           27  +
           28  +do_execsql_test 1.1 {
           29  +  INSERT INTO t1 VALUES('one string');
           30  +  INSERT INTO t1 VALUES('two string');
           31  +  INSERT INTO t1 VALUES('three string');
           32  +  SELECT last_insert_rowid();
           33  +} {3}
           34  +
           35  +do_execsql_test 1.2 {
           36  +  BEGIN;
           37  +    INSERT INTO t1 VALUES('one string');
           38  +    INSERT INTO t1 VALUES('two string');
           39  +    INSERT INTO t1 VALUES('three string');
           40  +  COMMIT;
           41  +  SELECT last_insert_rowid();
           42  +} {6}
           43  +
           44  +do_execsql_test 1.3 {
           45  +  INSERT INTO t1(rowid, str) VALUES(-22, 'some more text');
           46  +  SELECT last_insert_rowid();
           47  +} {-22}
           48  +
           49  +do_execsql_test 1.4 {
           50  +  BEGIN;
           51  +    INSERT INTO t1(rowid, str) VALUES(45, 'some more text');
           52  +    INSERT INTO t1(rowid, str) VALUES(46, 'some more text');
           53  +    INSERT INTO t1(rowid, str) VALUES(222, 'some more text');
           54  +    SELECT last_insert_rowid();
           55  +  COMMIT;
           56  +  SELECT last_insert_rowid();
           57  +} {222 222}
           58  +
           59  +do_execsql_test 1.5 {
           60  +  CREATE TABLE x1(x);
           61  +  INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo');
           62  +  INSERT INTO t1 SELECT x FROM x1;
           63  +  SELECT last_insert_rowid();
           64  +} {226}
           65  +
           66  +do_execsql_test 1.6 {
           67  +  INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
           68  +  SELECT last_insert_rowid();
           69  +} {14}
           70  +
           71  +
           72  +finish_test

Added ext/fts5/test/fts5leftjoin.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5leftjoin
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE vt USING fts5(x);
           26  +  INSERT INTO vt VALUES('abc');
           27  +  INSERT INTO vt VALUES('xyz');
           28  +
           29  +  CREATE TABLE t1(a INTEGER PRIMARY KEY);
           30  +  INSERT INTO t1 VALUES(1), (2);
           31  +}
           32  +
           33  +do_execsql_test 1.1 {
           34  +  SELECT * FROM t1 LEFT JOIN (
           35  +    SELECT rowid AS rrr, * FROM vt WHERE vt MATCH 'abc'
           36  +  ) ON t1.a = rrr
           37  +} {1 1 abc 2 {} {}}
           38  +
           39  +do_execsql_test 1.2 {
           40  +  SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc')
           41  +} {1 abc 2 abc}
           42  +
           43  +finish_test

Changes to ext/fts5/test/fts5matchinfo.test.

   468    468   } ;# foreach_detail_mode
   469    469   
   470    470   #-------------------------------------------------------------------------
   471    471   # Test that a bad fts5() return is detected
   472    472   #
   473    473   reset_db
   474    474   proc xyz {} {}
   475         -db func fts5 -argcount 0 xyz
          475  +db func fts5 -argcount 1 xyz
   476    476   do_test 13.1 {
   477    477     list [catch { sqlite3_fts5_register_matchinfo db } msg] $msg
   478    478   } {1 SQLITE_ERROR}
   479    479   
   480    480   #-------------------------------------------------------------------------
   481    481   # Test that an invalid matchinfo() flag is detected
   482    482   #
................................................................................
   488    488   } {}
   489    489   
   490    490   do_catchsql_test 14.2 {
   491    491     SELECT matchinfo(x1, 'd') FROM x1('a b c');
   492    492   } {1 {unrecognized matchinfo flag: d}}
   493    493   
   494    494   finish_test
   495         -

Changes to ext/fts5/test/fts5merge.test.

   237    237   do_execsql_test 6.3 {
   238    238     INSERT INTO g1(g1) VALUES('integrity-check');
   239    239   }
   240    240   
   241    241   
   242    242   
   243    243   finish_test
   244         -

Changes to ext/fts5/test/fts5merge2.test.

    51     51   do_execsql_test 1.2 {
    52     52     INSERT INTO t1(t1) VALUES('integrity-check');
    53     53   }
    54     54   
    55     55   }
    56     56   
    57     57   finish_test
    58         -

Changes to ext/fts5/test/fts5multiclient.test.

    41     41       sql1 { INSERT INTO t1 VALUES('a b c') }
    42     42       sql3 { INSERT INTO t1(t1) VALUES('integrity-check') }
    43     43     } {}
    44     44   
    45     45   };# do_multiclient_test
    46     46   };# foreach_detail_mode
    47     47   finish_test
    48         -

Changes to ext/fts5/test/fts5near.test.

    64     64   do_near_test 1.23 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 4) } 0
    65     65   
    66     66   do_near_test 1.24 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 5) } 1
    67     67   do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0
    68     68   
    69     69   
    70     70   finish_test
    71         -

Changes to ext/fts5/test/fts5onepass.test.

   174    174       UPDATE ttt SET x = 'A B C' WHERE rowid = 4;
   175    175       INSERT INTO ttt(rowid, x) VALUES(6, 'd e f');
   176    176     COMMIT;
   177    177   } {}
   178    178   do_test 4.2.2 { fts5_level_segs ttt } {3}
   179    179   
   180    180   finish_test
   181         -

Changes to ext/fts5/test/fts5optimize.test.

   102    102     do_execsql_test 2.$tn.5 {
   103    103       INSERT INTO t1(t1) VALUES('integrity-check');
   104    104     }
   105    105   
   106    106     do_test 2.$tn.6 { fts5_segcount t1 } 1
   107    107   }
   108    108   finish_test
   109         -

Changes to ext/fts5/test/fts5phrase.test.

   112    112     FROM t3('a:f+f')
   113    113   } {
   114    114     31 {h *f f*} {i j g e c} {j j f c a i j} 
   115    115     50 {*f f* c} {f f b i i} {f f a j e c i}
   116    116   }
   117    117   
   118    118   finish_test
   119         -

Changes to ext/fts5/test/fts5plan.test.

    26     26     CREATE VIRTUAL TABLE f1 USING fts5(ff);
    27     27   }
    28     28   
    29     29   do_eqp_test 1.1 {
    30     30     SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
    31     31   } {
    32     32     0 0 0 {SCAN TABLE t1} 
    33         -  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
           33  +  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
    34     34   }
    35     35   
    36     36   do_eqp_test 1.2 {
    37     37     SELECT * FROM t1, f1 WHERE f1 > t1.x
    38     38   } {
    39     39     0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
    40     40     0 1 0 {SCAN TABLE t1} 
    41     41   }
    42     42   
    43     43   do_eqp_test 1.3 {
    44     44     SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
    45     45   } {
    46         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
           46  +  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
    47     47     0 0 0 {USE TEMP B-TREE FOR ORDER BY}
    48     48   }
    49     49   
    50     50   do_eqp_test 1.4 {
    51     51     SELECT * FROM f1 ORDER BY rank
    52     52   } {
    53     53     0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
................................................................................
    60     60     0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
    61     61   }
    62     62   
    63     63   
    64     64   
    65     65   
    66     66   finish_test
    67         -

Changes to ext/fts5/test/fts5porter.test.

 11799  11799       lindex [sqlite3_fts5_tokenize db porter $in] 0
 11800  11800     } $out
 11801  11801     incr i
 11802  11802   }
 11803  11803   
 11804  11804   
 11805  11805   finish_test
 11806         -

Changes to ext/fts5/test/fts5porter2.test.

    63     63       lindex [sqlite3_fts5_tokenize db porter $in] 0
    64     64     } $out
    65     65     incr i
    66     66   }
    67     67   
    68     68   
    69     69   finish_test
    70         -

Changes to ext/fts5/test/fts5prefix.test.

     5      5   #
     6      6   #    May you do good and not evil.
     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   #
    12         -# This file containst tests focused on prefix indexes.
           12  +# This file contains tests focused on prefix indexes.
    13     13   #
    14     14   
    15     15   source [file join [file dirname [info script]] fts5_common.tcl]
    16     16   set testprefix fts5prefix
    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    19     19   ifcapable !fts5 {
................................................................................
   337    337       do_execsql_test 7.$tn {
   338    338         SELECT md5sum(id, block) FROM tt_data
   339    339       } [list $::checksum]
   340    340     }
   341    341   }
   342    342   
   343    343   finish_test
   344         -
   345         -

Changes to ext/fts5/test/fts5query.test.

    75     75       } {}
    76     76       incr ret
    77     77     }
    78     78   }
    79     79   
    80     80   
    81     81   finish_test
    82         -
    83         -

Changes to ext/fts5/test/fts5rank.test.

    86     86     execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
    87     87   } {1 3 2}
    88     88   
    89     89   do_test 2.7 {
    90     90     execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
    91     91   } {1 3 2}
    92     92   
           93  +db2 close
    93     94   
    94     95   #--------------------------------------------------------------------------
    95     96   # At one point there was a problem with queries such as:
    96     97   #
    97     98   #   ... MATCH 'x OR y' ORDER BY rank;
    98     99   #
    99    100   # if there were zero occurrences of token 'y' in the dataset. The
................................................................................
   147    148     VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
   148    149   } {{wrinkle in time} {Bill Smith}}
   149    150   
   150    151   
   151    152   
   152    153   
   153    154   finish_test
   154         -

Changes to ext/fts5/test/fts5rebuild.test.

    60     60     CREATE VIRTUAL TABLE nc USING fts5(doc, content=);
    61     61   }
    62     62   
    63     63   do_catchsql_test 2.2 {
    64     64     INSERT INTO nc(nc) VALUES('rebuild');
    65     65   } {1 {'rebuild' may not be used with a contentless fts5 table}}
    66     66   finish_test
    67         -

Changes to ext/fts5/test/fts5restart.test.

   145    145     }
   146    146     set res
   147    147   } {500 400 300}
   148    148   
   149    149   
   150    150   
   151    151   finish_test
   152         -

Changes to ext/fts5/test/fts5rowid.test.

   212    212   } {36}
   213    213   
   214    214   #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r}
   215    215   
   216    216   
   217    217   
   218    218   finish_test
   219         -

Changes to ext/fts5/test/fts5simple.test.

   407    407   
   408    408   do_catchsql_test 19.2 {
   409    409     SELECT * FROM x1 WHERE x1 MATCH 'c0 AND (c1 AND (c2 AND (c3 AND (c4 AND (c5 AND (c6 AND (c7 AND (c8 AND (c9 AND (c10 AND (c11 AND (c12 AND (c13 AND (c14 AND (c15 AND (c16 AND (c17 AND (c18 AND (c19 AND (c20 AND (c21 AND (c22 AND (c23 AND (c24 AND (c25 AND (c26 AND (c27 AND (c28 AND (c29 AND (c30 AND (c31 AND (c32 AND (c33 AND (c34 AND (c35 AND (c36 AND (c37 AND (c38 AND (c39 AND (c40 AND (c41 AND (c42 AND (c43 AND (c44 AND (c45 AND (c46 AND (c47 AND (c48 AND (c49 AND (c50 AND (c51 AND (c52 AND (c53 AND (c54 AND (c55 AND (c56 AND (c57 AND (c58 AND (c59 AND (c60 AND (c61 AND (c62 AND (c63 AND (c64 AND (c65 AND (c66 AND (c67 AND (c68 AND (c69 AND (c70 AND (c71 AND (c72 AND (c73 AND (c74 AND (c75 AND (c76 AND (c77 AND (c78 AND (c79 AND (c80 AND (c81 AND (c82 AND (c83 AND (c84 AND (c85 AND (c86 AND (c87 AND (c88 AND (c89 AND (c90 AND (c91 AND (c92 AND (c93 AND (c94 AND (c95 AND (c96 AND (c97 AND (c98 AND (c99 AND (c100 AND (c101 AND (c102 AND (c103 AND (c104 AND (c105 AND (c106 AND (c107 AND (c108 AND (c109 AND (c110 AND (c111 AND (c112 AND (c113 AND (c114 AND (c115 AND (c116 AND (c117 AND (c118 AND (c119 AND (c120 AND (c121 AND (c122 AND (c123 AND (c124 AND (c125 AND (c126 AND (c127 AND (c128 AND (c129 AND (c130 AND (c131 AND (c132 AND (c133 AND (c134 AND (c135 AND (c136 AND (c137 AND (c138 AND (c139 AND (c140 AND (c141 AND (c142 AND (c143 AND (c144 AND (c145 AND (c146 AND (c147 AND (c148 AND (c149 AND (c150 AND (c151 AND (c152 AND (c153 AND (c154 AND (c155 AND (c156 AND (c157 AND (c158 AND (c159 AND (c160 AND (c161 AND (c162 AND (c163 AND (c164 AND (c165 AND (c166 AND (c167 AND (c168 AND (c169 AND (c170 AND (c171 AND (c172 AND (c173 AND (c174 AND (c175 AND (c176 AND (c177 AND (c178 AND (c179 AND (c180 AND (c181 AND (c182 AND (c183 AND (c184 AND (c185 AND (c186 AND (c187 AND (c188 AND (c189 AND (c190 AND (c191 AND (c192 AND (c193 AND (c194 AND (c195 AND (c196 AND (c197 AND (c198 AND (c199 AND c200)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))';
   410    410   } {1 {fts5: parser stack overflow}}
   411    411   
   412    412   #-------------------------------------------------------------------------
   413    413   reset_db
   414         -breakpoint
   415    414   do_execsql_test 20.0 {
   416    415     CREATE VIRTUAL TABLE x1 USING fts5(x);
   417    416     INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
   418    417     INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree');
   419    418   }
   420    419   do_test 20.1 {
   421    420     for {set i 1} {$i <= 200} {incr i} {

Changes to ext/fts5/test/fts5simple2.test.

   327    327       INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
   328    328       INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
   329    329       INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
   330    330     COMMIT;
   331    331     UPDATE t2 SET x=x;
   332    332     DELETE FROM t2;
   333    333   }
          334  +
          335  +#-------------------------------------------------------------------------
          336  +#
          337  +reset_db
          338  +do_execsql_test 17.0 {
          339  +  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
          340  +  BEGIN;
          341  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          342  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          343  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          344  +  COMMIT;
          345  +}
          346  +do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 }
          347  +do_execsql_test 17.2 {
          348  +  BEGIN;
          349  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          350  +    SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 ;
          351  +}
          352  +do_execsql_test 17.3 {
          353  +  COMMIT
          354  +}
          355  +
          356  +reset_db
          357  +do_execsql_test 17.4 {
          358  +  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
          359  +  BEGIN;
          360  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          361  +    INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb');
          362  +    SELECT * FROM t2('y:a*') WHERE rowid>66;
          363  +}
          364  +do_execsql_test 17.5 { SELECT * FROM t2('x:b* OR y:a*') }
          365  +do_execsql_test 17.5 { COMMIT ; SELECT * FROM t2('x:b* OR y:a*') }
          366  +do_execsql_test 17.6 { 
          367  +  SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55
          368  +}
   334    369   
   335    370   #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
   336    371     
   337    372   finish_test
   338         -

Changes to ext/fts5/test/fts5simple3.test.

    76     76     CREATE VIRTUAL TABLE x3 USING fts5(one);
    77     77     INSERT INTO x3 VALUES('a b c');
    78     78     INSERT INTO x3 VALUES('c b a');
    79     79     INSERT INTO x3 VALUES('o t t');
    80     80     SELECT * FROM x3('x OR y OR z');
    81     81   }
    82     82   
           83  +#-------------------------------------------------------------------------
           84  +# Test that a crash occuring when the second or subsequent tokens in a
           85  +# phrase matched zero rows has been fixed.
           86  +#
           87  +do_execsql_test 4.0 {
           88  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           89  +  INSERT INTO t1 VALUES('ab');
           90  +  INSERT INTO t1 VALUES('cd');
           91  +  INSERT INTO t1 VALUES('ab cd');
           92  +  INSERT INTO t1 VALUES('ab cdXXX');
           93  +  INSERT INTO t1 VALUES('abXXX cd');
           94  +}
           95  +do_execsql_test 4.1 {
           96  +  SELECT * FROM t1('"ab cd" OR "ab cd" *');
           97  +} {{ab cd} {ab cdXXX}}
           98  +do_execsql_test 4.2 {
           99  +  SELECT * FROM t1('"xy zz" OR "ab cd" *');
          100  +} {{ab cd} {ab cdXXX}}
          101  +do_execsql_test 4.3 {
          102  +  SELECT * FROM t1('"xy zz" OR "xy zz" *');
          103  +}
          104  +do_execsql_test 4.4 {
          105  +  SELECT * FROM t1('"ab cd" OR "xy zz" *');
          106  +} {{ab cd}}
          107  +do_execsql_test 4.5 {
          108  +  CREATE VIRTUAL TABLE t2 USING fts5(x);
          109  +  INSERT INTO t2 VALUES('ab');
          110  +  INSERT INTO t2 VALUES('cd');
          111  +  INSERT INTO t2 VALUES('ef');
          112  +} 
          113  +do_execsql_test 4.6 {
          114  +  SELECT * FROM t2('ab + xyz');
          115  +}
          116  +
    83    117   
    84    118   finish_test
    85         -

Changes to ext/fts5/test/fts5synonym.test.

   148    148   reset_db
   149    149   fts5_tclnum_register db
   150    150   
   151    151   foreach {tn expr res} {
   152    152     1  {abc}                           {"abc"}
   153    153     2  {one}                           {"one"|"i"|"1"}
   154    154     3  {3}                             {"3"|"iii"|"three"}
   155         -  4  {3*}                            {"3"|"iii"|"three" *}
          155  +  4  {3*}                            {"3" *}
   156    156   } {
   157    157     do_execsql_test 4.1.$tn {
   158    158       SELECT fts5_expr($expr, 'tokenize=tclnum')
   159    159     } [list $res]
   160    160   }
   161    161   
   162    162   do_execsql_test 4.2.1 {
................................................................................
   417    417   do_execsql_test 7.1.2 {
   418    418     INSERT INTO t2(t2) VALUES('integrity-check');
   419    419   }
   420    420   
   421    421   } ;# foreach_detail_mode
   422    422   
   423    423   finish_test
   424         -

Changes to ext/fts5/test/fts5synonym2.test.

   157    157   
   158    158   }
   159    159   
   160    160   }
   161    161   }
   162    162   
   163    163   finish_test
   164         -

Changes to ext/fts5/test/fts5tok1.test.

   105    105   do_catchsql_test 2.0 {
   106    106     CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer);
   107    107   } {1 {vtable constructor failed: tX}}
   108    108   
   109    109   do_catchsql_test 2.1 {
   110    110     CREATE VIRTUAL TABLE t4 USING fts5tokenize;
   111    111     SELECT * FROM t4;
   112         -} {1 {SQL logic error or missing database}}
          112  +} {1 {SQL logic error}}
   113    113   
   114    114   
   115    115   finish_test

Changes to ext/fts5/test/fts5tokenizer.test.

   258    258     CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
   259    259     SELECT term FROM e7;
   260    260     ROLLBACK;
   261    261   } {
   262    262     brown dog fox jump lazi over quick the
   263    263   }
   264    264   
   265         -finish_test
          265  +#-------------------------------------------------------------------------
          266  +# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer
          267  +# implementation.
          268  +#
          269  +reset_db
          270  +proc tcl_create {args} { return "tcl_tokenize" }
          271  +sqlite3_fts5_create_tokenizer db tcl tcl_create
          272  +set ::flags [list]
          273  +proc tcl_tokenize {tflags text} {
          274  +  lappend ::flags $tflags
          275  +  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
          276  +    sqlite3_fts5_token $w $iStart $iEnd
          277  +  }
          278  +}
          279  +
          280  +do_execsql_test 9.1.1 {
          281  +  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
          282  +  INSERT INTO t1 VALUES('abc');
          283  +  INSERT INTO t1 VALUES('xyz');
          284  +} {}
          285  +do_test 9.1.2 { set ::flags } {document document}
          286  +
          287  +set ::flags [list]
          288  +do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc}
          289  +do_test 9.2.2 { set ::flags } {query}
          290  +
          291  +set ::flags [list]
          292  +do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc}
          293  +do_test 9.3.2 { set ::flags } {prefixquery}
          294  +
          295  +set ::flags [list]
          296  +do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {}
          297  +do_test 9.4.2 { set ::flags } {prefixquery}
          298  +
          299  +set ::flags [list]
          300  +do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
          301  +do_test 9.5.2 { set ::flags } {query}
          302  +
   266    303   
          304  +finish_test

Changes to ext/fts5/test/fts5unicode.test.

    46     46     CREATE VIRTUAL TABLE t1 USING fts5(x);
    47     47     CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
    48     48     CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii);
    49     49     INSERT INTO t1 VALUES('\xC0\xC8\xCC');
    50     50     INSERT INTO t2 VALUES('\xC0\xC8\xCC');
    51     51     INSERT INTO t3 VALUES('\xC0\xC8\xCC');
    52     52   "
    53         -breakpoint
    54     53   do_execsql_test 2.1 "
    55     54     SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC';
    56     55     SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC';
    57     56     SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
    58     57   " {t1 t2}
    59     58   
    60     59   
    61     60   finish_test
    62         -

Changes to ext/fts5/test/fts5unicode2.test.

   277    277       INSERT INTO t9(a) VALUES('abc%88def %89ghi%90');
   278    278     }
   279    279   } {0 {}}
   280    280   
   281    281   
   282    282   #-------------------------------------------------------------------------
   283    283   
   284         -breakpoint
   285    284   do_unicode_token_test3 5.1 {tokenchars {}} {
   286    285     sqlite3_reset sqlite3_column_int
   287    286   } {
   288    287     sqlite3 sqlite3 
   289    288     reset reset 
   290    289     sqlite3 sqlite3 
   291    290     column column 

Changes to ext/fts5/test/fts5unicode3.test.

   122    122     }
   123    123     append str {'");}
   124    124     execsql $str
   125    125   } {}
   126    126   
   127    127   
   128    128   finish_test
   129         -

Changes to ext/fts5/test/fts5unindexed.test.

    72     72     INSERT INTO t4(t4, rowid, a, b, c) VALUES('delete', 20, 'j k l', '', 'p q r');
    73     73     DELETE FROM x4 WHERE rowid=20;
    74     74     INSERT INTO t4(t4) VALUES('integrity-check');
    75     75   } {}
    76     76   
    77     77   
    78     78   finish_test
    79         -

Changes to ext/fts5/test/fts5update.test.

   113    113   } {}
   114    114   do_execsql_test 2.2.integrity {
   115    115     INSERT INTO x2(x2) VALUES('integrity-check');
   116    116   }
   117    117   
   118    118   }
   119    119   finish_test
   120         -
   121         -

Changes to ext/fts5/test/fts5version.test.

    57     57     db close
    58     58     sqlite3 db test.db
    59     59     catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
    60     60   } {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
    61     61   
    62     62   
    63     63   finish_test
    64         -

Changes to ext/fts5/test/fts5vocab.test.

   206    206     INSERT INTO temp.t1 VALUES('1 5 3');
   207    207   
   208    208     INSERT INTO aux.t1 VALUES('x y z');
   209    209     INSERT INTO aux.t1 VALUES('m n o');
   210    210     INSERT INTO aux.t1 VALUES('x n z');
   211    211   }
   212    212   
   213         -breakpoint
   214    213   do_execsql_test 5.1 {
   215    214     CREATE VIRTUAL TABLE temp.vm  USING fts5vocab(main, t1, row);
   216    215     CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row);
   217    216     CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row);
   218    217     CREATE VIRTUAL TABLE temp.va  USING fts5vocab(aux, t1, row);
   219    218   }
   220    219   

Added ext/fts5/test/fts5vocab2.test.

            1  +# 2017 August 10
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# The tests in this file focus on testing the fts5vocab module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5vocab
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
           26  +  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
           27  +
           28  +  INSERT INTO t1 VALUES('one two', 'two three');
           29  +  INSERT INTO t1 VALUES('three four', 'four five five five');
           30  +}
           31  +
           32  +do_execsql_test 1.1 {
           33  +  SELECT * FROM v1;
           34  +} {
           35  +  five  2 b 1
           36  +  five  2 b 2
           37  +  five  2 b 3
           38  +  four  2 a 1
           39  +  four  2 b 0
           40  +  one   1 a 0
           41  +  three 1 b 1
           42  +  three 2 a 0
           43  +  two   1 a 1
           44  +  two   1 b 0
           45  +}
           46  +
           47  +do_execsql_test 1.2 {
           48  +  SELECT * FROM v1 WHERE term='three';
           49  +} {
           50  +  three 1 b 1
           51  +  three 2 a 0
           52  +}
           53  +
           54  +do_execsql_test 1.3 {
           55  +  BEGIN;
           56  +    DELETE FROM t1 WHERE rowid=2;
           57  +    SELECT * FROM v1;
           58  +  ROLLBACK;
           59  +} {
           60  +  one   1 a 0
           61  +  three 1 b 1
           62  +  two   1 a 1
           63  +  two   1 b 0
           64  +}
           65  +
           66  +do_execsql_test 1.4 {
           67  +  BEGIN;
           68  +    DELETE FROM t1 WHERE rowid=1;
           69  +    SELECT * FROM v1;
           70  +  ROLLBACK;
           71  +} {
           72  +  five  2 b 1
           73  +  five  2 b 2
           74  +  five  2 b 3
           75  +  four  2 a 1
           76  +  four  2 b 0
           77  +  three 2 a 0
           78  +}
           79  +
           80  +do_execsql_test 1.5 {
           81  +  DELETE FROM t1;
           82  +  SELECT * FROM v1;
           83  +} {
           84  +}
           85  +
           86  +#-------------------------------------------------------------------------
           87  +#
           88  +do_execsql_test 2.0 {
           89  +  DROP TABLE IF EXISTS t1;
           90  +  DROP TABLE IF EXISTS v1;
           91  +
           92  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=column);
           93  +  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
           94  +
           95  +  INSERT INTO t1 VALUES('one two', 'two three');
           96  +  INSERT INTO t1 VALUES('three four', 'four five five five');
           97  +}
           98  +
           99  +do_execsql_test 2.1 {
          100  +  SELECT * FROM v1;
          101  +} {
          102  +  five  2 b {}
          103  +  four  2 a {}
          104  +  four  2 b {}
          105  +  one   1 a {}
          106  +  three 1 b {}
          107  +  three 2 a {}
          108  +  two   1 a {}
          109  +  two   1 b {}
          110  +}
          111  +
          112  +do_execsql_test 2.2 {
          113  +  SELECT * FROM v1 WHERE term='three';
          114  +} {
          115  +  three 1 b {}
          116  +  three 2 a {}
          117  +}
          118  +
          119  +do_execsql_test 2.3 {
          120  +  BEGIN;
          121  +    DELETE FROM t1 WHERE rowid=2;
          122  +    SELECT * FROM v1;
          123  +  ROLLBACK;
          124  +} {
          125  +  one   1 a {}
          126  +  three 1 b {}
          127  +  two   1 a {}
          128  +  two   1 b {}
          129  +}
          130  +
          131  +do_execsql_test 2.4 {
          132  +  BEGIN;
          133  +    DELETE FROM t1 WHERE rowid=1;
          134  +    SELECT * FROM v1;
          135  +  ROLLBACK;
          136  +} {
          137  +  five  2 b {}
          138  +  four  2 a {}
          139  +  four  2 b {}
          140  +  three 2 a {}
          141  +}
          142  +
          143  +do_execsql_test 2.5 {
          144  +  DELETE FROM t1;
          145  +  SELECT * FROM v1;
          146  +} {
          147  +}
          148  +
          149  +#-------------------------------------------------------------------------
          150  +#
          151  +do_execsql_test 3.0 {
          152  +  DROP TABLE IF EXISTS t1;
          153  +  DROP TABLE IF EXISTS v1;
          154  +
          155  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
          156  +  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance);
          157  +
          158  +  INSERT INTO t1 VALUES('one two', 'two three');
          159  +  INSERT INTO t1 VALUES('three four', 'four five five five');
          160  +}
          161  +
          162  +do_execsql_test 3.1 {
          163  +  SELECT * FROM v1;
          164  +} {
          165  +  five  2 {} {}
          166  +  four  2 {} {}
          167  +  one   1 {} {}
          168  +  three 1 {} {}
          169  +  three 2 {} {}
          170  +  two   1 {} {}
          171  +}
          172  +
          173  +do_execsql_test 3.2 {
          174  +  SELECT * FROM v1 WHERE term='three';
          175  +} {
          176  +  three 1 {} {}
          177  +  three 2 {} {}
          178  +}
          179  +
          180  +do_execsql_test 3.3 {
          181  +  BEGIN;
          182  +    DELETE FROM t1 WHERE rowid=2;
          183  +    SELECT * FROM v1;
          184  +  ROLLBACK;
          185  +} {
          186  +  one   1 {} {}
          187  +  three 1 {} {}
          188  +  two   1 {} {}
          189  +}
          190  +
          191  +do_execsql_test 3.4 {
          192  +  BEGIN;
          193  +    DELETE FROM t1 WHERE rowid=1;
          194  +    SELECT * FROM v1;
          195  +  ROLLBACK;
          196  +} {
          197  +  five  2 {} {}
          198  +  four  2 {} {}
          199  +  three 2 {} {}
          200  +}
          201  +
          202  +do_execsql_test 3.5 {
          203  +  DELETE FROM t1;
          204  +  SELECT * FROM v1;
          205  +} {
          206  +}
          207  +
          208  +finish_test
          209  +

Changes to ext/icu/icu.c.

    98     98   ** false (0) if they are different.
    99     99   */
   100    100   static int icuLikeCompare(
   101    101     const uint8_t *zPattern,   /* LIKE pattern */
   102    102     const uint8_t *zString,    /* The UTF-8 string to compare against */
   103    103     const UChar32 uEsc         /* The escape character */
   104    104   ){
   105         -  static const int MATCH_ONE = (UChar32)'_';
   106         -  static const int MATCH_ALL = (UChar32)'%';
          105  +  static const uint32_t MATCH_ONE = (uint32_t)'_';
          106  +  static const uint32_t MATCH_ALL = (uint32_t)'%';
   107    107   
   108    108     int prevEscape = 0;     /* True if the previous character was uEsc */
   109    109   
   110    110     while( 1 ){
   111    111   
   112    112       /* Read (and consume) the next character from the input pattern. */
   113         -    UChar32 uPattern;
          113  +    uint32_t uPattern;
   114    114       SQLITE_ICU_READ_UTF8(zPattern, uPattern);
   115    115       if( uPattern==0 ) break;
   116    116   
   117    117       /* There are now 4 possibilities:
   118    118       **
   119    119       **     1. uPattern is an unescaped match-all character "%",
   120    120       **     2. uPattern is an unescaped match-one character "_",
................................................................................
   148    148         return 0;
   149    149   
   150    150       }else if( !prevEscape && uPattern==MATCH_ONE ){
   151    151         /* Case 2. */
   152    152         if( *zString==0 ) return 0;
   153    153         SQLITE_ICU_SKIP_UTF8(zString);
   154    154   
   155         -    }else if( !prevEscape && uPattern==uEsc){
          155  +    }else if( !prevEscape && uPattern==(uint32_t)uEsc){
   156    156         /* Case 3. */
   157    157         prevEscape = 1;
   158    158   
   159    159       }else{
   160    160         /* Case 4. */
   161         -      UChar32 uString;
          161  +      uint32_t uString;
   162    162         SQLITE_ICU_READ_UTF8(zString, uString);
   163         -      uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT);
   164         -      uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT);
          163  +      uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT);
          164  +      uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT);
   165    165         if( uString!=uPattern ){
   166    166           return 0;
   167    167         }
   168    168         prevEscape = 0;
   169    169       }
   170    170     }
   171    171   
................................................................................
   489    489     }
   490    490   }
   491    491   
   492    492   /*
   493    493   ** Register the ICU extension functions with database db.
   494    494   */
   495    495   int sqlite3IcuInit(sqlite3 *db){
   496         -  struct IcuScalar {
          496  +  static const struct IcuScalar {
   497    497       const char *zName;                        /* Function name */
   498         -    int nArg;                                 /* Number of arguments */
   499         -    int enc;                                  /* Optimal text encoding */
   500         -    void *pContext;                           /* sqlite3_user_data() context */
          498  +    unsigned char nArg;                       /* Number of arguments */
          499  +    unsigned short enc;                       /* Optimal text encoding */
          500  +    unsigned char iContext;                   /* sqlite3_user_data() context */
   501    501       void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   502    502     } scalars[] = {
   503         -    {"regexp", 2, SQLITE_ANY,          0, icuRegexpFunc},
   504         -
   505         -    {"lower",  1, SQLITE_UTF16,        0, icuCaseFunc16},
   506         -    {"lower",  2, SQLITE_UTF16,        0, icuCaseFunc16},
   507         -    {"upper",  1, SQLITE_UTF16, (void*)1, icuCaseFunc16},
   508         -    {"upper",  2, SQLITE_UTF16, (void*)1, icuCaseFunc16},
   509         -
   510         -    {"lower",  1, SQLITE_UTF8,         0, icuCaseFunc16},
   511         -    {"lower",  2, SQLITE_UTF8,         0, icuCaseFunc16},
   512         -    {"upper",  1, SQLITE_UTF8,  (void*)1, icuCaseFunc16},
   513         -    {"upper",  2, SQLITE_UTF8,  (void*)1, icuCaseFunc16},
   514         -
   515         -    {"like",   2, SQLITE_UTF8,         0, icuLikeFunc},
   516         -    {"like",   3, SQLITE_UTF8,         0, icuLikeFunc},
   517         -
   518         -    {"icu_load_collation",  2, SQLITE_UTF8, (void*)db, icuLoadCollation},
          503  +    {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
          504  +    {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
          505  +    {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
          506  +    {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
          507  +    {"upper",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
          508  +    {"upper",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
          509  +    {"lower",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
          510  +    {"lower",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
          511  +    {"upper",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
          512  +    {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
          513  +    {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
          514  +    {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
   519    515     };
   520         -
   521    516     int rc = SQLITE_OK;
   522    517     int i;
   523    518   
          519  +  
   524    520     for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
   525         -    struct IcuScalar *p = &scalars[i];
          521  +    const struct IcuScalar *p = &scalars[i];
   526    522       rc = sqlite3_create_function(
   527         -        db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0
          523  +        db, p->zName, p->nArg, p->enc, 
          524  +        p->iContext ? (void*)db : (void*)0,
          525  +        p->xFunc, 0, 0
   528    526       );
   529    527     }
   530    528   
   531    529     return rc;
   532    530   }
   533    531   
   534    532   #if !SQLITE_CORE

Added ext/lsm1/Makefile.

            1  +#
            2  +# This Makefile is designed for use with main.mk in the root directory of
            3  +# this project. After including main.mk, the users makefile should contain:
            4  +#
            5  +#    LSMDIR=$(TOP)/ext/lsm1/
            6  +#    LSMOPTS=-fPIC
            7  +#    include $(LSMDIR)/Makefile
            8  +#
            9  +# The most useful targets are [lsmtest] and [lsm.so].
           10  +#
           11  +
           12  +LSMOBJ    = \
           13  +  lsm_ckpt.o \
           14  +  lsm_file.o \
           15  +  lsm_log.o \
           16  +  lsm_main.o \
           17  +  lsm_mem.o \
           18  +  lsm_mutex.o \
           19  +  lsm_shared.o \
           20  +  lsm_sorted.o \
           21  +  lsm_str.o \
           22  +  lsm_tree.o \
           23  +  lsm_unix.o \
           24  +  lsm_win32.o \
           25  +  lsm_varint.o \
           26  +  lsm_vtab.o
           27  +
           28  +LSMHDR   = \
           29  +  $(LSMDIR)/lsm.h \
           30  +  $(LSMDIR)/lsmInt.h
           31  +
           32  +LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c     \
           33  +             $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c     \
           34  +             $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c     \
           35  +             $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c     \
           36  +             $(LSMDIR)/lsm-test/lsmtest9.c                                   \
           37  +             $(LSMDIR)/lsm-test/lsmtest_datasource.c \
           38  +             $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c  \
           39  +             $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \
           40  +             $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \
           41  +             $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c
           42  +
           43  +
           44  +# all: lsm.so
           45  +
           46  +LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
           47  +
           48  +lsm.so:	$(LSMOBJ)
           49  +	$(TCCX) -shared -o lsm.so $(LSMOBJ)
           50  +
           51  +%.o:	$(LSMDIR)/%.c $(LSMHDR) sqlite3.h
           52  +	$(TCCX) $(LSMOPTS) -c $<
           53  +	
           54  +lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
           55  +	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
           56  +	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)

Added ext/lsm1/Makefile.msc.

            1  +#
            2  +# This Makefile is designed for use with Makefile.msc in the root directory
            3  +# of this project.  The Makefile.msc should contain:
            4  +#
            5  +#    LSMDIR=$(TOP)\ext\lsm1
            6  +#    !INCLUDE $(LSMDIR)\Makefile.msc
            7  +#
            8  +# The most useful targets are [lsmtest.exe] and [lsm.dll].
            9  +#
           10  +
           11  +LSMOBJ    = \
           12  +  lsm_ckpt.lo \
           13  +  lsm_file.lo \
           14  +  lsm_log.lo \
           15  +  lsm_main.lo \
           16  +  lsm_mem.lo \
           17  +  lsm_mutex.lo \
           18  +  lsm_shared.lo \
           19  +  lsm_sorted.lo \
           20  +  lsm_str.lo \
           21  +  lsm_tree.lo \
           22  +  lsm_unix.lo \
           23  +  lsm_win32.lo \
           24  +  lsm_varint.lo \
           25  +  lsm_vtab.lo
           26  +
           27  +LSMHDR   = \
           28  +  $(LSMDIR)\lsm.h \
           29  +  $(LSMDIR)\lsmInt.h
           30  +
           31  +LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c     \
           32  +             $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c     \
           33  +             $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c     \
           34  +             $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c     \
           35  +             $(LSMDIR)\lsm-test\lsmtest9.c                                   \
           36  +             $(LSMDIR)\lsm-test\lsmtest_datasource.c \
           37  +             $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c  \
           38  +             $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \
           39  +             $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \
           40  +             $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c
           41  +
           42  +# all: lsm.dll lsmtest.exe
           43  +
           44  +LSMOPTS = $(NO_WARN) -DLSM_MUTEX_WIN32=1 -I$(LSMDIR)
           45  +
           46  +!IF $(DEBUG)>2
           47  +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1
           48  +!ENDIF
           49  +
           50  +!IF $(MEMDEBUG)!=0
           51  +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1
           52  +!ENDIF
           53  +
           54  +lsm_ckpt.lo:	$(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H)
           55  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c
           56  +
           57  +lsm_file.lo:	$(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H)
           58  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c
           59  +
           60  +lsm_log.lo:	$(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H)
           61  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c
           62  +
           63  +lsm_main.lo:	$(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H)
           64  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c
           65  +
           66  +lsm_mem.lo:	$(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H)
           67  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c
           68  +
           69  +lsm_mutex.lo:	$(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H)
           70  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c
           71  +
           72  +lsm_shared.lo:	$(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H)
           73  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c
           74  +
           75  +lsm_sorted.lo:	$(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H)
           76  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c
           77  +
           78  +lsm_str.lo:	$(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H)
           79  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c
           80  +
           81  +lsm_tree.lo:	$(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H)
           82  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c
           83  +
           84  +lsm_unix.lo:	$(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H)
           85  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c
           86  +
           87  +lsm_win32.lo:	$(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H)
           88  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c
           89  +
           90  +lsm_varint.lo:	$(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H)
           91  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c
           92  +
           93  +lsm_vtab.lo:	$(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H)
           94  +	$(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c
           95  +
           96  +lsm.dll:	$(LSMOBJ)
           97  +	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ)
           98  +	copy /Y $@ $(LSMDIR)\$@
           99  +
          100  +lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJ)
          101  +	$(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJ)
          102  +	copy /Y $@ $(LSMDIR)\$@

Added ext/lsm1/lsm-test/README.

            1  +
            2  +
            3  +Organization of test case files:
            4  +
            5  +  lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a 
            6  +              database file, then verify that the contents of the database can
            7  +              be queried.
            8  +
            9  +  lsmtest2.c: Crash tests. Tests that attempt to verify that the database 
           10  +              recovers correctly following an application or system crash.
           11  +
           12  +  lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of
           13  +              transactions and sub-transactions.
           14  +
           15  +  lsmtest4.c: Multi-client tests.
           16  +
           17  +  lsmtest5.c: Multi-client tests with a different thread for each client.
           18  +
           19  +  lsmtest6.c: OOM injection tests.
           20  +
           21  +  lsmtest7.c: API tests.
           22  +
           23  +  lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that
           24  +              the system recovers and other clients proceed unaffected if
           25  +              a process fails in the middle of a write transaction.
           26  +
           27  +              The difference from lsmtest2.c is that this file tests
           28  +              live-recovery (recovery from a failure that occurs while other
           29  +              clients are still running) whereas lsmtest2.c tests recovery
           30  +              from a system or power failure.
           31  +
           32  +  lsmtest9.c: More data tests. These focus on testing that calling
           33  +              lsm_work(nMerge=1) to compact the database does not corrupt it.
           34  +              In other words, that databases containing block-redirects
           35  +              can be read and written.
           36  +
           37  +
           38  +
           39  +
           40  +

Added ext/lsm1/lsm-test/lsmtest.h.

            1  +
            2  +#ifndef __WRAPPER_INT_H_
            3  +#define __WRAPPER_INT_H_
            4  +
            5  +#include "lsmtest_tdb.h"
            6  +#include "sqlite3.h"
            7  +#include "lsm.h"
            8  +
            9  +#include <assert.h>
           10  +#include <stdarg.h>
           11  +#include <stdlib.h>
           12  +#include <string.h>
           13  +#include <stdio.h>
           14  +#ifndef _WIN32
           15  +# include <unistd.h>
           16  +#endif
           17  +#include <sys/types.h>
           18  +#include <sys/stat.h>
           19  +#include <fcntl.h>
           20  +#include <ctype.h>
           21  +#include <stdlib.h>
           22  +#include <errno.h>
           23  +
           24  +#ifdef __cplusplus
           25  +extern "C" {
           26  +#endif
           27  +
           28  +#ifdef _WIN32
           29  +# include "windows.h"
           30  +# define gettimeofday win32GetTimeOfDay
           31  +# define F_OK  (0)
           32  +# define sleep(sec) Sleep(1000 * (sec))
           33  +# define usleep(usec) Sleep(((usec) + 999) / 1000)
           34  +# ifdef _MSC_VER
           35  +#  include <io.h>
           36  +#  define snprintf _snprintf
           37  +#  define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
           38  +#  define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd)))
           39  +#  define __va_copy(dst,src) ((dst) = (src))
           40  +#  define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1)
           41  +# else
           42  +#  error Unsupported C compiler for Windows.
           43  +# endif
           44  +int win32GetTimeOfDay(struct timeval *, void *);
           45  +#endif
           46  +
           47  +#ifndef _LSM_INT_H
           48  +typedef unsigned int  u32;
           49  +typedef unsigned char u8;
           50  +typedef long long int i64;
           51  +typedef unsigned long long int u64;
           52  +#endif
           53  +
           54  +
           55  +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0])))
           56  +
           57  +#define MIN(x,y) ((x)<(y) ? (x) : (y))
           58  +#define MAX(x,y) ((x)>(y) ? (x) : (y))
           59  +
           60  +#define unused_parameter(x) (void)(x)
           61  +
           62  +#define TESTDB_DEFAULT_PAGE_SIZE   4096
           63  +#define TESTDB_DEFAULT_CACHE_SIZE  2048
           64  +
           65  +#ifndef _O_BINARY
           66  +# define _O_BINARY (0)
           67  +#endif
           68  +
           69  +/*
           70  +** Ideally, these should be in wrapper.c. But they are here instead so that 
           71  +** they can be used by the C++ database wrappers in wrapper2.cc.
           72  +*/
           73  +typedef struct DatabaseMethods DatabaseMethods;
           74  +struct TestDb {
           75  +  DatabaseMethods const *pMethods;          /* Database methods */
           76  +  const char *zLibrary;                     /* Library name for tdb_open() */
           77  +};
           78  +struct DatabaseMethods {
           79  +  int (*xClose)(TestDb *);
           80  +  int (*xWrite)(TestDb *, void *, int , void *, int);
           81  +  int (*xDelete)(TestDb *, void *, int);
           82  +  int (*xDeleteRange)(TestDb *, void *, int, void *, int);
           83  +  int (*xFetch)(TestDb *, void *, int, void **, int *);
           84  +  int (*xScan)(TestDb *, void *, int, void *, int, void *, int,
           85  +    void (*)(void *, void *, int , void *, int)
           86  +  );
           87  +  int (*xBegin)(TestDb *, int);
           88  +  int (*xCommit)(TestDb *, int);
           89  +  int (*xRollback)(TestDb *, int);
           90  +};
           91  +
           92  +/* 
           93  +** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the
           94  +** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but
           95  +** the primary interface is the C++ API.
           96  +*/
           97  +int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb);
           98  +int test_kc_close(TestDb *);
           99  +int test_kc_write(TestDb *, void *, int , void *, int);
          100  +int test_kc_delete(TestDb *, void *, int);
          101  +int test_kc_delete_range(TestDb *, void *, int, void *, int);
          102  +int test_kc_fetch(TestDb *, void *, int, void **, int *);
          103  +int test_kc_scan(TestDb *, void *, int, void *, int, void *, int,
          104  +  void (*)(void *, void *, int , void *, int)
          105  +);
          106  +
          107  +int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          108  +int test_mdb_close(TestDb *);
          109  +int test_mdb_write(TestDb *, void *, int , void *, int);
          110  +int test_mdb_delete(TestDb *, void *, int);
          111  +int test_mdb_fetch(TestDb *, void *, int, void **, int *);
          112  +int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int,
          113  +  void (*)(void *, void *, int , void *, int)
          114  +);
          115  +
          116  +/* 
          117  +** Functions in wrapper3.c. This file contains the tdb wrapper for lsm.
          118  +** The wrapper for lsm is a bit more involved than the others, as it 
          119  +** includes code for a couple of different lsm configurations, and for
          120  +** various types of fault injection and robustness testing.
          121  +*/
          122  +int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          123  +int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
          124  +int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
          125  +int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
          126  +int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
          127  +int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
          128  +
          129  +int tdb_lsm_configure(lsm_db *, const char *);
          130  +
          131  +/* Functions in lsmtest_tdb4.c */
          132  +int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          133  +int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          134  +int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
          135  +
          136  +
          137  +/* Functions in testutil.c. */
          138  +int  testPrngInit(void);
          139  +u32  testPrngValue(u32 iVal);
          140  +void testPrngArray(u32 iVal, u32 *aOut, int nOut);
          141  +void testPrngString(u32 iVal, char *aOut, int nOut);
          142  +
          143  +void testErrorInit(int argc, char **);
          144  +void testPrintError(const char *zFormat, ...);
          145  +void testPrintUsage(const char *zArgs);
          146  +void testPrintFUsage(const char *zFormat, ...);
          147  +void testTimeInit(void);
          148  +int  testTimeGet(void);
          149  +
          150  +/* Functions in testmem.c. */
          151  +void testMallocInstall(lsm_env *pEnv);
          152  +void testMallocUninstall(lsm_env *pEnv);
          153  +void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *);
          154  +void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *);
          155  +void testMallocOomEnable(lsm_env *pEnv, int);
          156  +
          157  +/* lsmtest.c */
          158  +TestDb *testOpen(const char *zSystem, int, int *pRc);
          159  +void testReopen(TestDb **ppDb, int *pRc);
          160  +void testClose(TestDb **ppDb);
          161  +
          162  +void testFetch(TestDb *, void *, int, void *, int, int *);
          163  +void testWrite(TestDb *, void *, int, void *, int, int *);
          164  +void testDelete(TestDb *, void *, int, int *);
          165  +void testDeleteRange(TestDb *, void *, int, void *, int, int *);
          166  +void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc);
          167  +void testFetchStr(TestDb *, const char *, const char *, int *pRc);
          168  +
          169  +void testBegin(TestDb *pDb, int iTrans, int *pRc);
          170  +void testCommit(TestDb *pDb, int iTrans, int *pRc);
          171  +
          172  +void test_failed(void);
          173  +
          174  +char *testMallocPrintf(const char *zFormat, ...);
          175  +char *testMallocVPrintf(const char *zFormat, va_list ap);
          176  +int testGlobMatch(const char *zPattern, const char *zStr);
          177  +
          178  +void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *);
          179  +void testFetchCompare(TestDb *, TestDb *, void *, int, int *);
          180  +
          181  +void *testMalloc(int);
          182  +void *testMallocCopy(void *pCopy, int nByte);
          183  +void *testRealloc(void *, int);
          184  +void testFree(void *);
          185  +
          186  +/* lsmtest_bt.c */
          187  +int do_bt(int nArg, char **azArg);
          188  +
          189  +/* testio.c */
          190  +int testVfsConfigureDb(TestDb *pDb);
          191  +
          192  +/* testfunc.c */
          193  +int do_show(int nArg, char **azArg);
          194  +int do_work(int nArg, char **azArg);
          195  +
          196  +/* testio.c */
          197  +int do_io(int nArg, char **azArg);
          198  +
          199  +/* lsmtest2.c */
          200  +void do_crash_test(const char *zPattern, int *pRc);
          201  +int do_rollback_test(int nArg, char **azArg);
          202  +
          203  +/* test3.c */
          204  +void test_rollback(const char *zSystem, const char *zPattern, int *pRc);
          205  +
          206  +/* test4.c */
          207  +void test_mc(const char *zSystem, const char *zPattern, int *pRc);
          208  +
          209  +/* test5.c */
          210  +void test_mt(const char *zSystem, const char *zPattern, int *pRc);
          211  +
          212  +/* lsmtest6.c */
          213  +void test_oom(const char *zPattern, int *pRc);
          214  +void testDeleteLsmdb(const char *zFile);
          215  +
          216  +void testSaveDb(const char *zFile, const char *zAuxExt);
          217  +void testRestoreDb(const char *zFile, const char *zAuxExt);
          218  +void testCopyLsmdb(const char *zFrom, const char *zTo);
          219  +
          220  +/* lsmtest7.c */
          221  +void test_api(const char *zPattern, int *pRc);
          222  +
          223  +/* lsmtest8.c */
          224  +void do_writer_crash_test(const char *zPattern, int *pRc);
          225  +
          226  +/*************************************************************************
          227  +** Interface to functionality in test_datasource.c.
          228  +*/
          229  +typedef struct Datasource Datasource;
          230  +typedef struct DatasourceDefn DatasourceDefn;
          231  +
          232  +struct DatasourceDefn {
          233  +  int eType;                      /* A TEST_DATASOURCE_* value */
          234  +  int nMinKey;                    /* Minimum key size */
          235  +  int nMaxKey;                    /* Maximum key size */
          236  +  int nMinVal;                    /* Minimum value size */
          237  +  int nMaxVal;                    /* Maximum value size */
          238  +};
          239  +
          240  +#define TEST_DATASOURCE_RANDOM    1
          241  +#define TEST_DATASOURCE_SEQUENCE  2
          242  +
          243  +char *testDatasourceName(const DatasourceDefn *);
          244  +Datasource *testDatasourceNew(const DatasourceDefn *);
          245  +void testDatasourceFree(Datasource *);
          246  +void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *);
          247  +/* End of test_datasource.c interface.
          248  +*************************************************************************/
          249  +void testDatasourceFetch(
          250  +  TestDb *pDb,                    /* Database handle */
          251  +  Datasource *pData,
          252  +  int iKey,
          253  +  int *pRc                        /* IN/OUT: Error code */
          254  +);
          255  +
          256  +void testWriteDatasource(TestDb *, Datasource *, int, int *);
          257  +void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *);
          258  +void testDeleteDatasource(TestDb *, Datasource *, int, int *);
          259  +void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *);
          260  +
          261  +
          262  +/* test1.c */
          263  +void test_data_1(const char *, const char *, int *pRc);
          264  +void test_data_2(const char *, const char *, int *pRc);
          265  +void test_data_3(const char *, const char *, int *pRc);
          266  +void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *);
          267  +void testCaseProgress(int, int, int, int *);
          268  +int testCaseNDot(void);
          269  +
          270  +void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *);
          271  +int testControlDb(TestDb **ppDb);
          272  +
          273  +typedef struct CksumDb CksumDb;
          274  +CksumDb *testCksumArrayNew(Datasource *, int, int, int);
          275  +char *testCksumArrayGet(CksumDb *, int);
          276  +void testCksumArrayFree(CksumDb *);
          277  +void testCaseStart(int *pRc, char *zFmt, ...);
          278  +void testCaseFinish(int rc);
          279  +void testCaseSkip(void);
          280  +int testCaseBegin(int *, const char *, const char *, ...);
          281  +
          282  +#define TEST_CKSUM_BYTES 29
          283  +int testCksumDatabase(TestDb *pDb, char *zOut);
          284  +int testCountDatabase(TestDb *pDb);
          285  +void testCompareInt(int, int, int *);
          286  +void testCompareStr(const char *z1, const char *z2, int *pRc);
          287  +
          288  +/* lsmtest9.c */
          289  +void test_data_4(const char *, const char *, int *pRc);
          290  +
          291  +
          292  +/*
          293  +** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function.
          294  +*/
          295  +#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z)
          296  +int testArgSelectX(void *, const char *, int, const char *, int *);
          297  +
          298  +#ifdef __cplusplus
          299  +}  /* End of the 'extern "C"' block */
          300  +#endif
          301  +
          302  +#endif

Added ext/lsm1/lsm-test/lsmtest1.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE
            5  +#define DATA_RANDOM     TEST_DATASOURCE_RANDOM
            6  +
            7  +typedef struct Datatest1 Datatest1;
            8  +typedef struct Datatest2 Datatest2;
            9  +
           10  +/*
           11  +** An instance of the following structure contains parameters used to
           12  +** customize the test function in this file. Test procedure:
           13  +**
           14  +**   1. Create a data-source based on the "datasource definition" vars.
           15  +**
           16  +**   2. Insert nRow key value pairs into the database.
           17  +**
           18  +**   3. Delete all keys from the database. Deletes are done in the same 
           19  +**      order as the inserts.
           20  +**
           21  +** During steps 2 and 3 above, after each Datatest1.nVerify inserts or
           22  +** deletes, the following:
           23  +**
           24  +**   a. Run Datasource.nTest key lookups and check the results are as expected.
           25  +**
           26  +**   b. If Datasource.bTestScan is true, run a handful (8) of range
           27  +**      queries (scanning forwards and backwards). Check that the results
           28  +**      are as expected.
           29  +**
           30  +**   c. Close and reopen the database. Then run (a) and (b) again.
           31  +*/
           32  +struct Datatest1 {
           33  +  /* Datasource definition */
           34  +  DatasourceDefn defn;
           35  +
           36  +  /* Test procedure parameters */
           37  +  int nRow;                       /* Number of rows to insert then delete */
           38  +  int nVerify;                    /* How often to verify the db contents */
           39  +  int nTest;                      /* Number of keys to test (0==all) */
           40  +  int bTestScan;                  /* True to do scan tests */
           41  +};
           42  +
           43  +/*
           44  +** An instance of the following data structure is used to describe the
           45  +** second type of test case in this file. The chief difference between 
           46  +** these tests and those described by Datatest1 is that these tests also
           47  +** experiment with range-delete operations. Tests proceed as follows:
           48  +**
           49  +**     1. Open the datasource described by Datatest2.defn. 
           50  +**
           51  +**     2. Open a connection on an empty database.
           52  +**
           53  +**     3. Do this Datatest2.nIter times:
           54  +**
           55  +**        a) Insert Datatest2.nWrite key-value pairs from the datasource.
           56  +**
           57  +**        b) Select two pseudo-random keys and use them as the start
           58  +**           and end points of a range-delete operation.
           59  +**
           60  +**        c) Verify that the contents of the database are as expected (see
           61  +**           below for details).
           62  +**
           63  +**        d) Close and then reopen the database handle.
           64  +**
           65  +**        e) Verify that the contents of the database are still as expected.
           66  +**
           67  +** The inserts and range deletes are run twice - once on the database being
           68  +** tested and once using a control system (sqlite3, kc etc. - something that 
           69  +** works). In order to verify that the contents of the db being tested are
           70  +** correct, the test runs a bunch of scans and lookups on both the test and
           71  +** control databases. If the results are the same, the test passes.
           72  +*/
           73  +struct Datatest2 {
           74  +  DatasourceDefn defn;
           75  +  int nRange;
           76  +  int nWrite;                     /* Number of writes per iteration */
           77  +  int nIter;                      /* Total number of iterations to run */
           78  +};
           79  +
           80  +/*
           81  +** Generate a unique name for the test case pTest with database system
           82  +** zSystem.
           83  +*/
           84  +static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){
           85  +  char *zRet;
           86  +  char *zData;
           87  +  zData = testDatasourceName(&pTest->defn);
           88  +  zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", 
           89  +      zSystem, zData, bRecover, pTest->nRow, pTest->nVerify
           90  +  );
           91  +  testFree(zData);
           92  +  return zRet;
           93  +}
           94  +
           95  +int testControlDb(TestDb **ppDb){
           96  +#ifdef HAVE_KYOTOCABINET
           97  +  return tdb_open("kyotocabinet", "tmp.db", 1, ppDb);
           98  +#else
           99  +  return tdb_open("sqlite3", "", 1, ppDb);
          100  +#endif
          101  +}
          102  +
          103  +void testDatasourceFetch(
          104  +  TestDb *pDb,                    /* Database handle */
          105  +  Datasource *pData,
          106  +  int iKey,
          107  +  int *pRc                        /* IN/OUT: Error code */
          108  +){
          109  +  void *pKey; int nKey;           /* Database key to query for */
          110  +  void *pVal; int nVal;           /* Expected result of query */
          111  +
          112  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          113  +  testFetch(pDb, pKey, nKey, pVal, nVal, pRc);
          114  +}
          115  +
          116  +/*
          117  +** This function is called to test that the contents of database pDb
          118  +** are as expected. In this case, expected is defined as containing
          119  +** key-value pairs iFirst through iLast, inclusive, from data source 
          120  +** pData. In other words, a loop like the following could be used to
          121  +** construct a database with identical contents from scratch.
          122  +**
          123  +**   for(i=iFirst; i<=iLast; i++){
          124  +**     testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
          125  +**     // insert (pKey, nKey) -> (pVal, nVal) into database
          126  +**   }
          127  +**
          128  +** The key domain consists of keys 0 to (nRow-1), inclusive, from
          129  +** data source pData. For both scan and lookup tests, keys are selected
          130  +** pseudo-randomly from within this set.
          131  +**
          132  +** This function runs nLookupTest lookup tests and nScanTest scan tests.
          133  +**
          134  +** A lookup test consists of selecting a key from the domain and querying
          135  +** pDb for it. The test fails if the presence of the key and, if present,
          136  +** the associated value do not match the expectations defined above.
          137  +**
          138  +** A scan test involves selecting a key from the domain and running
          139  +** the following queries:
          140  +**
          141  +**   1. Scan all keys equal to or greater than the key, in ascending order.
          142  +**   2. Scan all keys equal to or smaller than the key, in descending order.
          143  +**
          144  +** Additionally, if nLookupTest is greater than zero, the following are
          145  +** run once:
          146  +**
          147  +**   1. Scan all keys in the db, in ascending order.
          148  +**   2. Scan all keys in the db, in descending order.
          149  +**
          150  +** As you would assume, the test fails if the returned values do not match
          151  +** expectations.
          152  +*/
          153  +void testDbContents(
          154  +  TestDb *pDb,                    /* Database handle being tested */
          155  +  Datasource *pData,              /* pDb contains data from here */
          156  +  int nRow,                       /* Size of key domain */
          157  +  int iFirst,                     /* Index of first key from pData in pDb */
          158  +  int iLast,                      /* Index of last key from pData in pDb */
          159  +  int nLookupTest,                /* Number of lookup tests to run */
          160  +  int nScanTest,                  /* Number of scan tests to run */
          161  +  int *pRc                        /* IN/OUT: Error code */
          162  +){
          163  +  int j;
          164  +  int rc = *pRc;
          165  +
          166  +  if( rc==0 && nScanTest ){
          167  +    TestDb *pDb2 = 0;
          168  +
          169  +    /* Open a control db (i.e. one that we assume works) */
          170  +    rc = testControlDb(&pDb2);
          171  +
          172  +    for(j=iFirst; rc==0 && j<=iLast; j++){
          173  +      void *pKey; int nKey;         /* Database key to insert */
          174  +      void *pVal; int nVal;         /* Database value to insert */
          175  +      testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal);
          176  +      rc = tdb_write(pDb2, pKey, nKey, pVal, nVal);
          177  +    }
          178  +
          179  +    if( rc==0 ){
          180  +      int iKey1;
          181  +      int iKey2;
          182  +      void *pKey1; int nKey1;       /* Start key */
          183  +      void *pKey2; int nKey2;       /* Final key */
          184  +
          185  +      iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow;
          186  +      iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow;
          187  +      testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
          188  +      pKey1 = testMalloc(nKey1+1);
          189  +      memcpy(pKey1, pKey2, nKey1+1);
          190  +      testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
          191  +
          192  +      testScanCompare(pDb2, pDb, 0, 0, 0,         0, 0,         &rc);
          193  +      testScanCompare(pDb2, pDb, 0, 0, 0,         pKey2, nKey2, &rc);
          194  +      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0,         &rc);
          195  +      testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc);
          196  +      testScanCompare(pDb2, pDb, 1, 0, 0,         0, 0,         &rc);
          197  +      testScanCompare(pDb2, pDb, 1, 0, 0,         pKey2, nKey2, &rc);
          198  +      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0,         &rc);
          199  +      testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc);
          200  +      testFree(pKey1);
          201  +    }
          202  +    tdb_close(pDb2);
          203  +  }
          204  +
          205  +  /* Test some lookups. */
          206  +  for(j=0; rc==0 && j<nLookupTest; j++){
          207  +    int iKey;                     /* Datasource key to test */
          208  +    void *pKey; int nKey;         /* Database key to query for */
          209  +    void *pVal; int nVal;         /* Expected result of query */
          210  +
          211  +    if( nLookupTest>=nRow ){
          212  +      iKey = j;
          213  +    }else{
          214  +      iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow;
          215  +    }
          216  +
          217  +    testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          218  +    if( iFirst>iKey || iKey>iLast ){
          219  +      pVal = 0;
          220  +      nVal = -1;
          221  +    }
          222  +
          223  +    testFetch(pDb, pKey, nKey, pVal, nVal, &rc);
          224  +  }
          225  +
          226  +  *pRc = rc;
          227  +}
          228  +
          229  +/*
          230  +** This function should be called during long running test cases to output
          231  +** the progress dots (...) to stdout.
          232  +*/
          233  +void testCaseProgress(int i, int n, int nDot, int *piDot){
          234  +  int iDot = *piDot;
          235  +  while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){
          236  +    printf(".");
          237  +    fflush(stdout);
          238  +    iDot++;
          239  +  }
          240  +  *piDot = iDot;
          241  +}
          242  +
          243  +int testCaseNDot(void){ return 20; }
          244  +
          245  +#if 0
          246  +static void printScanCb(
          247  +    void *pCtx, void *pKey, int nKey, void *pVal, int nVal
          248  +){
          249  +  printf("%s\n", (char *)pKey);
          250  +  fflush(stdout);
          251  +}
          252  +#endif
          253  +
          254  +void testReopenRecover(TestDb **ppDb, int *pRc){
          255  +  if( *pRc==0 ){
          256  +    const char *zLib = tdb_library_name(*ppDb);
          257  +    const char *zDflt = tdb_default_db(zLib);
          258  +    testCopyLsmdb(zDflt, "bak.db");
          259  +    testClose(ppDb);
          260  +    testCopyLsmdb("bak.db", zDflt);
          261  +    *pRc = tdb_open(zLib, 0, 0, ppDb);
          262  +  }
          263  +}
          264  +
          265  +
          266  +static void doDataTest1(
          267  +  const char *zSystem,            /* Database system to test */
          268  +  int bRecover,
          269  +  Datatest1 *p,                   /* Structure containing test parameters */
          270  +  int *pRc                        /* OUT: Error code */
          271  +){
          272  +  int i;
          273  +  int iDot;
          274  +  int rc = LSM_OK;
          275  +  Datasource *pData;
          276  +  TestDb *pDb;
          277  +
          278  +  /* Start the test case, open a database and allocate the datasource. */
          279  +  pDb = testOpen(zSystem, 1, &rc);
          280  +  pData = testDatasourceNew(&p->defn);
          281  +
          282  +  i = 0;
          283  +  iDot = 0;
          284  +  while( rc==LSM_OK && i<p->nRow ){
          285  +
          286  +    /* Insert some data */
          287  +    testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
          288  +    i += p->nVerify;
          289  +
          290  +    /* Check that the db content is correct. */
          291  +    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
          292  +
          293  +    if( bRecover ){
          294  +      testReopenRecover(&pDb, &rc);
          295  +    }else{
          296  +      testReopen(&pDb, &rc);
          297  +    }
          298  +
          299  +    /* Check that the db content is still correct. */
          300  +    testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
          301  +
          302  +    /* Update the progress dots... */
          303  +    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
          304  +  }
          305  +
          306  +  i = 0;
          307  +  iDot = 0;
          308  +  while( rc==LSM_OK && i<p->nRow ){
          309  +
          310  +    /* Delete some entries */
          311  +    testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
          312  +    i += p->nVerify;
          313  +
          314  +    /* Check that the db content is correct. */
          315  +    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
          316  +
          317  +    /* Close and reopen the database. */
          318  +    if( bRecover ){
          319  +      testReopenRecover(&pDb, &rc);
          320  +    }else{
          321  +      testReopen(&pDb, &rc);
          322  +    }
          323  +
          324  +    /* Check that the db content is still correct. */
          325  +    testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc);
          326  +
          327  +    /* Update the progress dots... */
          328  +    testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot);
          329  +  }
          330  +
          331  +  /* Free the datasource, close the database and finish the test case. */
          332  +  testDatasourceFree(pData);
          333  +  tdb_close(pDb);
          334  +  testCaseFinish(rc);
          335  +  *pRc = rc;
          336  +}
          337  +
          338  +
          339  +void test_data_1(
          340  +  const char *zSystem,            /* Database system name */
          341  +  const char *zPattern,           /* Run test cases that match this pattern */
          342  +  int *pRc                        /* IN/OUT: Error code */
          343  +){
          344  +  Datatest1 aTest[] = {
          345  +    { {DATA_RANDOM,     500,600,   1000,2000},     1000,  100,  10,  0},
          346  +    { {DATA_RANDOM,     20,25,     100,200},       1000,  250, 1000, 1},
          347  +    { {DATA_RANDOM,     8,10,      100,200},       1000,  250, 1000, 1},
          348  +    { {DATA_RANDOM,     8,10,      10,20},         1000,  250, 1000, 1},
          349  +    { {DATA_RANDOM,     8,10,      1000,2000},     1000,  250, 1000, 1},
          350  +    { {DATA_RANDOM,     8,100,     10000,20000},    100,   25,  100, 1},
          351  +    { {DATA_RANDOM,     80,100,    10,20},         1000,  250, 1000, 1},
          352  +    { {DATA_RANDOM,     5000,6000, 10,20},          100,   25,  100, 1},
          353  +    { {DATA_SEQUENTIAL, 5,10,      10,20},         1000,  250, 1000, 1},
          354  +    { {DATA_SEQUENTIAL, 5,10,      100,200},       1000,  250, 1000, 1},
          355  +    { {DATA_SEQUENTIAL, 5,10,      1000,2000},     1000,  250, 1000, 1},
          356  +    { {DATA_SEQUENTIAL, 5,100,     10000,20000},    100,   25,  100, 1},
          357  +    { {DATA_RANDOM,     10,10,     100,100},     100000, 1000,  100, 0},
          358  +    { {DATA_SEQUENTIAL, 10,10,     100,100},     100000, 1000,  100, 0},
          359  +  };
          360  +
          361  +  int i;
          362  +  int bRecover;
          363  +
          364  +  for(bRecover=0; bRecover<2; bRecover++){
          365  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          366  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          367  +      char *zName = getName(zSystem, bRecover, &aTest[i]);
          368  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          369  +        doDataTest1(zSystem, bRecover, &aTest[i], pRc);
          370  +      }
          371  +      testFree(zName);
          372  +    }
          373  +  }
          374  +}
          375  +
          376  +void testCompareDb(
          377  +  Datasource *pData,
          378  +  int nData,
          379  +  int iSeed,
          380  +  TestDb *pControl,
          381  +  TestDb *pDb,
          382  +  int *pRc
          383  +){
          384  +  int i;
          385  +
          386  +  static int nCall = 0;
          387  +  nCall++;
          388  +
          389  +  testScanCompare(pControl, pDb, 0, 0, 0,         0, 0,         pRc);
          390  +  testScanCompare(pControl, pDb, 1, 0, 0,         0, 0,         pRc);
          391  +
          392  +  if( *pRc==0 ){
          393  +    int iKey1;
          394  +    int iKey2;
          395  +    void *pKey1; int nKey1;       /* Start key */
          396  +    void *pKey2; int nKey2;       /* Final key */
          397  +
          398  +    iKey1 = testPrngValue(iSeed) % nData;
          399  +    iKey2 = testPrngValue(iSeed+1) % nData;
          400  +    testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0);
          401  +    pKey1 = testMalloc(nKey1+1);
          402  +    memcpy(pKey1, pKey2, nKey1+1);
          403  +    testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0);
          404  +
          405  +    testScanCompare(pControl, pDb, 0, 0, 0,         pKey2, nKey2, pRc);
          406  +    testScanCompare(pControl, pDb, 0, pKey1, nKey1, 0, 0,         pRc);
          407  +    testScanCompare(pControl, pDb, 0, pKey1, nKey1, pKey2, nKey2, pRc);
          408  +    testScanCompare(pControl, pDb, 1, 0, 0,         pKey2, nKey2, pRc);
          409  +    testScanCompare(pControl, pDb, 1, pKey1, nKey1, 0, 0,         pRc);
          410  +    testScanCompare(pControl, pDb, 1, pKey1, nKey1, pKey2, nKey2, pRc);
          411  +    testFree(pKey1);
          412  +  }
          413  +
          414  +  for(i=0; i<nData && *pRc==0; i++){
          415  +    void *pKey; int nKey;
          416  +    testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
          417  +    testFetchCompare(pControl, pDb, pKey, nKey, pRc);
          418  +  }
          419  +}
          420  +
          421  +static void doDataTest2(
          422  +  const char *zSystem,            /* Database system to test */
          423  +  int bRecover,
          424  +  Datatest2 *p,                   /* Structure containing test parameters */
          425  +  int *pRc                        /* OUT: Error code */
          426  +){
          427  +  TestDb *pDb;
          428  +  TestDb *pControl;
          429  +  Datasource *pData;
          430  +  int i;
          431  +  int rc = LSM_OK;
          432  +  int iDot = 0;
          433  +
          434  +  /* Start the test case, open a database and allocate the datasource. */
          435  +  pDb = testOpen(zSystem, 1, &rc);
          436  +  pData = testDatasourceNew(&p->defn);
          437  +  rc = testControlDb(&pControl);
          438  +
          439  +  if( tdb_lsm(pDb) ){
          440  +    int nBuf = 32 * 1024 * 1024;
          441  +    lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf);
          442  +  }
          443  +
          444  +  for(i=0; rc==0 && i<p->nIter; i++){
          445  +    void *pKey1; int nKey1;
          446  +    void *pKey2; int nKey2;
          447  +    int ii;
          448  +    int nRange = MIN(p->nIter*p->nWrite, p->nRange);
          449  +
          450  +    for(ii=0; rc==0 && ii<p->nWrite; ii++){
          451  +      int iKey = (i*p->nWrite + ii) % p->nRange;
          452  +      testWriteDatasource(pControl, pData, iKey, &rc);
          453  +      testWriteDatasource(pDb, pData, iKey, &rc);
          454  +    }
          455  +
          456  +    testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0);
          457  +    pKey1 = testMallocCopy(pKey1, nKey1);
          458  +    testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0);
          459  +
          460  +    testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc);
          461  +    testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc);
          462  +    testFree(pKey1);
          463  +
          464  +    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
          465  +    if( bRecover ){
          466  +      testReopenRecover(&pDb, &rc);
          467  +    }else{
          468  +      testReopen(&pDb, &rc);
          469  +    }
          470  +    testCompareDb(pData, nRange, i, pControl, pDb, &rc);
          471  +
          472  +    /* Update the progress dots... */
          473  +    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
          474  +  }
          475  +
          476  +  testClose(&pDb);
          477  +  testClose(&pControl);
          478  +  testDatasourceFree(pData);
          479  +  testCaseFinish(rc);
          480  +  *pRc = rc;
          481  +}
          482  +
          483  +static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){
          484  +  char *zRet;
          485  +  char *zData;
          486  +  zData = testDatasourceName(&pTest->defn);
          487  +  zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", 
          488  +      zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter
          489  +  );
          490  +  testFree(zData);
          491  +  return zRet;
          492  +}
          493  +
          494  +void test_data_2(
          495  +  const char *zSystem,            /* Database system name */
          496  +  const char *zPattern,           /* Run test cases that match this pattern */
          497  +  int *pRc                        /* IN/OUT: Error code */
          498  +){
          499  +  Datatest2 aTest[] = {
          500  +      /* defn,                                 nRange, nWrite, nIter */
          501  +    { {DATA_RANDOM,     20,25,     100,200},   10000,  10,     50   },
          502  +    { {DATA_RANDOM,     20,25,     100,200},   10000,  200,    50   },
          503  +    { {DATA_RANDOM,     20,25,     100,200},   100,    10,     1000 },
          504  +    { {DATA_RANDOM,     20,25,     100,200},   100,    200,    50   },
          505  +  };
          506  +
          507  +  int i;
          508  +  int bRecover;
          509  +
          510  +  for(bRecover=0; bRecover<2; bRecover++){
          511  +    if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break;
          512  +    for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          513  +      char *zName = getName2(zSystem, bRecover, &aTest[i]);
          514  +      if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          515  +        doDataTest2(zSystem, bRecover, &aTest[i], pRc);
          516  +      }
          517  +      testFree(zName);
          518  +    }
          519  +  }
          520  +}
          521  +
          522  +/*************************************************************************
          523  +** Test case data3.*
          524  +*/
          525  +
          526  +typedef struct Datatest3 Datatest3;
          527  +struct Datatest3 {
          528  +  int nRange;                     /* Keys are between 1 and this value, incl. */
          529  +  int nIter;                      /* Number of iterations */
          530  +  int nWrite;                     /* Number of writes per iteration */
          531  +  int nDelete;                    /* Number of deletes per iteration */
          532  +
          533  +  int nValMin;                    /* Minimum value size for writes */
          534  +  int nValMax;                    /* Maximum value size for writes */
          535  +};
          536  +
          537  +void testPutU32(u8 *aBuf, u32 iVal){
          538  +  aBuf[0] = (iVal >> 24) & 0xFF;
          539  +  aBuf[1] = (iVal >> 16) & 0xFF;
          540  +  aBuf[2] = (iVal >>  8) & 0xFF;
          541  +  aBuf[3] = (iVal >>  0) & 0xFF;
          542  +}
          543  +
          544  +void dt3PutKey(u8 *aBuf, int iKey){
          545  +  assert( iKey<100000 && iKey>=0 );
          546  +  sprintf((char *)aBuf, "%.5d", iKey);
          547  +}
          548  +
          549  +static void doDataTest3(
          550  +  const char *zSystem,            /* Database system to test */
          551  +  Datatest3 *p,                   /* Structure containing test parameters */
          552  +  int *pRc                        /* OUT: Error code */
          553  +){
          554  +  int iDot = 0;
          555  +  int rc = *pRc;
          556  +  TestDb *pDb;
          557  +  u8 *abPresent;                  /* Array of boolean */
          558  +  char *aVal;                     /* Buffer to hold values */
          559  +  int i;
          560  +  u32 iSeq = 10;                  /* prng counter */
          561  +
          562  +  abPresent = (u8 *)testMalloc(p->nRange+1);
          563  +  aVal = (char *)testMalloc(p->nValMax+1);
          564  +  pDb = testOpen(zSystem, 1, &rc);
          565  +
          566  +  for(i=0; i<p->nIter && rc==0; i++){
          567  +    int ii;
          568  +
          569  +    testCaseProgress(i, p->nIter, testCaseNDot(), &iDot);
          570  +
          571  +    /* Perform nWrite inserts */
          572  +    for(ii=0; ii<p->nWrite; ii++){
          573  +      u8 aKey[6];
          574  +      u32 iKey;
          575  +      int nVal;
          576  +
          577  +      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
          578  +      nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin;
          579  +      testPrngString(testPrngValue(iSeq++), aVal, nVal);
          580  +      dt3PutKey(aKey, iKey);
          581  +
          582  +      testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc);
          583  +      abPresent[iKey] = 1;
          584  +    }
          585  +
          586  +    /* Perform nDelete deletes */
          587  +    for(ii=0; ii<p->nDelete; ii++){
          588  +      u8 aKey1[6];
          589  +      u8 aKey2[6];
          590  +      u32 iKey;
          591  +
          592  +      iKey = (testPrngValue(iSeq++) % p->nRange) + 1;
          593  +      dt3PutKey(aKey1, iKey-1);
          594  +      dt3PutKey(aKey2, iKey+1);
          595  +
          596  +      testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc);
          597  +      abPresent[iKey] = 0;
          598  +    }
          599  +
          600  +    testReopen(&pDb, &rc);
          601  +
          602  +    for(ii=1; rc==0 && ii<=p->nRange; ii++){
          603  +      int nDbVal;
          604  +      void *pDbVal;
          605  +      u8 aKey[6];
          606  +      int dbrc;
          607  +
          608  +      dt3PutKey(aKey, ii);
          609  +      dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal);
          610  +      testCompareInt(0, dbrc, &rc);
          611  +
          612  +      if( abPresent[ii] ){
          613  +        testCompareInt(1, (nDbVal>0), &rc);
          614  +      }else{
          615  +        testCompareInt(1, (nDbVal<0), &rc);
          616  +      }
          617  +    }
          618  +  }
          619  +
          620  +  testClose(&pDb);
          621  +  testCaseFinish(rc);
          622  +  *pRc = rc;
          623  +}
          624  +
          625  +static char *getName3(const char *zSystem, Datatest3 *p){
          626  +  return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)",
          627  +      zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, 
          628  +      p->nValMin, p->nValMax
          629  +  );
          630  +}
          631  +
          632  +void test_data_3(
          633  +  const char *zSystem,            /* Database system name */
          634  +  const char *zPattern,           /* Run test cases that match this pattern */
          635  +  int *pRc                        /* IN/OUT: Error code */
          636  +){
          637  +  Datatest3 aTest[] = {
          638  +    /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */
          639  +    {  100,    1000,  5,      5,       50,      100 },
          640  +    {  100,    1000,  2,      2,        5,       10 },
          641  +  };
          642  +
          643  +  int i;
          644  +
          645  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          646  +    char *zName = getName3(zSystem, &aTest[i]);
          647  +    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
          648  +      doDataTest3(zSystem, &aTest[i], pRc);
          649  +    }
          650  +    testFree(zName);
          651  +  }
          652  +}
          653  +
          654  +

Added ext/lsm1/lsm-test/lsmtest2.c.

            1  +
            2  +/*
            3  +** This file contains tests related to recovery following application 
            4  +** and system crashes (power failures) while writing to the database.
            5  +*/
            6  +
            7  +#include "lsmtest.h"
            8  +
            9  +/*
           10  +** Structure used by testCksumDatabase() to accumulate checksum values in.
           11  +*/
           12  +typedef struct Cksum Cksum;
           13  +struct Cksum {
           14  +  int nRow;
           15  +  int cksum1;
           16  +  int cksum2;
           17  +};
           18  +
           19  +/*
           20  +** tdb_scan() callback used by testCksumDatabase()
           21  +*/
           22  +static void scanCksumDb(
           23  +  void *pCtx, 
           24  +  void *pKey, int nKey,
           25  +  void *pVal, int nVal
           26  +){
           27  +  Cksum *p = (Cksum *)pCtx;
           28  +  int i;
           29  +
           30  +  p->nRow++;
           31  +  for(i=0; i<nKey; i++){
           32  +    p->cksum1 += ((u8 *)pKey)[i];
           33  +    p->cksum2 += p->cksum1;
           34  +  }
           35  +  for(i=0; i<nVal; i++){
           36  +    p->cksum1 += ((u8 *)pVal)[i];
           37  +    p->cksum2 += p->cksum1;
           38  +  }
           39  +}
           40  +
           41  +/*
           42  +** tdb_scan() callback used by testCountDatabase()
           43  +*/
           44  +static void scanCountDb(
           45  +  void *pCtx, 
           46  +  void *pKey, int nKey,
           47  +  void *pVal, int nVal
           48  +){
           49  +  Cksum *p = (Cksum *)pCtx;
           50  +  p->nRow++;
           51  +
           52  +  unused_parameter(pKey);
           53  +  unused_parameter(nKey);
           54  +  unused_parameter(pVal);
           55  +  unused_parameter(nVal);
           56  +}
           57  +
           58  +
           59  +/*
           60  +** Iterate through the entire contents of database pDb. Write a checksum
           61  +** string based on the db contents into buffer zOut before returning. A
           62  +** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size:
           63  +**
           64  +**    * 32-bit integer (10 bytes)
           65  +**    * 1 space        (1 byte)
           66  +**    * 32-bit hex     (8 bytes)
           67  +**    * 1 space        (1 byte)
           68  +**    * 32-bit hex     (8 bytes)
           69  +**    * nul-terminator (1 byte)
           70  +**
           71  +** The number of entries in the database is returned.
           72  +*/
           73  +int testCksumDatabase(
           74  +  TestDb *pDb,                    /* Database handle */
           75  +  char *zOut                      /* Buffer to write checksum to */
           76  +){
           77  +  Cksum cksum;
           78  +  memset(&cksum, 0, sizeof(Cksum));
           79  +  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb);
           80  +  sprintf(zOut, "%d %x %x", 
           81  +      cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2
           82  +  );
           83  +  assert( strlen(zOut)<TEST_CKSUM_BYTES );
           84  +  return cksum.nRow;
           85  +}
           86  +
           87  +int testCountDatabase(TestDb *pDb){
           88  +  Cksum cksum;
           89  +  memset(&cksum, 0, sizeof(Cksum));
           90  +  tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCountDb);
           91  +  return cksum.nRow;
           92  +}
           93  +
           94  +/*
           95  +** This function is a no-op if *pRc is not 0 when it is called.
           96  +**
           97  +** Otherwise, the two nul-terminated strings z1 and z1 are compared. If
           98  +** they are the same, the function returns without doing anything. Otherwise,
           99  +** an error message is printed, *pRc is set to 1 and the test_failed()
          100  +** function called.
          101  +*/
          102  +void testCompareStr(const char *z1, const char *z2, int *pRc){
          103  +  if( *pRc==0 ){
          104  +    if( strcmp(z1, z2) ){
          105  +      testPrintError("testCompareStr: \"%s\" != \"%s\"\n", z1, z2);
          106  +      *pRc = 1;
          107  +      test_failed();
          108  +    }
          109  +  }
          110  +}
          111  +
          112  +/*
          113  +** This function is a no-op if *pRc is not 0 when it is called.
          114  +**
          115  +** Otherwise, the two integers i1 and i2 are compared. If they are equal,
          116  +** the function returns without doing anything. Otherwise, an error message 
          117  +** is printed, *pRc is set to 1 and the test_failed() function called.
          118  +*/
          119  +void testCompareInt(int i1, int i2, int *pRc){
          120  +  if( *pRc==0 && i1!=i2 ){
          121  +    testPrintError("testCompareInt: %d != %d\n", i1, i2);
          122  +    *pRc = 1;
          123  +    test_failed();
          124  +  }
          125  +}
          126  +
          127  +void testCaseStart(int *pRc, char *zFmt, ...){
          128  +  va_list ap;
          129  +  va_start(ap, zFmt);
          130  +  vprintf(zFmt, ap);
          131  +  printf(" ...");
          132  +  va_end(ap);
          133  +  *pRc = 0;
          134  +  fflush(stdout);
          135  +}
          136  +
          137  +/*
          138  +** This function is a no-op if *pRc is non-zero when it is called. Zero
          139  +** is returned in this case.
          140  +**
          141  +** Otherwise, the zFmt (a printf style format string) and following arguments 
          142  +** are used to create a test case name. If zPattern is NULL or a glob pattern
          143  +** that matches the test case name, 1 is returned and the test case started.
          144  +** Otherwise, zero is returned and the test case does not start.
          145  +*/
          146  +int testCaseBegin(int *pRc, const char *zPattern, const char *zFmt, ...){
          147  +  int res = 0;
          148  +  if( *pRc==0 ){
          149  +    char *zTest;
          150  +    va_list ap;
          151  +
          152  +    va_start(ap, zFmt);
          153  +    zTest = testMallocVPrintf(zFmt, ap);
          154  +    va_end(ap);
          155  +    if( zPattern==0 || testGlobMatch(zPattern, zTest) ){
          156  +      printf("%-50s ...", zTest);
          157  +      res = 1;
          158  +    }
          159  +    testFree(zTest);
          160  +    fflush(stdout);
          161  +  }
          162  +
          163  +  return res;
          164  +}
          165  +
          166  +void testCaseFinish(int rc){
          167  +  if( rc==0 ){
          168  +    printf("Ok\n");
          169  +  }else{
          170  +    printf("FAILED\n");
          171  +  }
          172  +  fflush(stdout);
          173  +}
          174  +
          175  +void testCaseSkip(){
          176  +  printf("Skipped\n");
          177  +}
          178  +
          179  +void testSetupSavedLsmdb(
          180  +  const char *zCfg,
          181  +  const char *zFile,
          182  +  Datasource *pData,
          183  +  int nRow,
          184  +  int *pRc
          185  +){
          186  +  if( *pRc==0 ){
          187  +    int rc;
          188  +    TestDb *pDb;
          189  +    rc = tdb_lsm_open(zCfg, zFile, 1, &pDb);
          190  +    if( rc==0 ){
          191  +      testWriteDatasourceRange(pDb, pData, 0, nRow, &rc);
          192  +      testClose(&pDb);
          193  +      if( rc==0 ) testSaveDb(zFile, "log");
          194  +    }
          195  +    *pRc = rc;
          196  +  }
          197  +}
          198  +
          199  +/*
          200  +** This function is a no-op if *pRc is non-zero when it is called.
          201  +**
          202  +** Open the LSM database identified by zFile and compute its checksum
          203  +** (a string, as returned by testCksumDatabase()). If the checksum is
          204  +** identical to zExpect1 or, if it is not NULL, zExpect2, the test passes.
          205  +** Otherwise, print an error message and set *pRc to 1.
          206  +*/
          207  +static void testCompareCksumLsmdb(
          208  +  const char *zFile,              /* Path to LSM database */
          209  +  int bCompress,                  /* True if db is compressed */
          210  +  const char *zExpect1,           /* Expected checksum 1 */
          211  +  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
          212  +  int *pRc                        /* IN/OUT: Test case error code */
          213  +){
          214  +  if( *pRc==0 ){
          215  +    char zCksum[TEST_CKSUM_BYTES];
          216  +    TestDb *pDb;
          217  +
          218  +    *pRc = tdb_lsm_open((bCompress?"compression=1 mmap=0":""), zFile, 0, &pDb);
          219  +    testCksumDatabase(pDb, zCksum);
          220  +    testClose(&pDb);
          221  +
          222  +    if( *pRc==0 ){
          223  +      int r1 = 0;
          224  +      int r2 = -1;
          225  +
          226  +      r1 = strcmp(zCksum, zExpect1);
          227  +      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
          228  +      if( r1 && r2 ){
          229  +        if( zExpect2 ){
          230  +          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
          231  +              zCksum, zExpect1, zExpect2
          232  +          );
          233  +        }else{
          234  +          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
          235  +              zCksum, zExpect1
          236  +          );
          237  +        }
          238  +        *pRc = 1;
          239  +        test_failed();
          240  +      }
          241  +    }
          242  +  }
          243  +}
          244  +
          245  +#if 0 /* not used */
          246  +static void testCompareCksumBtdb(
          247  +  const char *zFile,              /* Path to LSM database */
          248  +  const char *zExpect1,           /* Expected checksum 1 */
          249  +  const char *zExpect2,           /* Expected checksum 2 (or NULL) */
          250  +  int *pRc                        /* IN/OUT: Test case error code */
          251  +){
          252  +  if( *pRc==0 ){
          253  +    char zCksum[TEST_CKSUM_BYTES];
          254  +    TestDb *pDb;
          255  +
          256  +    *pRc = tdb_open("bt", zFile, 0, &pDb);
          257  +    testCksumDatabase(pDb, zCksum);
          258  +    testClose(&pDb);
          259  +
          260  +    if( *pRc==0 ){
          261  +      int r1 = 0;
          262  +      int r2 = -1;
          263  +
          264  +      r1 = strcmp(zCksum, zExpect1);
          265  +      if( zExpect2 ) r2 = strcmp(zCksum, zExpect2);
          266  +      if( r1 && r2 ){
          267  +        if( zExpect2 ){
          268  +          testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")",
          269  +              zCksum, zExpect1, zExpect2
          270  +          );
          271  +        }else{
          272  +          testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"",
          273  +              zCksum, zExpect1
          274  +          );
          275  +        }
          276  +        *pRc = 1;
          277  +        test_failed();
          278  +      }
          279  +    }
          280  +  }
          281  +}
          282  +#endif /* not used */
          283  +
          284  +/* Above this point are reusable test routines. Not clear that they
          285  +** should really be in this file.
          286  +*************************************************************************/
          287  +
          288  +/*
          289  +** This test verifies that if a system crash occurs while doing merge work
          290  +** on the db, no data is lost.
          291  +*/
          292  +static void crash_test1(int bCompress, int *pRc){
          293  +  const char *DBNAME = "testdb.lsm";
          294  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 200, 200};
          295  +
          296  +  const int nRow = 5000;          /* Database size */
          297  +  const int nIter = 200;          /* Number of test iterations */
          298  +  const int nWork = 20;           /* Maximum lsm_work() calls per iteration */
          299  +  const int nPage = 15;           /* Pages per lsm_work call */
          300  +
          301  +  int i;
          302  +  int iDot = 0;
          303  +  Datasource *pData;
          304  +  CksumDb *pCksumDb;
          305  +  TestDb *pDb;
          306  +  char *zCfg;
          307  +
          308  +  const char *azConfig[2] = {
          309  +    "page_size=1024 block_size=65536 autoflush=16384 safety=2 mmap=0", 
          310  +    "page_size=1024 block_size=65536 autoflush=16384 safety=2 "
          311  +    " compression=1 mmap=0"
          312  +  };
          313  +  assert( bCompress==0 || bCompress==1 );
          314  +
          315  +  /* Allocate datasource. And calculate the expected checksums. */
          316  +  pData = testDatasourceNew(&defn);
          317  +  pCksumDb = testCksumArrayNew(pData, nRow, nRow, 1);
          318  +
          319  +  /* Setup and save the initial database. */
          320  +
          321  +  zCfg = testMallocPrintf("%s automerge=7", azConfig[bCompress]);
          322  +  testSetupSavedLsmdb(zCfg, DBNAME, pData, 5000, pRc);
          323  +  testFree(zCfg);
          324  +
          325  +  for(i=0; i<nIter && *pRc==0; i++){
          326  +    int iWork;
          327  +    int testrc = 0;
          328  +
          329  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          330  +
          331  +    /* Restore and open the database. */
          332  +    testRestoreDb(DBNAME, "log");
          333  +    testrc = tdb_lsm_open(azConfig[bCompress], DBNAME, 0, &pDb);
          334  +    assert( testrc==0 );
          335  +
          336  +    /* Call lsm_work() on the db */
          337  +    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nWork*2)));
          338  +    for(iWork=0; testrc==0 && iWork<nWork; iWork++){
          339  +      int nWrite = 0;
          340  +      lsm_db *db = tdb_lsm(pDb);
          341  +      testrc = lsm_work(db, 0, nPage, &nWrite);
          342  +      /* assert( testrc!=0 || nWrite>0 ); */
          343  +      if( testrc==0 ) testrc = lsm_checkpoint(db, 0);
          344  +    }
          345  +    tdb_close(pDb);
          346  +
          347  +    /* Check that the database content is still correct */
          348  +    testCompareCksumLsmdb(DBNAME, 
          349  +        bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc);
          350  +  }
          351  +
          352  +  testCksumArrayFree(pCksumDb);
          353  +  testDatasourceFree(pData);
          354  +}
          355  +
          356  +/*
          357  +** This test verifies that if a system crash occurs while committing a
          358  +** transaction to the log file, no earlier transactions are lost or damaged.
          359  +*/
          360  +static void crash_test2(int bCompress, int *pRc){
          361  +  const char *DBNAME = "testdb.lsm";
          362  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
          363  +
          364  +  const int nIter = 200;
          365  +  const int nInsert = 20;
          366  +
          367  +  int i;
          368  +  int iDot = 0;
          369  +  Datasource *pData;
          370  +  CksumDb *pCksumDb;
          371  +  TestDb *pDb;
          372  +
          373  +  /* Allocate datasource. And calculate the expected checksums. */
          374  +  pData = testDatasourceNew(&defn);
          375  +  pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1);
          376  +
          377  +  /* Setup and save the initial database. */
          378  +  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
          379  +
          380  +  for(i=0; i<nIter && *pRc==0; i++){
          381  +    int iIns;
          382  +    int testrc = 0;
          383  +
          384  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          385  +
          386  +    /* Restore and open the database. */
          387  +    testRestoreDb(DBNAME, "log");
          388  +    testrc = tdb_lsm_open("safety=2", DBNAME, 0, &pDb);
          389  +    assert( testrc==0 );
          390  +
          391  +    /* Insert nInsert records into the database. Crash midway through. */
          392  +    tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nInsert+2)));
          393  +    for(iIns=0; iIns<nInsert; iIns++){
          394  +      void *pKey; int nKey;
          395  +      void *pVal; int nVal;
          396  +
          397  +      testDatasourceEntry(pData, 100+iIns, &pKey, &nKey, &pVal, &nVal);
          398  +      testrc = tdb_write(pDb, pKey, nKey, pVal, nVal);
          399  +      if( testrc ) break;
          400  +    }
          401  +    tdb_close(pDb);
          402  +
          403  +    /* Check that no data was lost when the system crashed. */
          404  +    testCompareCksumLsmdb(DBNAME, bCompress,
          405  +      testCksumArrayGet(pCksumDb, 100 + iIns),
          406  +      testCksumArrayGet(pCksumDb, 100 + iIns + 1),
          407  +      pRc
          408  +    );
          409  +  }
          410  +
          411  +  testDatasourceFree(pData);
          412  +  testCksumArrayFree(pCksumDb);
          413  +}
          414  +
          415  +
          416  +/*
          417  +** This test verifies that if a system crash occurs when checkpointing
          418  +** the database, data is not lost (assuming that any writes not synced
          419  +** to the db have been synced into the log file).
          420  +*/
          421  +static void crash_test3(int bCompress, int *pRc){
          422  +  const char *DBNAME = "testdb.lsm";
          423  +  const int nIter = 100;
          424  +  const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000};
          425  +
          426  +  int i;
          427  +  int iDot = 0;
          428  +  Datasource *pData;
          429  +  CksumDb *pCksumDb;
          430  +  TestDb *pDb;
          431  +
          432  +  /* Allocate datasource. And calculate the expected checksums. */
          433  +  pData = testDatasourceNew(&defn);
          434  +  pCksumDb = testCksumArrayNew(pData, 110, 150, 10);
          435  +
          436  +  /* Setup and save the initial database. */
          437  +  testSetupSavedLsmdb("", DBNAME, pData, 100, pRc);
          438  +
          439  +  for(i=0; i<nIter && *pRc==0; i++){
          440  +    int iOpen;
          441  +    testCaseProgress(i, nIter, testCaseNDot(), &iDot);
          442  +    testRestoreDb(DBNAME, "log");
          443  +
          444  +    for(iOpen=0; iOpen<5; iOpen++){
          445  +      /* Open the database. Insert 10 more records. */
          446  +      pDb = testOpen("lsm", 0, pRc);
          447  +      testWriteDatasourceRange(pDb, pData, 100+iOpen*10, 10, pRc);
          448  +
          449  +      /* Schedule a crash simulation then close the db. */
          450  +      tdb_lsm_prepare_sync_crash(pDb, 1 + (i%2));
          451  +      tdb_close(pDb);
          452  +
          453  +      /* Open the database and check that the crash did not cause any
          454  +      ** data loss.  */
          455  +      testCompareCksumLsmdb(DBNAME, bCompress,
          456  +        testCksumArrayGet(pCksumDb, 110 + iOpen*10), 0,
          457  +        pRc
          458  +      );
          459  +    }
          460  +  }
          461  +
          462  +  testDatasourceFree(pData);
          463  +  testCksumArrayFree(pCksumDb);
          464  +}
          465  +
          466  +void do_crash_test(const char *zPattern, int *pRc){
          467  +  struct Test {
          468  +    const char *zTest;
          469  +    void (*x)(int, int *);
          470  +    int bCompress;
          471  +  } aTest [] = {
          472  +    { "crash.lsm.1",     crash_test1, 0 },
          473  +#ifdef HAVE_ZLIB
          474  +    { "crash.lsm_zip.1", crash_test1, 1 },
          475  +#endif
          476  +    { "crash.lsm.2",     crash_test2, 0 },
          477  +    { "crash.lsm.3",     crash_test3, 0 },
          478  +  };
          479  +  int i;
          480  +
          481  +  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
          482  +    struct Test *p = &aTest[i];
          483  +    if( testCaseBegin(pRc, zPattern, "%s", p->zTest) ){
          484  +      p->x(p->bCompress, pRc);
          485  +      testCaseFinish(*pRc);
          486  +    }
          487  +  }
          488  +}

Added ext/lsm1/lsm-test/lsmtest3.c.

            1  +
            2  +
            3  +/*
            4  +** This file contains tests related to the explicit rollback of database
            5  +** transactions and sub-transactions.
            6  +*/
            7  +
            8  +
            9  +/*
           10  +** Repeat 2000 times (until the db contains 100,000 entries):
           11  +**
           12  +**   1. Open a transaction and insert 500 rows, opening a nested 
           13  +**      sub-transaction each 100 rows.
           14  +**
           15  +**   2. Roll back to each sub-transaction savepoint. Check the database
           16  +**      checksum looks Ok.
           17  +**
           18  +**   3. Every second iteration, roll back the main transaction. Check the
           19  +**      db checksum is correct. Every other iteration, commit the main
           20  +**      transaction (increasing the size of the db by 100 rows).
           21  +*/
           22  +
           23  +
           24  +#include "lsmtest.h"
           25  +
           26  +struct CksumDb {
           27  +  int nFirst;
           28  +  int nLast;
           29  +  int nStep;
           30  +  char **azCksum;
           31  +};
           32  +
           33  +CksumDb *testCksumArrayNew(
           34  +  Datasource *pData, 
           35  +  int nFirst, 
           36  +  int nLast, 
           37  +  int nStep
           38  +){
           39  +  TestDb *pDb;
           40  +  CksumDb *pRet;
           41  +  int i;
           42  +  int nEntry;
           43  +  int rc = 0;
           44  +
           45  +  assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 );
           46  + 
           47  +  pRet = malloc(sizeof(CksumDb));
           48  +  memset(pRet, 0, sizeof(CksumDb));
           49  +  pRet->nFirst = nFirst;
           50  +  pRet->nLast = nLast;
           51  +  pRet->nStep = nStep;
           52  +  nEntry = 1 + ((nLast - nFirst) / nStep);
           53  +
           54  +  /* Allocate space so that azCksum is an array of nEntry pointers to
           55  +  ** buffers each TEST_CKSUM_BYTES in size.  */
           56  +  pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES));
           57  +  for(i=0; i<nEntry; i++){
           58  +    char *pStart = (char *)(&pRet->azCksum[nEntry]);
           59  +    pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES];
           60  +  }
           61  +
           62  +  tdb_open("lsm", "tempdb.lsm", 1, &pDb);
           63  +  testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc);
           64  +  for(i=0; i<nEntry; i++){
           65  +    testCksumDatabase(pDb, pRet->azCksum[i]);
           66  +    if( i==nEntry ) break;
           67  +    testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc);
           68  +  }
           69  +
           70  +  tdb_close(pDb);
           71  +
           72  +  return pRet;
           73  +}
           74  +
           75  +char *testCksumArrayGet(CksumDb *p, int nRow){
           76  +  int i;
           77  +  assert( nRow>=p->nFirst );
           78  +  assert( nRow<=p->nLast );
           79  +  assert( ((nRow-p->nFirst) % p->nStep)==0 );
           80  +
           81  +  i = (nRow - p->nFirst) / p->nStep;
           82  +  return p->azCksum[i];
           83  +}
           84  +
           85  +void testCksumArrayFree(CksumDb *p){
           86  +  free(p->azCksum);
           87  +  memset(p, 0x55, sizeof(*p));
           88  +  free(p);
           89  +}
           90  +
           91  +/* End of CksumDb code.
           92  +**************************************************************************/
           93  +
           94  +/*
           95  +** Test utility function. Write key-value pair $i from datasource pData 
           96  +** into database pDb.
           97  +*/
           98  +void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
           99  +  void *pKey; int nKey;
          100  +  void *pVal; int nVal;
          101  +  testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
          102  +  testWrite(pDb, pKey, nKey, pVal, nVal, pRc);
          103  +}
          104  +
          105  +/*
          106  +** Test utility function. Delete datasource pData key $i from database pDb.
          107  +*/
          108  +void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){
          109  +  void *pKey; int nKey;
          110  +  testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0);
          111  +  testDelete(pDb, pKey, nKey, pRc);
          112  +}
          113  +
          114  +/*
          115  +** This function inserts nWrite key/value pairs into database pDb - the
          116  +** nWrite key value pairs starting at iFirst from data source pData.
          117  +*/
          118  +void testWriteDatasourceRange(
          119  +  TestDb *pDb,                    /* Database to write to */
          120  +  Datasource *pData,              /* Data source to read values from */
          121  +  int iFirst,                     /* Index of first key/value pair */
          122  +  int nWrite,                     /* Number of key/value pairs to write */
          123  +  int *pRc                        /* IN/OUT: Error code */
          124  +){
          125  +  int i;
          126  +  for(i=0; i<nWrite; i++){
          127  +    testWriteDatasource(pDb, pData, iFirst+i, pRc);
          128  +  }
          129  +}
          130  +
          131  +void testDeleteDatasourceRange(
          132  +  TestDb *pDb,                    /* Database to write to */
          133  +  Datasource *pData,              /* Data source to read keys from */
          134  +  int iFirst,                     /* Index of first key */
          135  +  int nWrite,                     /* Number of keys to delete */
          136  +  int *pRc                        /* IN/OUT: Error code */
          137  +){
          138  +  int i;
          139  +  for(i=0; i<nWrite; i++){
          140  +    testDeleteDatasource(pDb, pData, iFirst+i, pRc);
          141  +  }
          142  +}
          143  +
          144  +static char *getName(const char *zSystem){ 
          145  +  char *zRet; 
          146  +  zRet = testMallocPrintf("rollback.%s", zSystem);
          147  +  return zRet;
          148  +}
          149  +
          150  +static int rollback_test_1(
          151  +  const char *zSystem,
          152  +  Datasource *pData
          153  +){
          154  +  const int nRepeat = 100;
          155  +
          156  +  TestDb *pDb;
          157  +  int rc;
          158  +  int i;
          159  +  CksumDb *pCksum;
          160  +  char *zName;
          161  +
          162  +  zName = getName(zSystem);
          163  +  testCaseStart(&rc, zName);
          164  +  testFree(zName);
          165  +
          166  +  pCksum = testCksumArrayNew(pData, 0, nRepeat*100, 100);
          167  +  pDb = 0;
          168  +  rc = tdb_open(zSystem, 0, 1, &pDb);
          169  +  if( pDb && tdb_transaction_support(pDb)==0 ){
          170  +    testCaseSkip();
          171  +    goto skip_rollback_test;
          172  +  }
          173  +
          174  +  for(i=0; i<nRepeat && rc==0; i++){
          175  +    char zCksum[TEST_CKSUM_BYTES];
          176  +    int nCurrent = (((i+1)/2) * 100);
          177  +    int nDbRow;
          178  +    int iTrans;
          179  +
          180  +    /* Check that the database is the expected size. */
          181  +    nDbRow = testCountDatabase(pDb);
          182  +    testCompareInt(nCurrent, nDbRow, &rc);
          183  +
          184  +    for(iTrans=2; iTrans<=6 && rc==0; iTrans++){
          185  +      tdb_begin(pDb, iTrans);
          186  +      testWriteDatasourceRange(pDb, pData, nCurrent, 100, &rc);
          187  +      nCurrent += 100;
          188  +    }
          189  +
          190  +    testCksumDatabase(pDb, zCksum);
          191  +    testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          192  +
          193  +    for(iTrans=6; iTrans>2 && rc==0; iTrans--){
          194  +      tdb_rollback(pDb, iTrans);
          195  +      nCurrent -= 100;
          196  +      testCksumDatabase(pDb, zCksum);
          197  +      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          198  +    }
          199  +
          200  +    if( i%2 ){
          201  +      tdb_rollback(pDb, 0);
          202  +      nCurrent -= 100;
          203  +      testCksumDatabase(pDb, zCksum);
          204  +      testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc);
          205  +    }else{
          206  +      tdb_commit(pDb, 0);
          207  +    }
          208  +  }
          209  +  testCaseFinish(rc);
          210  +
          211  + skip_rollback_test:
          212  +  tdb_close(pDb);
          213  +  testCksumArrayFree(pCksum);
          214  +  return rc;
          215  +}
          216  +
          217  +void test_rollback(
          218  +  const char *zSystem, 
          219  +  const char *zPattern, 
          220  +  int *pRc
          221  +){
          222  +  if( *pRc==0 ){
          223  +    int bRun = 1;
          224  +
          225  +    if( zPattern ){
          226  +      char *zName = getName(zSystem);
          227  +      bRun = testGlobMatch(zPattern, zName);
          228  +      testFree(zName);
          229  +    }
          230  +
          231  +    if( bRun ){
          232  +      DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 };
          233  +      Datasource *pData = testDatasourceNew(&defn);
          234  +      *pRc = rollback_test_1(zSystem, pData);
          235  +      testDatasourceFree(pData);
          236  +    }
          237  +  }
          238  +}

Added ext/lsm1/lsm-test/lsmtest4.c.

            1  +
            2  +/*
            3  +** This file contains test cases involving multiple database clients.
            4  +*/
            5  +
            6  +#include "lsmtest.h"
            7  +
            8  +/*
            9  +** The following code implements test cases "mc1.*".
           10  +**
           11  +** This test case uses one writer and $nReader readers. All connections
           12  +** are driven by a single thread. All connections are opened at the start
           13  +** of the test and remain open until the test is finished.
           14  +**
           15  +** The test consists of $nStep steps. Each step the following is performed:
           16  +**
           17  +**   1. The writer inserts $nWriteStep records into the db.
           18  +**
           19  +**   2. The writer checks that the contents of the db are as expected.
           20  +**
           21  +**   3. Each reader that currently has an open read transaction also checks
           22  +**      that the contents of the db are as expected (according to the snapshot
           23  +**      the read transaction is reading - see below).
           24  +**
           25  +** After step 1, reader 1 opens a read transaction. After step 2, reader
           26  +** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1
           27  +** closes the current read transaction and opens a new one. And so on.
           28  +** The result is that at step N (for N > $nReader), there exists a reader
           29  +** with an open read transaction reading the snapshot committed following
           30  +** steps (N-$nReader-1) to N. 
           31  +*/
           32  +typedef struct Mctest Mctest;
           33  +struct Mctest {
           34  +  DatasourceDefn defn;            /* Datasource to use */
           35  +  int nStep;                      /* Total number of steps in test */
           36  +  int nWriteStep;                 /* Number of rows to insert each step */
           37  +  int nReader;                    /* Number of read connections */
           38  +};
           39  +static void do_mc_test(
           40  +  const char *zSystem,            /* Database system to test */
           41  +  Mctest *pTest,
           42  +  int *pRc                        /* IN/OUT: return code */
           43  +){
           44  +  const int nDomain = pTest->nStep * pTest->nWriteStep;
           45  +  Datasource *pData;              /* Source of data */
           46  +  TestDb *pDb;                    /* First database connection (writer) */
           47  +  int iReader;                    /* Used to iterate through aReader */
           48  +  int iStep;                      /* Current step in test */
           49  +  int iDot = 0;                   /* Current step in test */
           50  +
           51  +  /* Array of reader connections */
           52  +  struct Reader {
           53  +    TestDb *pDb;                  /* Connection handle */
           54  +    int iLast;                    /* Current snapshot contains keys 0..iLast */
           55  +  } *aReader;
           56  +
           57  +  /* Create a data source */
           58  +  pData = testDatasourceNew(&pTest->defn);
           59  +
           60  +  /* Open the writer connection */
           61  +  pDb = testOpen(zSystem, 1, pRc);
           62  +
           63  +  /* Allocate aReader */
           64  +  aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader);
           65  +  for(iReader=0; iReader<pTest->nReader; iReader++){
           66  +    aReader[iReader].pDb = testOpen(zSystem, 0, pRc);
           67  +  }
           68  +
           69  +  for(iStep=0; iStep<pTest->nStep; iStep++){
           70  +    int iLast;
           71  +    int iBegin;                   /* Start read trans using aReader[iBegin] */
           72  +
           73  +    /* Insert nWriteStep more records into the database */
           74  +    int iFirst = iStep*pTest->nWriteStep;
           75  +    testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc);
           76  +
           77  +    /* Check that the db is Ok according to the writer */
           78  +    iLast = (iStep+1) * pTest->nWriteStep - 1;
           79  +    testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc);
           80  +
           81  +    /* Have reader (iStep % nReader) open a read transaction here. */
           82  +    iBegin = (iStep % pTest->nReader);
           83  +    if( iBegin<iStep ) tdb_commit(aReader[iBegin].pDb, 0);
           84  +    tdb_begin(aReader[iBegin].pDb, 1);
           85  +    aReader[iBegin].iLast = iLast;
           86  +
           87  +    /* Check that the db is Ok for each open reader */
           88  +    for(iReader=0; iReader<pTest->nReader && aReader[iReader].iLast; iReader++){
           89  +      iLast = aReader[iReader].iLast;
           90  +      testDbContents(
           91  +          aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc
           92  +      );
           93  +    }
           94  +
           95  +    /* Report progress */
           96  +    testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot);
           97  +  }
           98  +
           99  +  /* Close all readers */
          100  +  for(iReader=0; iReader<pTest->nReader; iReader++){
          101  +    testClose(&aReader[iReader].pDb);
          102  +  }
          103  +  testFree(aReader);
          104  +
          105  +  /* Close the writer-connection and free the datasource */
          106  +  testClose(&pDb);
          107  +  testDatasourceFree(pData);
          108  +}
          109  +
          110  +
          111  +void test_mc(
          112  +  const char *zSystem,            /* Database system name */
          113  +  const char *zPattern,           /* Run test cases that match this pattern */
          114  +  int *pRc                        /* IN/OUT: Error code */
          115  +){
          116  +  int i;
          117  +  Mctest aTest[] = {
          118  +    { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 },
          119  +  };
          120  +
          121  +  for(i=0; i<ArraySize(aTest); i++){
          122  +    if( testCaseBegin(pRc, zPattern, "mc1.%s.%d", zSystem, i) ){
          123  +      do_mc_test(zSystem, &aTest[i], pRc);
          124  +      testCaseFinish(*pRc);
          125  +    }
          126  +  }
          127  +}

Added ext/lsm1/lsm-test/lsmtest5.c.

            1  +
            2  +/*
            3  +** This file is broken into three semi-autonomous parts:
            4  +**
            5  +**   1. The database functions.
            6  +**   2. The thread wrappers.
            7  +**   3. The implementation of the mt1.* tests.
            8  +*/
            9  +
           10  +/*************************************************************************
           11  +** DATABASE CONTENTS:
           12  +**
           13  +**   The database contains up to N key/value pairs, where N is some large 
           14  +**   number (say 10,000,000). Keys are integer values between 0 and (N-1).
           15  +**   The value associated with each key is a pseudo-random blob of data.
           16  +**
           17  +**   Key/value pair keys are encoded as the two bytes "k." followed by a 
           18  +**   10-digit decimal number. i.e. key 45 -> "k.0000000045".
           19  +**
           20  +**   As well as the key/value pairs, the database also contains checksum 
           21  +**   entries. The checksums form a hierarchy - for every F key/value
           22  +**   entries there is one level 1 checksum. And for each F level 1 checksums
           23  +**   there is one level 2 checksum. And so on.
           24  +**
           25  +**   Checksum keys are encoded as the two byte "c." followed by the 
           26  +**   checksum level, followed by a 10 digit decimal number containing
           27  +**   the value of the first key that contributes to the checksum value.
           28  +**   For example, assuming F==10, the level 1 checksum that spans keys
           29  +**   10 to 19 is "c.1.0000000010".
           30  +**
           31  +**   Clients may perform one of two operations on the database: a read
           32  +**   or a write.
           33  +** 
           34  +** READ OPERATIONS:
           35  +**
           36  +**   A read operation scans a range of F key/value pairs. It computes
           37  +**   the expected checksum and then compares the computed value to the
           38  +**   actual value stored in the level 1 checksum entry. It then scans 
           39  +**   the group of F level 1 checksums, and compares the computed checksum 
           40  +**   to the associated level 2 checksum value, and so on until the 
           41  +**   highest level checksum value has been verified.
           42  +**
           43  +**   If a checksum ever fails to match the expected value, the test 
           44  +**   has failed.
           45  +**
           46  +** WRITE OPERATIONS:
           47  +**
           48  +**   A write operation involves writing (possibly clobbering) a single
           49  +**   key/value pair. The associated level 1 checksum is then recalculated
           50  +**   updated. Then the level 2 checksum, and so on until the highest
           51  +**   level checksum has been modified.
           52  +**
           53  +**   All updates occur inside a single transaction.
           54  +**
           55  +** INTERFACE:
           56  +**
           57  +**   The interface used by test cases to read and write the db consists
           58  +**   of type DbParameters and the following functions:
           59  +**
           60  +**       dbReadOperation()
           61  +**       dbWriteOperation()
           62  +*/
           63  +
           64  +#include "lsmtest.h"
           65  +
           66  +typedef struct DbParameters DbParameters;
           67  +struct DbParameters {
           68  +  int nFanout;                    /* Checksum fanout (F) */
           69  +  int nKey;                       /* Size of key space (N) */
           70  +};
           71  +
           72  +#define DB_KEY_BYTES          (2+5+10+1)
           73  +
           74  +/*
           75  +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
           76  +** This function populates the buffer with a nul-terminated key string 
           77  +** corresponding to key iKey.
           78  +*/
           79  +static void dbFormatKey(
           80  +  DbParameters *pParam,
           81  +  int iLevel,
           82  +  int iKey,                       /* Key value */
           83  +  char *aBuf                      /* Write key string here */
           84  +){
           85  +  if( iLevel==0 ){
           86  +    snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey);
           87  +  }else{
           88  +    int f = 1;
           89  +    int i;
           90  +    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
           91  +    snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f));
           92  +  }
           93  +}
           94  +
           95  +/*
           96  +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size.
           97  +** This function populates the buffer with the string representation of
           98  +** checksum value iVal.
           99  +*/
          100  +static void dbFormatCksumValue(u32 iVal, char *aBuf){
          101  +  snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal);
          102  +}
          103  +
          104  +/*
          105  +** Return the highest level of checksum in the database described
          106  +** by *pParam.
          107  +*/
          108  +static int dbMaxLevel(DbParameters *pParam){
          109  +  int iMax;
          110  +  int n = 1;
          111  +  for(iMax=0; n<pParam->nKey; iMax++){
          112  +    n = n * pParam->nFanout;
          113  +  }
          114  +  return iMax;
          115  +}
          116  +
          117  +static void dbCksum(
          118  +  void *pCtx,                     /* IN/OUT: Pointer to u32 containing cksum */
          119  +  void *pKey, int nKey,           /* Database key. Unused. */
          120  +  void *pVal, int nVal            /* Database value. Checksum this. */
          121  +){
          122  +  u8 *aVal = (u8 *)pVal;
          123  +  u32 *pCksum = (u32 *)pCtx;
          124  +  u32 cksum = *pCksum;
          125  +  int i;
          126  +
          127  +  unused_parameter(pKey);
          128  +  unused_parameter(nKey);
          129  +
          130  +  for(i=0; i<nVal; i++){
          131  +    cksum += (cksum<<3) + (int)aVal[i];
          132  +  }
          133  +
          134  +  *pCksum = cksum;
          135  +}
          136  +
          137  +/*
          138  +** Compute the value of the checksum stored on level iLevel that contains
          139  +** data from key iKey by scanning the pParam->nFanout entries at level 
          140  +** iLevel-1.
          141  +*/
          142  +static u32 dbComputeCksum(
          143  +  DbParameters *pParam,           /* Database parameters */
          144  +  TestDb *pDb,                    /* Database connection handle */
          145  +  int iLevel,                     /* Level of checksum to compute */
          146  +  int iKey,                       /* Compute checksum for this key */
          147  +  int *pRc                        /* IN/OUT: Error code */
          148  +){
          149  +  u32 cksum = 0;
          150  +  if( *pRc==0 ){
          151  +    int nFirst;
          152  +    int nLast;
          153  +    int iFirst = 0;
          154  +    int iLast = 0;
          155  +    int i;
          156  +    int f = 1;
          157  +    char zFirst[DB_KEY_BYTES];
          158  +    char zLast[DB_KEY_BYTES];
          159  +
          160  +    assert( iLevel>=1 );
          161  +    for(i=0; i<iLevel; i++) f = f * pParam->nFanout;
          162  +
          163  +    iFirst = f*(iKey/f);
          164  +    iLast = iFirst + f - 1;
          165  +    dbFormatKey(pParam, iLevel-1, iFirst, zFirst);
          166  +    dbFormatKey(pParam, iLevel-1, iLast, zLast);
          167  +    nFirst = strlen(zFirst);
          168  +    nLast = strlen(zLast);
          169  +
          170  +    *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum);
          171  +  }
          172  +
          173  +  return cksum;
          174  +}
          175  +
          176  +static void dbReadOperation(
          177  +  DbParameters *pParam,           /* Database parameters */
          178  +  TestDb *pDb,                    /* Database connection handle */
          179  +  void (*xDelay)(void *),
          180  +  void *pDelayCtx,
          181  +  int iKey,                       /* Key to read */
          182  +  int *pRc                        /* IN/OUT: Error code */
          183  +){
          184  +  const int iMax = dbMaxLevel(pParam);
          185  +  int i;
          186  +
          187  +  if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc);
          188  +  for(i=1; *pRc==0 && i<=iMax; i++){
          189  +    char zCksum[DB_KEY_BYTES];
          190  +    char zKey[DB_KEY_BYTES];
          191  +    u32 iCksum = 0;
          192  +
          193  +    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
          194  +    if( iCksum ){
          195  +      if( xDelay && i==1 ) xDelay(pDelayCtx);
          196  +      dbFormatCksumValue(iCksum, zCksum);
          197  +      dbFormatKey(pParam, i, iKey, zKey);
          198  +      testFetchStr(pDb, zKey, zCksum, pRc);
          199  +    }
          200  +  }
          201  +  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
          202  +}
          203  +
          204  +static int dbWriteOperation(
          205  +  DbParameters *pParam,           /* Database parameters */
          206  +  TestDb *pDb,                    /* Database connection handle */
          207  +  int iKey,                       /* Key to write to */
          208  +  const char *zValue,             /* Nul-terminated value to write */
          209  +  int *pRc                        /* IN/OUT: Error code */
          210  +){
          211  +  const int iMax = dbMaxLevel(pParam);
          212  +  char zKey[DB_KEY_BYTES];
          213  +  int i;
          214  +  int rc;
          215  +
          216  +  assert( iKey>=0 && iKey<pParam->nKey );
          217  +  dbFormatKey(pParam, 0, iKey, zKey);
          218  +
          219  +  /* Open a write transaction. This may fail - SQLITE4_BUSY */
          220  +  if( *pRc==0 && tdb_transaction_support(pDb) ){
          221  +    rc = tdb_begin(pDb, 2);
          222  +    if( rc==5 ) return 0;
          223  +    *pRc = rc;
          224  +  }
          225  +
          226  +  testWriteStr(pDb, zKey, zValue, pRc);
          227  +  for(i=1; i<=iMax; i++){
          228  +    char zCksum[DB_KEY_BYTES];
          229  +    u32 iCksum = 0;
          230  +
          231  +    iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc);
          232  +    dbFormatCksumValue(iCksum, zCksum);
          233  +    dbFormatKey(pParam, i, iKey, zKey);
          234  +    testWriteStr(pDb, zKey, zCksum, pRc);
          235  +  }
          236  +  if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc);
          237  +  return 1;
          238  +}
          239  +
          240  +/*************************************************************************
          241  +** The following block contains testXXX() functions that implement a
          242  +** wrapper around the systems native multi-thread support. There are no
          243  +** synchronization primitives - just functions to launch and join 
          244  +** threads. Wrapper functions are:
          245  +**
          246  +**    testThreadSupport()
          247  +**
          248  +**    testThreadInit()
          249  +**    testThreadShutdown()
          250  +**    testThreadLaunch()
          251  +**    testThreadWait()
          252  +**
          253  +**    testThreadSetHalt()
          254  +**    testThreadGetHalt()
          255  +**    testThreadSetResult()
          256  +**    testThreadGetResult()
          257  +**
          258  +**    testThreadEnterMutex()
          259  +**    testThreadLeaveMutex()
          260  +*/
          261  +typedef struct ThreadSet ThreadSet;
          262  +#ifdef LSM_MUTEX_PTHREADS
          263  +
          264  +#include <pthread.h>
          265  +#include <unistd.h>
          266  +
          267  +typedef struct Thread Thread;
          268  +struct Thread {
          269  +  int rc;
          270  +  char *zMsg;
          271  +  pthread_t id;
          272  +  void (*xMain)(ThreadSet *, int, void *);
          273  +  void *pCtx;
          274  +  ThreadSet *pThreadSet;
          275  +};
          276  +
          277  +struct ThreadSet {
          278  +  int bHalt;                      /* Halt flag */
          279  +  int nThread;                    /* Number of threads */
          280  +  Thread *aThread;                /* Array of Thread structures */
          281  +  pthread_mutex_t mutex;          /* Mutex used for cheating */
          282  +};
          283  +
          284  +/*
          285  +** Return true if this build supports threads, or false otherwise. If
          286  +** this function returns false, no other testThreadXXX() functions should
          287  +** be called.
          288  +*/
          289  +static int testThreadSupport(){ return 1; }
          290  +
          291  +/*
          292  +** Allocate and return a thread-set handle with enough space allocated
          293  +** to handle up to nMax threads. Each call to this function should be
          294  +** matched by a call to testThreadShutdown() to delete the object.
          295  +*/
          296  +static ThreadSet *testThreadInit(int nMax){
          297  +  int nByte;                      /* Total space to allocate */
          298  +  ThreadSet *p;                   /* Return value */
          299  +
          300  +  nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax;
          301  +  p = (ThreadSet *)testMalloc(nByte);
          302  +  p->nThread = nMax;
          303  +  p->aThread = (Thread *)&p[1];
          304  +  pthread_mutex_init(&p->mutex, 0);
          305  +
          306  +  return p;
          307  +}
          308  +
          309  +/*
          310  +** Delete a thread-set object and release all resources held by it.
          311  +*/
          312  +static void testThreadShutdown(ThreadSet *p){
          313  +  int i;
          314  +  for(i=0; i<p->nThread; i++){
          315  +    testFree(p->aThread[i].zMsg);
          316  +  }
          317  +  pthread_mutex_destroy(&p->mutex);
          318  +  testFree(p);
          319  +}
          320  +
          321  +static void *ttMain(void *pArg){
          322  +  Thread *pThread = (Thread *)pArg;
          323  +  int iThread;
          324  +  iThread = (pThread - pThread->pThreadSet->aThread);
          325  +  pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx);
          326  +  return 0;
          327  +}
          328  +
          329  +/*
          330  +** Launch a new thread.
          331  +*/
          332  +static int testThreadLaunch(
          333  +  ThreadSet *p,
          334  +  int iThread,
          335  +  void (*xMain)(ThreadSet *, int, void *),
          336  +  void *pCtx
          337  +){
          338  +  int rc;
          339  +  Thread *pThread;
          340  +
          341  +  assert( iThread>=0 && iThread<p->nThread );
          342  +
          343  +  pThread = &p->aThread[iThread];
          344  +  assert( pThread->pThreadSet==0 );
          345  +  pThread->xMain = xMain;
          346  +  pThread->pCtx = pCtx;
          347  +  pThread->pThreadSet = p;
          348  +  rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread);
          349  +
          350  +  return rc;
          351  +}
          352  +
          353  +/*
          354  +** Set the thread-set "halt" flag.
          355  +*/
          356  +static void testThreadSetHalt(ThreadSet *pThreadSet){
          357  +  pThreadSet->bHalt = 1;
          358  +}
          359  +
          360  +/*
          361  +** Return the current value of the thread-set "halt" flag.
          362  +*/
          363  +static int testThreadGetHalt(ThreadSet *pThreadSet){
          364  +  return pThreadSet->bHalt;
          365  +}
          366  +
          367  +static void testThreadSleep(ThreadSet *pThreadSet, int nMs){
          368  +  int nRem = nMs;
          369  +  while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){
          370  +    usleep(50000);
          371  +    nRem -= 50;
          372  +  }
          373  +}
          374  +
          375  +/*
          376  +** Wait for all threads launched to finish before returning. If nMs
          377  +** is greater than zero, set the "halt" flag to tell all threads
          378  +** to halt after waiting nMs milliseconds.
          379  +*/
          380  +static void testThreadWait(ThreadSet *pThreadSet, int nMs){
          381  +  int i;
          382  +
          383  +  testThreadSleep(pThreadSet, nMs);
          384  +  testThreadSetHalt(pThreadSet);
          385  +  for(i=0; i<pThreadSet->nThread; i++){
          386  +    Thread *pThread = &pThreadSet->aThread[i];
          387  +    if( pThread->xMain ){
          388  +      pthread_join(pThread->id, 0);
          389  +    }
          390  +  }
          391  +}
          392  +
          393  +/*
          394  +** Set the result for thread iThread. 
          395  +*/
          396  +static void testThreadSetResult(
          397  +  ThreadSet *pThreadSet,          /* Thread-set handle */
          398  +  int iThread,                    /* Set result for this thread */
          399  +  int rc,                         /* Result error code */
          400  +  char *zFmt,                     /* Result string format */
          401  +  ...                             /* Result string formatting args... */
          402  +){
          403  +  va_list ap;
          404  +
          405  +  testFree(pThreadSet->aThread[iThread].zMsg);
          406  +  pThreadSet->aThread[iThread].rc = rc;
          407  +  pThreadSet->aThread[iThread].zMsg = 0;
          408  +  if( zFmt ){
          409  +    va_start(ap, zFmt);
          410  +    pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap);
          411  +    va_end(ap);
          412  +  }
          413  +}
          414  +
          415  +/*
          416  +** Retrieve the result for thread iThread. 
          417  +*/
          418  +static int testThreadGetResult(
          419  +  ThreadSet *pThreadSet,          /* Thread-set handle */
          420  +  int iThread,                    /* Get result for this thread */
          421  +  const char **pzRes              /* OUT: Pointer to result string */
          422  +){
          423  +  if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg;
          424  +  return pThreadSet->aThread[iThread].rc;
          425  +}
          426  +
          427  +/*
          428  +** Enter and leave the test case mutex.
          429  +*/
          430  +#if 0
          431  +static void testThreadEnterMutex(ThreadSet *p){
          432  +  pthread_mutex_lock(&p->mutex);
          433  +}
          434  +static void testThreadLeaveMutex(ThreadSet *p){
          435  +  pthread_mutex_unlock(&p->mutex);
          436  +}
          437  +#endif
          438  +#endif
          439  +
          440  +#if !defined(LSM_MUTEX_PTHREADS)
          441  +static int testThreadSupport(){ return 0; }
          442  +
          443  +#define testThreadInit(a) 0
          444  +#define testThreadShutdown(a)
          445  +#define testThreadLaunch(a,b,c,d) 0
          446  +#define testThreadWait(a,b)
          447  +#define testThreadSetHalt(a)
          448  +#define testThreadGetHalt(a) 0
          449  +#define testThreadGetResult(a,b,c) 0
          450  +#define testThreadSleep(a,b) 0
          451  +
          452  +static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){
          453  +  unused_parameter(a);
          454  +  unused_parameter(b);
          455  +  unused_parameter(c);
          456  +  unused_parameter(d);
          457  +}
          458  +#endif
          459  +/* End of threads wrapper.
          460  +*************************************************************************/
          461  +
          462  +/*************************************************************************
          463  +** Below this point is the third part of this file - the implementation
          464  +** of the mt1.* tests.
          465  +*/
          466  +typedef struct Mt1Test Mt1Test;
          467  +struct Mt1Test {
          468  +  DbParameters param;             /* Description of database to read/write */
          469  +  int nReadwrite;                 /* Number of read/write threads */
          470  +  int nFastReader;                /* Number of fast reader threads */
          471  +  int nSlowReader;                /* Number of slow reader threads */
          472  +  int nMs;                        /* How long to run for */
          473  +  const char *zSystem;            /* Database system to test */
          474  +};
          475  +
          476  +typedef struct Mt1DelayCtx Mt1DelayCtx;
          477  +struct Mt1DelayCtx {
          478  +  ThreadSet *pSet;                /* Threadset to sleep within */
          479  +  int nMs;                        /* Sleep in ms */
          480  +};
          481  +
          482  +static void xMt1Delay(void *pCtx){
          483  +  Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx;
          484  +  testThreadSleep(p->pSet, p->nMs);
          485  +}
          486  +
          487  +#define MT1_THREAD_RDWR 0
          488  +#define MT1_THREAD_SLOW 1
          489  +#define MT1_THREAD_FAST 2
          490  +
          491  +static void xMt1Work(lsm_db *pDb, void *pCtx){
          492  +#if 0
          493  +  char *z = 0;
          494  +  lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z);
          495  +  printf("%s\n", z);
          496  +  fflush(stdout);
          497  +#endif
          498  +}
          499  +
          500  +/*
          501  +** This is the main() proc for all threads in test case "mt1".
          502  +*/
          503  +static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){
          504  +  Mt1Test *p = (Mt1Test *)pCtx;   /* Test parameters */
          505  +  Mt1DelayCtx delay;
          506  +  int nRead = 0;                  /* Number of calls to dbReadOperation() */
          507  +  int nWrite = 0;                 /* Number of completed database writes */
          508  +  int rc = 0;                     /* Error code */
          509  +  int iPrng;                      /* Prng argument variable */
          510  +  TestDb *pDb;                    /* Database handle */
          511  +  int eType;
          512  +
          513  +  delay.pSet = pThreadSet;
          514  +  delay.nMs = 0;
          515  +  if( iThread<p->nReadwrite ){
          516  +    eType = MT1_THREAD_RDWR;
          517  +  }else if( iThread<(p->nReadwrite+p->nFastReader) ){
          518  +    eType = MT1_THREAD_FAST;
          519  +  }else{
          520  +    eType = MT1_THREAD_SLOW;
          521  +    delay.nMs = (p->nMs / 20);
          522  +  }
          523  +
          524  +  /* Open a new database connection. Initialize the pseudo-random number
          525  +  ** argument based on the thread number.  */
          526  +  iPrng = testPrngValue(iThread);
          527  +  pDb = testOpen(p->zSystem, 0, &rc);
          528  +
          529  +  if( rc==0 ){
          530  +    tdb_lsm_config_work_hook(pDb, xMt1Work, 0);
          531  +  }
          532  +
          533  +  /* Loop until either an error occurs or some other thread sets the
          534  +  ** halt flag.  */
          535  +  while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){
          536  +    int iKey;
          537  +
          538  +    /* Perform a read operation on an arbitrarily selected key. */
          539  +    iKey = (testPrngValue(iPrng++) % p->param.nKey);
          540  +    dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc);
          541  +    if( rc ) continue;
          542  +    nRead++;
          543  +
          544  +    /* Attempt to write an arbitrary key value pair (and update the associated
          545  +    ** checksum entries). dbWriteOperation() returns 1 if the write is
          546  +    ** successful, or 0 if it failed with an LSM_BUSY error.  */
          547  +    if( eType==MT1_THREAD_RDWR ){
          548  +      char aValue[50];
          549  +      char aRnd[25];
          550  +
          551  +      iKey = (testPrngValue(iPrng++) % p->param.nKey);
          552  +      testPrngString(iPrng, aRnd, sizeof(aRnd));
          553  +      iPrng += sizeof(aRnd);
          554  +      snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd);
          555  +      nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc);
          556  +    }
          557  +  }
          558  +  testClose(&pDb);
          559  +
          560  +  /* If an error has occured, set the thread error code and the threadset 
          561  +  ** halt flag to tell the other test threads to halt. Otherwise, set the
          562  +  ** thread error code to 0 and post a message with the number of read
          563  +  ** and write operations completed.  */
          564  +  if( rc ){
          565  +    testThreadSetResult(pThreadSet, iThread, rc, 0);
          566  +    testThreadSetHalt(pThreadSet);
          567  +  }else{
          568  +    testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite);
          569  +  }
          570  +}
          571  +
          572  +static void do_test_mt1(
          573  +  const char *zSystem,            /* Database system name */
          574  +  const char *zPattern,           /* Run test cases that match this pattern */
          575  +  int *pRc                        /* IN/OUT: Error code */
          576  +){
          577  +  Mt1Test aTest[] = {
          578  +    /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */
          579  +    { {10, 1000},     4, 0, 0,   10000,   0 },
          580  +    { {10, 1000},     4, 4, 2,   100000,  0 },
          581  +    { {10, 100000},   4, 0, 0,   10000,   0 },
          582  +    { {10, 100000},   4, 4, 2,   100000,  0 },
          583  +  };
          584  +  int i;
          585  +
          586  +  for(i=0; *pRc==0 && i<ArraySize(aTest); i++){
          587  +    Mt1Test *p = &aTest[i];
          588  +    int bRun = testCaseBegin(pRc, zPattern, 
          589  +        "mt1.%s.db=%d,%d.ms=%d.rdwr=%d.fast=%d.slow=%d", 
          590  +        zSystem, p->param.nFanout, p->param.nKey, 
          591  +        p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader
          592  +    );
          593  +    if( bRun ){
          594  +      TestDb *pDb;
          595  +      ThreadSet *pSet;
          596  +      int iThread;
          597  +      int nThread;
          598  +
          599  +      p->zSystem = zSystem;
          600  +      pDb = testOpen(zSystem, 1, pRc);
          601  +
          602  +      nThread = p->nReadwrite + p->nFastReader + p->nSlowReader;
          603  +      pSet = testThreadInit(nThread);
          604  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          605  +        testThreadLaunch(pSet, iThread, mt1Main, (void *)p);
          606  +      }
          607  +
          608  +      testThreadWait(pSet, p->nMs);
          609  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          610  +        *pRc = testThreadGetResult(pSet, iThread, 0);
          611  +      }
          612  +      testCaseFinish(*pRc);
          613  +
          614  +      for(iThread=0; *pRc==0 && iThread<nThread; iThread++){
          615  +        const char *zMsg = 0;
          616  +        *pRc = testThreadGetResult(pSet, iThread, &zMsg);
          617  +        printf("  Info: thread %d (%d): %s\n", iThread, *pRc, zMsg);
          618  +      }
          619  +
          620  +      testThreadShutdown(pSet);
          621  +      testClose(&pDb);
          622  +    }
          623  +  }
          624  +}
          625  +
          626  +void test_mt(
          627  +  const char *zSystem,            /* Database system name */
          628  +  const char *zPattern,           /* Run test cases that match this pattern */
          629  +  int *pRc                        /* IN/OUT: Error code */
          630  +){
          631  +  if( testThreadSupport()==0 ) return;
          632  +  do_test_mt1(zSystem, zPattern, pRc);
          633  +}

Added ext/lsm1/lsm-test/lsmtest6.c.

            1  +
            2  +#include "lsmtest.h"
            3  +
            4  +typedef struct OomTest OomTest;
            5  +struct OomTest {
            6  +  lsm_env *pEnv;
            7  +  int iNext;                      /* Next value to pass to testMallocOom() */
            8  +  int nFail;                      /* Number of OOM events injected */
            9  +  int bEnable;
           10  +  int rc;                         /* Test case error code */
           11  +};
           12  +
           13  +static void testOomStart(OomTest *p){
           14  +  memset(p, 0, sizeof(OomTest));
           15  +  p->iNext = 1;
           16  +  p->bEnable = 1;
           17  +  p->nFail = 1;
           18  +  p->pEnv = tdb_lsm_env();
           19  +}
           20  +
           21  +static void xOomHook(OomTest *p){
           22  +  p->nFail++;
           23  +}
           24  +
           25  +static int testOomContinue(OomTest *p){
           26  +  if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){
           27  +    return 0;
           28  +  }
           29  +  p->nFail = 0;
           30  +  testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p);
           31  +  return 1;
           32  +}
           33  +
           34  +static void testOomEnable(OomTest *p, int bEnable){
           35  +  p->bEnable = bEnable;
           36  +  testMallocOomEnable(p->pEnv, bEnable);
           37  +}
           38  +
           39  +static void testOomNext(OomTest *p){
           40  +  p->iNext++;
           41  +}
           42  +
           43  +static int testOomHit(OomTest *p){
           44  +  return (p->nFail>0);
           45  +}
           46  +
           47  +static int testOomFinish(OomTest *p){
           48  +  return p->rc;
           49  +}
           50  +
           51  +static void testOomAssert(OomTest *p, int bVal){
           52  +  if( bVal==0 ){
           53  +    test_failed();
           54  +    p->rc = 1;
           55  +  }
           56  +}
           57  +
           58  +/*
           59  +** Test that the error code matches the state of the OomTest object passed
           60  +** as the first argument. Specifically, check that rc is LSM_NOMEM if an 
           61  +** OOM error has already been injected, or LSM_OK if not.
           62  +*/
           63  +static void testOomAssertRc(OomTest *p, int rc){
           64  +  testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM);
           65  +  testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 );
           66  +}
           67  +
           68  +static void testOomOpen(
           69  +  OomTest *pOom,
           70  +  const char *zName,
           71  +  lsm_db **ppDb,
           72  +  int *pRc
           73  +){
           74  +  if( *pRc==LSM_OK ){
           75  +    int rc;
           76  +    rc = lsm_new(tdb_lsm_env(), ppDb);
           77  +    if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName);
           78  +    testOomAssertRc(pOom, rc);
           79  +    *pRc = rc;
           80  +  }
           81  +}
           82  +
           83  +static void testOomFetch(
           84  +  OomTest *pOom,
           85  +  lsm_db *pDb,
           86  +  void *pKey, int nKey,
           87  +  void *pVal, int nVal,
           88  +  int *pRc
           89  +){
           90  +  testOomAssertRc(pOom, *pRc);
           91  +  if( *pRc==LSM_OK ){
           92  +    lsm_cursor *pCsr;
           93  +    int rc;
           94  +
           95  +    rc = lsm_csr_open(pDb, &pCsr);
           96  +    if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0);
           97  +    testOomAssertRc(pOom, rc);
           98  +
           99  +    if( rc==LSM_OK ){
          100  +      const void *p; int n;
          101  +      testOomAssert(pOom, lsm_csr_valid(pCsr));
          102  +
          103  +      rc = lsm_csr_key(pCsr, &p, &n);
          104  +      testOomAssertRc(pOom, rc);
          105  +      testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) );
          106  +    }
          107  +
          108  +    if( rc==LSM_OK ){
          109  +      const void *p; int n;
          110  +      testOomAssert(pOom, lsm_csr_valid(pCsr));
          111  +
          112  +      rc = lsm_csr_value(pCsr, &p, &n);
          113  +      testOomAssertRc(pOom, rc);
          114  +      testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) );
          115  +    }
          116  +
          117  +    lsm_csr_close(pCsr);
          118  +    *pRc = rc;
          119  +  }
          120  +}
          121  +
          122  +static void testOomWrite(
          123  +  OomTest *pOom,
          124  +  lsm_db *pDb,
          125  +  void *pKey, int nKey,
          126  +  void *pVal, int nVal,
          127  +  int *pRc
          128  +){
          129  +  testOomAssertRc(pOom, *pRc);
          130  +  if( *pRc==LSM_OK ){
          131  +    int rc;
          132  +
          133  +    rc = lsm_insert(pDb, pKey, nKey, pVal, nVal);
          134  +    testOomAssertRc(pOom, rc);
          135  +
          136  +    *pRc = rc;
          137  +  }
          138  +}
          139  +
          140  +
          141  +static void testOomFetchStr(
          142  +  OomTest *pOom,
          143  +  lsm_db *pDb,
          144  +  const char *zKey,
          145  +  const char *zVal,
          146  +  int *pRc
          147  +){
          148  +  int nKey = strlen(zKey);
          149  +  int nVal = strlen(zVal);
          150  +  testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
          151  +}
          152  +
          153  +static void testOomFetchData(
          154  +  OomTest *pOom,
          155  +  lsm_db *pDb,
          156  +  Datasource *pData,
          157  +  int iKey,
          158  +  int *pRc
          159  +){
          160  +  void *pKey; int nKey;
          161  +  void *pVal; int nVal;
          162  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          163  +  testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
          164  +}
          165  +
          166  +static void testOomWriteStr(
          167  +  OomTest *pOom,
          168  +  lsm_db *pDb,
          169  +  const char *zKey,
          170  +  const char *zVal,
          171  +  int *pRc
          172  +){
          173  +  int nKey = strlen(zKey);
          174  +  int nVal = strlen(zVal);
          175  +  testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc);
          176  +}
          177  +
          178  +static void testOomWriteData(
          179  +  OomTest *pOom,
          180  +  lsm_db *pDb,
          181  +  Datasource *pData,
          182  +  int iKey,
          183  +  int *pRc
          184  +){
          185  +  void *pKey; int nKey;
          186  +  void *pVal; int nVal;
          187  +  testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal);
          188  +  testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc);
          189  +}
          190  +
          191  +static void testOomScan(
          192  +  OomTest *pOom, 
          193  +  lsm_db *pDb, 
          194  +  int bReverse,
          195  +  const void *pKey, int nKey,
          196  +  int nScan,
          197  +  int *pRc
          198  +){
          199  +  if( *pRc==0 ){