/ Check-in [00990020]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge updates from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | winCeLocalTime
Files: files | file ages | folders
SHA1: 00990020d07d7c87b922cdbfa5373298a86bb4b3
User & Date: mistachkin 2016-04-05 17:59:56
Context
2016-04-11
22:10
Fix some compilation issues. check-in: 016481b3 user: mistachkin tags: winCeLocalTime
2016-04-05
17:59
Merge updates from trunk. check-in: 00990020 user: mistachkin tags: winCeLocalTime
15:59
Incorrect version of build.c checked in at [87e5f5a6c60e3] (because I neglected to press the "Save" button on the text editor). There is no change to the logic, just improved presentation. check-in: cf569f9f user: drh tags: trunk
2016-01-23
01:54
Cleanup localtime() support for Windows CE. check-in: 39759a55 user: mistachkin tags: winCeLocalTime
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

    26     26   # will run on the target platform.  (BCC and TCC are usually the
    27     27   # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
    28     28   # are provide so that these aspects of the build process can be changed
    29     29   # on the "make" command-line.  Ex:  "make CC=clang CFLAGS=-fsanitize=undefined"
    30     30   #
    31     31   CC = @CC@
    32     32   CFLAGS = @CPPFLAGS@ @CFLAGS@
    33         -TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3
           33  +TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
           34  +TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
    34     35   
    35     36   # Define this for the autoconf-based build, so that the code knows it can
    36     37   # include the generated config.h
    37     38   # 
    38     39   TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
    39     40   
    40     41   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
................................................................................
    77     78   TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
    78     79   
    79     80   # Enable/disable loadable extensions, and other optional features
    80     81   # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
    81     82   # The same set of OMIT and ENABLE flags should be passed to the 
    82     83   # LEMON parser generator and the mkkeywordhash tool as well.
    83     84   OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
           85  +OPT_FEATURE_FLAGS += -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK
    84     86   
    85     87   TCC += $(OPT_FEATURE_FLAGS)
    86     88   
    87     89   # Add in any optional parameters specified on the make commane line
    88     90   # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
    89     91   TCC += $(OPTS)
    90     92   
................................................................................
   169    171            expr.lo fault.lo fkey.lo \
   170    172            fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
   171    173            fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
   172    174            fts3_tokenize_vtab.lo \
   173    175            fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
   174    176   	 fts5.lo \
   175    177            func.lo global.lo hash.lo \
   176         -         icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \
          178  +         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
   177    179            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   178    180            memjournal.lo \
   179    181            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   180    182            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   181    183            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   182         -         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
          184  +         random.lo resolve.lo rowset.lo rtree.lo \
          185  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
   183    186            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   184    187            update.lo util.lo vacuum.lo \
   185    188            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   186    189            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   187    190            utf.lo vtab.lo
   188    191   
   189    192   # Object files for the amalgamation.
................................................................................
   220    223     $(TOP)/src/fkey.c \
   221    224     $(TOP)/src/func.c \
   222    225     $(TOP)/src/global.c \
   223    226     $(TOP)/src/hash.c \
   224    227     $(TOP)/src/hash.h \
   225    228     $(TOP)/src/hwtime.h \
   226    229     $(TOP)/src/insert.c \
   227         -  $(TOP)/src/journal.c \
   228    230     $(TOP)/src/legacy.c \
   229    231     $(TOP)/src/loadext.c \
   230    232     $(TOP)/src/main.c \
   231    233     $(TOP)/src/malloc.c \
   232    234     $(TOP)/src/mem0.c \
   233    235     $(TOP)/src/mem1.c \
   234    236     $(TOP)/src/mem2.c \
................................................................................
   339    341   SRC += \
   340    342     $(TOP)/ext/icu/sqliteicu.h \
   341    343     $(TOP)/ext/icu/icu.c
   342    344   SRC += \
   343    345     $(TOP)/ext/rtree/rtree.h \
   344    346     $(TOP)/ext/rtree/rtree.c
   345    347   SRC += \
          348  +  $(TOP)/ext/session/sqlite3session.c \
          349  +  $(TOP)/ext/session/sqlite3session.h
          350  +SRC += \
   346    351     $(TOP)/ext/rbu/sqlite3rbu.h \
   347    352     $(TOP)/ext/rbu/sqlite3rbu.c
   348    353   SRC += \
   349    354     $(TOP)/ext/misc/json1.c
   350         -
   351         -
   352    355   
   353    356   # Generated source code files
   354    357   #
   355    358   SRC += \
   356    359     keywordhash.h \
   357    360     opcodes.c \
   358    361     opcodes.h \
................................................................................
   372    375     $(TOP)/src/test6.c \
   373    376     $(TOP)/src/test7.c \
   374    377     $(TOP)/src/test8.c \
   375    378     $(TOP)/src/test9.c \
   376    379     $(TOP)/src/test_autoext.c \
   377    380     $(TOP)/src/test_async.c \
   378    381     $(TOP)/src/test_backup.c \
          382  +  $(TOP)/src/test_bestindex.c \
   379    383     $(TOP)/src/test_blob.c \
   380    384     $(TOP)/src/test_btree.c \
   381    385     $(TOP)/src/test_config.c \
   382    386     $(TOP)/src/test_demovfs.c \
   383    387     $(TOP)/src/test_devsym.c \
   384    388     $(TOP)/src/test_fs.c \
   385    389     $(TOP)/src/test_func.c \
................................................................................
   401    405     $(TOP)/src/test_syscall.c \
   402    406     $(TOP)/src/test_tclvar.c \
   403    407     $(TOP)/src/test_thread.c \
   404    408     $(TOP)/src/test_vfs.c \
   405    409     $(TOP)/src/test_windirent.c \
   406    410     $(TOP)/src/test_wsd.c       \
   407    411     $(TOP)/ext/fts3/fts3_term.c \
   408         -  $(TOP)/ext/fts3/fts3_test.c \
          412  +  $(TOP)/ext/fts3/fts3_test.c  \
          413  +  $(TOP)/ext/session/test_session.c \
   409    414     $(TOP)/ext/rbu/test_rbu.c 
   410    415   
   411    416   # Statically linked extensions
   412    417   #
   413    418   TESTSRC += \
   414    419     $(TOP)/ext/misc/amatch.c \
   415    420     $(TOP)/ext/misc/closure.c \
................................................................................
   470    475     parse.c \
   471    476     $(TOP)/ext/fts3/fts3.c \
   472    477     $(TOP)/ext/fts3/fts3_aux.c \
   473    478     $(TOP)/ext/fts3/fts3_expr.c \
   474    479     $(TOP)/ext/fts3/fts3_term.c \
   475    480     $(TOP)/ext/fts3/fts3_tokenizer.c \
   476    481     $(TOP)/ext/fts3/fts3_write.c \
   477         -  $(TOP)/ext/async/sqlite3async.c
          482  +  $(TOP)/ext/async/sqlite3async.c \
          483  +  $(TOP)/ext/session/sqlite3session.c 
   478    484   
   479    485   # Header files used by all library source files.
   480    486   #
   481    487   HDR = \
   482    488      $(TOP)/src/btree.h \
   483    489      $(TOP)/src/btreeInt.h \
   484    490      $(TOP)/src/hash.h \
................................................................................
   545    551   
   546    552   # Standard options to testfixture
   547    553   #
   548    554   TESTOPTS = --verbose=file --output=test-out.txt
   549    555   
   550    556   # Extra compiler options for various shell tools
   551    557   #
   552         -SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
          558  +SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
          559  +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   553    560   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   554    561   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
   555    562   
   556    563   # This is the default Makefile target.  The objects listed here
   557    564   # are what get build when you type just "make" with no arguments.
   558    565   #
   559    566   all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
................................................................................
   571    578   libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
   572    579   	$(LTLINK) -no-undefined -o $@ tclsqlite.lo \
   573    580   		libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \
   574    581   		-rpath "$(TCLLIBDIR)" \
   575    582   		-version-info "8:6:8" \
   576    583   		-avoid-version
   577    584   
   578         -sqlite3$(TEXE):	$(TOP)/src/shell.c libsqlite3.la sqlite3.h
          585  +sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
   579    586   	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
   580         -		$(TOP)/src/shell.c libsqlite3.la \
          587  +		$(TOP)/src/shell.c sqlite3.c \
   581    588   		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
   582    589   
   583    590   sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
   584    591   	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
          592  +
          593  +srcck1$(BEXE):	$(TOP)/tool/srcck1.c
          594  +	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
          595  +
          596  +sourcetest:	srcck1$(BEXE) sqlite3.c
          597  +	./srcck1 sqlite3.c
   585    598   
   586    599   fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
   587    600   	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
   588    601   	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
   589    602   
   590    603   fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
   591    604   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
................................................................................
   623    636   	mv vdbe.new tsrc/vdbe.c
   624    637   	cp fts5.c fts5.h tsrc
   625    638   	touch .target_source
   626    639   
   627    640   sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
   628    641   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
   629    642   	cp tsrc/shell.c tsrc/sqlite3ext.h .
          643  +	cp $(TOP)/ext/session/sqlite3session.h .
   630    644   
   631    645   sqlite3ext.h:	.target_source
   632    646   	cp tsrc/sqlite3ext.h .
   633    647   
   634    648   tclsqlite3.c:	sqlite3.c
   635    649   	echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
   636    650   	cat sqlite3.c >>tclsqlite3.c
................................................................................
   727    741   
   728    742   hash.lo:	$(TOP)/src/hash.c $(HDR)
   729    743   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/hash.c
   730    744   
   731    745   insert.lo:	$(TOP)/src/insert.c $(HDR)
   732    746   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c
   733    747   
   734         -journal.lo:	$(TOP)/src/journal.c $(HDR)
   735         -	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/journal.c
   736         -
   737    748   legacy.lo:	$(TOP)/src/legacy.c $(HDR)
   738    749   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c
   739    750   
   740    751   loadext.lo:	$(TOP)/src/loadext.c $(HDR)
   741    752   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/loadext.c
   742    753   
   743    754   main.lo:	$(TOP)/src/main.c $(HDR)
................................................................................
   989   1000   
   990   1001   fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
   991   1002   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
   992   1003   
   993   1004   rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
   994   1005   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
   995   1006   
         1007  +sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
         1008  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
         1009  +
   996   1010   json1.lo:	$(TOP)/ext/misc/json1.c
   997   1011   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
   998   1012   
   999   1013   # FTS5 things
  1000   1014   #
  1001   1015   FTS5_SRC = \
  1002   1016      $(TOP)/ext/fts5/fts5.h \
................................................................................
  1039   1053   # fixture.  Otherwise link against libsqlite3.la.  (This distinction is
  1040   1054   # necessary because the test fixture requires non-API symbols which are
  1041   1055   # hidden when the library is built via the amalgamation).
  1042   1056   #
  1043   1057   TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1044   1058   TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
  1045   1059   TESTFIXTURE_FLAGS += -DBUILD_sqlite
         1060  +TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
         1061  +TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
  1046   1062   
  1047   1063   TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
  1048   1064   TESTFIXTURE_SRC1 = sqlite3.c
  1049   1065   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
  1050   1066   TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
  1051   1067   
  1052   1068   testfixture$(TEXE):	$(TESTFIXTURE_SRC)
................................................................................
  1079   1095   #
  1080   1096   quicktest:	./testfixture$(TEXE)
  1081   1097   	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
  1082   1098   
  1083   1099   # This is the common case.  Run many tests that do not take too long,
  1084   1100   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1085   1101   #
  1086         -test:	$(TESTPROGS) fastfuzztest
         1102  +test:	$(TESTPROGS) sourcetest fastfuzztest
  1087   1103   	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
  1088   1104   
  1089   1105   # Run a test using valgrind.  This can take a really long time
  1090   1106   # because valgrind is so much slower than a native machine.
  1091   1107   #
  1092   1108   valgrindtest:	$(TESTPROGS) valgrindfuzz
  1093   1109   	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS)
................................................................................
  1119   1135   
  1120   1136   showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
  1121   1137   	$(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS)
  1122   1138   
  1123   1139   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1124   1140   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1125   1141   
         1142  +changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
         1143  +	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
         1144  +
  1126   1145   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
  1127   1146   	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
  1128   1147   
  1129   1148   LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
  1130   1149   	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
  1131   1150   
  1132   1151   wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
................................................................................
  1142   1161   	$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
  1143   1162   
  1144   1163   # This target will fail if the SQLite amalgamation contains any exported
  1145   1164   # symbols that do not begin with "sqlite3_". It is run as part of the
  1146   1165   # releasetest.tcl script.
  1147   1166   #
  1148   1167   checksymbols: sqlite3.lo
  1149         -	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
         1168  +	nm -g --defined-only sqlite3.o | egrep -v ' sqlite3(changeset|session)?_' ; test $$? -ne 0
  1150   1169   	echo '0 errors out of 1 tests'
  1151   1170   
  1152         -# Build the amalgamation-autoconf package.
         1171  +# Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
         1172  +# a tarball named for the version number.  Ex:  sqlite-autoconf-3110000.tar.gz.
         1173  +# The snapshot-tarball target builds a tarball named by the SHA1 hash
  1153   1174   #
  1154   1175   amalgamation-tarball: sqlite3.c
  1155         -	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh
         1176  +	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal
         1177  +
         1178  +snapshot-tarball: sqlite3.c
         1179  +	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
  1156   1180   
  1157   1181   # The next two rules are used to support the "threadtest" target. Building
  1158   1182   # threadtest runs a few thread-safety tests that are implemented in C. This
  1159   1183   # target is invoked by the releasetest.tcl script.
  1160   1184   # 
  1161   1185   THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
  1162   1186                     $(TOP)/test/tt3_checkpoint.c \
................................................................................
  1206   1230   	rm -f *.da *.bb *.bbg gmon.out
  1207   1231   	rm -rf quota2a quota2b quota2c
  1208   1232   	rm -rf tsrc .target_source
  1209   1233   	rm -f tclsqlite3$(TEXE)
  1210   1234   	rm -f testfixture$(TEXE) test.db
  1211   1235   	rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
  1212   1236   	rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
  1213         -	rm -f wordcount$(TEXE)
         1237  +	rm -f wordcount$(TEXE) changeset$(TEXE)
  1214   1238   	rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
  1215   1239   	rm -f sqlite3.c
  1216   1240   	rm -f sqlite3rc.h
  1217   1241   	rm -f shell.c sqlite3ext.h
  1218   1242   	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
  1219   1243   	rm -f sqlite-*-output.vsix
  1220   1244   	rm -f mptester mptester.exe
  1221   1245   	rm -f rbu rbu.exe
         1246  +	rm -f srcck1 srcck1.exe
  1222   1247   	rm -f fuzzershell fuzzershell.exe
  1223   1248   	rm -f fuzzcheck fuzzcheck.exe
  1224   1249   	rm -f sqldiff sqldiff.exe
  1225   1250   	rm -f fts5.* fts5parse.*
  1226   1251   
  1227   1252   distclean:	clean
  1228   1253   	rm -f config.h config.log config.status libtool Makefile sqlite3.pc

Changes to Makefile.msc.

     6      6   ###############################################################################
     7      7   
     8      8   # The toplevel directory of the source tree.  This is the directory
     9      9   # that contains this "Makefile.msc".
    10     10   #
    11     11   TOP = .
    12     12   
           13  +# <<mark>>
    13     14   # Set this non-0 to create and use the SQLite amalgamation file.
    14     15   #
    15     16   !IFNDEF USE_AMALGAMATION
    16     17   USE_AMALGAMATION = 1
    17     18   !ENDIF
           19  +# <</mark>>
    18     20   
    19     21   # Set this non-0 to enable full warnings (-W4, etc) when compiling.
    20     22   #
    21     23   !IFNDEF USE_FULLWARN
    22     24   USE_FULLWARN = 0
    23     25   !ENDIF
    24     26   
................................................................................
    64     66   # Set this non-0 to split the SQLite amalgamation file into chunks to
    65     67   # be used for debugging with Visual Studio.
    66     68   #
    67     69   !IFNDEF SPLIT_AMALGAMATION
    68     70   SPLIT_AMALGAMATION = 0
    69     71   !ENDIF
    70     72   
           73  +# <<mark>>
    71     74   # Set this non-0 to use the International Components for Unicode (ICU).
    72     75   #
    73     76   !IFNDEF USE_ICU
    74     77   USE_ICU = 0
    75     78   !ENDIF
           79  +# <</mark>>
    76     80   
    77     81   # Set this non-0 to dynamically link to the MSVC runtime library.
    78     82   #
    79     83   !IFNDEF USE_CRT_DLL
    80     84   USE_CRT_DLL = 0
    81     85   !ENDIF
    82     86   
................................................................................
   118    122   # This setting does not apply to any binaries that require Tcl to operate
   119    123   # properly (i.e. the text fixture, etc).
   120    124   #
   121    125   !IFNDEF FOR_WINRT
   122    126   FOR_WINRT = 0
   123    127   !ENDIF
   124    128   
   125         -# Set this non-0 to compile binaries suitable for the UAP environment.
          129  +# Set this non-0 to compile binaries suitable for the UWP environment.
   126    130   # This setting does not apply to any binaries that require Tcl to operate
   127    131   # properly (i.e. the text fixture, etc).
   128    132   #
   129         -!IFNDEF FOR_UAP
   130         -FOR_UAP = 0
          133  +!IFNDEF FOR_UWP
          134  +FOR_UWP = 0
   131    135   !ENDIF
   132    136   
   133    137   # Set this non-0 to compile binaries suitable for the Windows 10 platform.
   134    138   #
   135    139   !IFNDEF FOR_WIN10
   136    140   FOR_WIN10 = 0
   137    141   !ENDIF
   138    142   
          143  +# <<mark>>
   139    144   # Set this non-0 to skip attempting to look for and/or link with the Tcl
   140    145   # runtime library.
   141    146   #
   142    147   !IFNDEF NO_TCL
   143    148   NO_TCL = 0
   144    149   !ENDIF
          150  +# <</mark>>
   145    151   
   146    152   # Set this to non-0 to create and use PDBs.
   147    153   #
   148    154   !IFNDEF SYMBOLS
   149    155   SYMBOLS = 1
   150    156   !ENDIF
   151    157   
................................................................................
   209    215   !IFNDEF SQLITE3H
   210    216   SQLITE3H = sqlite3.h
   211    217   !ENDIF
   212    218   
   213    219   # This is the name to use for the SQLite dynamic link library (DLL).
   214    220   #
   215    221   !IFNDEF SQLITE3DLL
          222  +!IF $(FOR_WIN10)!=0
          223  +SQLITE3DLL = winsqlite3.dll
          224  +!ELSE
   216    225   SQLITE3DLL = sqlite3.dll
          226  +!ENDIF
   217    227   !ENDIF
   218    228   
   219    229   # This is the name to use for the SQLite import library (LIB).
   220    230   #
   221    231   !IFNDEF SQLITE3LIB
          232  +!IF $(FOR_WIN10)!=0
          233  +SQLITE3LIB = winsqlite3.lib
          234  +!ELSE
   222    235   SQLITE3LIB = sqlite3.lib
          236  +!ENDIF
   223    237   !ENDIF
   224    238   
   225    239   # This is the name to use for the SQLite shell executable (EXE).
   226    240   #
   227    241   !IFNDEF SQLITE3EXE
          242  +!IF $(FOR_WIN10)!=0
          243  +SQLITE3EXE = winsqlite3shell.exe
          244  +!ELSE
   228    245   SQLITE3EXE = sqlite3.exe
          246  +!ENDIF
   229    247   !ENDIF
   230    248   
   231    249   # This is the argument used to set the program database (PDB) file for the
   232    250   # SQLite shell executable (EXE).
   233    251   #
   234    252   !IFNDEF SQLITE3EXEPDB
          253  +!IF $(FOR_WIN10)!=0
          254  +SQLITE3EXEPDB =
          255  +!ELSE
   235    256   SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
          257  +!ENDIF
   236    258   !ENDIF
   237    259   
   238    260   # These are the "standard" SQLite compilation options used when compiling for
   239    261   # the Windows platform.
   240    262   #
   241    263   !IFNDEF OPT_FEATURE_FLAGS
   242    264   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
   243    265   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
   244    266   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
          267  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
          268  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
   245    269   !ENDIF
   246    270   
   247    271   # These are the "extended" SQLite compilation options used when compiling for
   248    272   # the Windows 10 platform.
   249    273   #
   250    274   !IFNDEF EXT_FEATURE_FLAGS
   251    275   !IF $(FOR_WIN10)!=0
................................................................................
   256    280   EXT_FEATURE_FLAGS =
   257    281   !ENDIF
   258    282   !ENDIF
   259    283   
   260    284   ###############################################################################
   261    285   ############################### END OF OPTIONS ################################
   262    286   ###############################################################################
          287  +
          288  +# When compiling for the Windows 10 platform, the PLATFORM macro must be set
          289  +# to an appropriate value (e.g. x86, x64, arm, arm64, etc).
          290  +#
          291  +!IF $(FOR_WIN10)!=0
          292  +!IFNDEF PLATFORM
          293  +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM.
          294  +!ENDIF
          295  +!ENDIF
   263    296   
   264    297   # This assumes that MSVC is always installed in 32-bit Program Files directory
   265    298   # and sets the variable for use in locating other 32-bit installs accordingly.
   266    299   #
   267    300   PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
   268    301   PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
   269    302   
................................................................................
   270    303   # Check for the predefined command macro CC.  This should point to the compiler
   271    304   # binary for the target platform.  If it is not defined, simply define it to
   272    305   # the legacy default value 'cl.exe'.
   273    306   #
   274    307   !IFNDEF CC
   275    308   CC = cl.exe
   276    309   !ENDIF
          310  +
          311  +# Check for the predefined command macro CSC.  This should point to a working
          312  +# C Sharp compiler binary.  If it is not defined, simply define it to the
          313  +# legacy default value 'csc.exe'.
          314  +#
          315  +!IFNDEF CSC
          316  +CSC = csc.exe
          317  +!ENDIF
   277    318   
   278    319   # Check for the command macro LD.  This should point to the linker binary for
   279    320   # the target platform.  If it is not defined, simply define it to the legacy
   280    321   # default value 'link.exe'.
   281    322   #
   282    323   !IFNDEF LD
   283    324   LD = link.exe
................................................................................
   287    328   # compiler binary for the target platform.  If it is not defined, simply define
   288    329   # it to the legacy default value 'rc.exe'.
   289    330   #
   290    331   !IFNDEF RC
   291    332   RC = rc.exe
   292    333   !ENDIF
   293    334   
   294         -# Check for the MSVC runtime library path macro.  Othertise, this value will
          335  +# Check for the MSVC runtime library path macro.  Otherwise, this value will
   295    336   # default to the 'lib' directory underneath the MSVC installation directory.
   296    337   #
   297    338   !IFNDEF CRTLIBPATH
   298    339   CRTLIBPATH = $(VCINSTALLDIR)\lib
   299    340   !ENDIF
   300    341   
   301    342   CRTLIBPATH = $(CRTLIBPATH:\\=\)
................................................................................
   324    365   !ELSEIF $(XCOMPILE)!=0
   325    366   NCC = "$(VCINSTALLDIR)\bin\$(CC)"
   326    367   NCC = $(NCC:\\=\)
   327    368   !ELSE
   328    369   NCC = $(CC)
   329    370   !ENDIF
   330    371   
   331         -# Check for the MSVC native runtime library path macro.  Othertise,
          372  +# Check for the MSVC native runtime library path macro.  Otherwise,
   332    373   # this value will default to the 'lib' directory underneath the MSVC
   333    374   # installation directory.
   334    375   #
   335    376   !IFNDEF NCRTLIBPATH
   336    377   NCRTLIBPATH = $(VCINSTALLDIR)\lib
   337    378   !ENDIF
   338    379   
   339    380   NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
   340    381   
   341         -# Check for the Platform SDK library path macro.  Othertise, this
          382  +# Check for the Platform SDK library path macro.  Otherwise, this
   342    383   # value will default to the 'lib' directory underneath the Windows
   343    384   # SDK installation directory (the environment variable used appears
   344    385   # to be available when using Visual C++ 2008 or later via the
   345    386   # command line).
   346    387   #
   347    388   !IFNDEF NSDKLIBPATH
   348    389   NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
   349    390   !ENDIF
   350    391   
   351    392   NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
          393  +
          394  +# Check for the UCRT library path macro.  Otherwise, this value will
          395  +# default to the version-specific, platform-specific 'lib' directory
          396  +# underneath the Windows SDK installation directory.
          397  +#
          398  +!IFNDEF UCRTLIBPATH
          399  +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM)
          400  +!ENDIF
          401  +
          402  +UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
   352    403   
   353    404   # C compiler and options for use in building executables that
   354    405   # will run on the platform that is doing the build.
   355    406   #
   356    407   !IF $(USE_FULLWARN)!=0
   357    408   BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
   358    409   !ELSE
................................................................................
   385    436   #
   386    437   !IF $(USE_FULLWARN)!=0
   387    438   TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
   388    439   !ELSE
   389    440   TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
   390    441   !ENDIF
   391    442   
   392         -TCC = $(TCC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src -fp:precise
   393         -RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
   394         -
   395         -# Adjust the names of the primary targets for use with Windows 10.
   396         -#
   397         -!IF $(FOR_WIN10)!=0
   398         -SQLITE3DLL = winsqlite3.dll
   399         -SQLITE3LIB = winsqlite3.lib
   400         -SQLITE3EXE = winsqlite3shell.exe
   401         -SQLITE3EXEPDB =
   402         -!ENDIF
          443  +TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise
          444  +RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
   403    445   
   404    446   # Check if we want to use the "stdcall" calling convention when compiling.
   405    447   # This is not supported by the compilers for non-x86 platforms.  It should
   406    448   # also be noted here that building any target with these "stdcall" options
   407    449   # will most likely fail if the Tcl library is also required.  This is due
   408    450   # to how the Tcl library functions are declared and exported (i.e. without
   409    451   # an explicit calling convention, which results in "cdecl").
................................................................................
   436    478   !ENDIF
   437    479   !ENDIF
   438    480   
   439    481   # These are the additional targets that the core library should depend on
   440    482   # when linking.
   441    483   #
   442    484   !IFNDEF CORE_LINK_DEP
   443         -!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          485  +!IF $(DYNAMIC_SHELL)!=0
   444    486   CORE_LINK_DEP =
   445         -!ELSE
          487  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
   446    488   CORE_LINK_DEP = sqlite3.def
          489  +!ELSE
          490  +CORE_LINK_DEP =
   447    491   !ENDIF
   448    492   !ENDIF
   449    493   
   450    494   # These are additional linker options used for the core library.
   451    495   #
   452    496   !IFNDEF CORE_LINK_OPTS
   453         -!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          497  +!IF $(DYNAMIC_SHELL)!=0
   454    498   CORE_LINK_OPTS =
   455         -!ELSE
          499  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
   456    500   CORE_LINK_OPTS = /DEF:sqlite3.def
          501  +!ELSE
          502  +CORE_LINK_OPTS =
   457    503   !ENDIF
   458    504   !ENDIF
   459    505   
   460    506   # These are additional compiler options used for the shell executable.
   461    507   #
   462    508   !IFNDEF SHELL_COMPILE_OPTS
   463    509   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
................................................................................
   522    568   TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
   523    569   RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
   524    570   !ENDIF
   525    571   
   526    572   # C compiler options for the Windows 10 platform (needs MSVC 2015).
   527    573   #
   528    574   !IF $(FOR_WIN10)!=0
   529         -TCC = $(TCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
   530         -BCC = $(BCC) /guard:cf -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          575  +TCC = $(TCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          576  +BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
   531    577   !ENDIF
   532    578   
   533    579   # Also, we need to dynamically link to the correct MSVC runtime
   534    580   # when compiling for WinRT (e.g. debug or release) OR if the
   535    581   # USE_CRT_DLL option is set to force dynamically linking to the
   536    582   # MSVC runtime library.
   537    583   #
   538         -!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0
          584  +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
   539    585   !IF $(DEBUG)>1
   540    586   TCC = $(TCC) -MDd
   541    587   BCC = $(BCC) -MDd
   542    588   !ELSE
   543    589   TCC = $(TCC) -MD
   544    590   BCC = $(BCC) -MD
   545    591   !ENDIF
................................................................................
   549    595   BCC = $(BCC) -MTd
   550    596   !ELSE
   551    597   TCC = $(TCC) -MT
   552    598   BCC = $(BCC) -MT
   553    599   !ENDIF
   554    600   !ENDIF
   555    601   
          602  +# <<mark>>
   556    603   # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in
   557    604   # any extension header files by default.  For non-amalgamation
   558    605   # builds, we need to make sure the compiler can find these.
   559    606   #
   560    607   !IF $(USE_AMALGAMATION)==0
   561    608   TCC = $(TCC) -I$(TOP)\ext\fts3
   562    609   RCC = $(RCC) -I$(TOP)\ext\fts3
   563    610   TCC = $(TCC) -I$(TOP)\ext\rtree
   564    611   RCC = $(RCC) -I$(TOP)\ext\rtree
          612  +TCC = $(TCC) -I$(TOP)\ext\session
          613  +RCC = $(RCC) -I$(TOP)\ext\session
   565    614   !ENDIF
   566    615   
   567    616   # The mksqlite3c.tcl script accepts some options on the command
   568    617   # line.  When compiling with debugging enabled, some of these
   569    618   # options are necessary in order to allow debugging symbols to
   570    619   # work correctly with Visual Studio when using the amalgamation.
   571    620   #
................................................................................
   572    621   !IFNDEF MKSQLITE3C_ARGS
   573    622   !IF $(DEBUG)>1
   574    623   MKSQLITE3C_ARGS = --linemacros
   575    624   !ELSE
   576    625   MKSQLITE3C_ARGS =
   577    626   !ENDIF
   578    627   !ENDIF
          628  +# <</mark>>
   579    629   
   580    630   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
   581    631   # Omitting the define will cause extra debugging code to be inserted and
   582    632   # includes extra comments when "EXPLAIN stmt" is used.
   583    633   #
   584    634   !IF $(DEBUG)==0
   585    635   TCC = $(TCC) -DNDEBUG
................................................................................
   636    686   #
   637    687   !IF $(DEBUG)>3
   638    688   TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
   639    689   RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
   640    690   !ENDIF
   641    691   !ENDIF
   642    692   
          693  +# <<mark>>
   643    694   # The locations of the Tcl header and library files.  Also, the library that
   644    695   # non-stubs enabled programs using Tcl must link against.  These variables
   645    696   # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
   646    697   # prior to running nmake in order to match the actual installed location and
   647    698   # version on this machine.
   648    699   #
   649    700   !IFNDEF TCLINCDIR
................................................................................
   687    738   # know the specific version we want to use.  This variable (TCLSH_CMD) may be
   688    739   # overridden via the environment prior to running nmake in order to select a
   689    740   # specific Tcl shell to use.
   690    741   #
   691    742   !IFNDEF TCLSH_CMD
   692    743   TCLSH_CMD = tclsh85
   693    744   !ENDIF
          745  +# <</mark>>
   694    746   
   695    747   # Compiler options needed for programs that use the readline() library.
   696    748   #
   697    749   !IFNDEF READLINE_FLAGS
   698    750   READLINE_FLAGS = -DHAVE_READLINE=0
   699    751   !ENDIF
   700    752   
................................................................................
   785    837   # If symbols are enabled (or compiling for debugging), enable PDBs.
   786    838   #
   787    839   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   788    840   TCC = $(TCC) -Zi
   789    841   BCC = $(BCC) -Zi
   790    842   !ENDIF
   791    843   
          844  +# <<mark>>
   792    845   # If ICU support is enabled, add the compiler options for it.
   793    846   #
   794    847   !IF $(USE_ICU)!=0
   795    848   TCC = $(TCC) -DSQLITE_ENABLE_ICU=1
   796    849   RCC = $(RCC) -DSQLITE_ENABLE_ICU=1
   797    850   TCC = $(TCC) -I$(TOP)\ext\icu
   798    851   RCC = $(RCC) -I$(TOP)\ext\icu
   799    852   TCC = $(TCC) -I$(ICUINCDIR)
   800    853   RCC = $(RCC) -I$(ICUINCDIR)
   801    854   !ENDIF
          855  +# <</mark>>
   802    856   
   803    857   # Command line prefixes for compiling code, compiling resources,
   804    858   # linking, etc.
   805    859   #
   806    860   LTCOMPILE = $(TCC) -Fo$@
   807    861   LTRCOMPILE = $(RCC) -r
   808    862   LTLIB = lib.exe
................................................................................
   872    926   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
   873    927   !ENDIF
   874    928   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
   875    929   LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
   876    930   LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
   877    931   !ENDIF
   878    932   
   879         -# When compiling for UAP, some extra linker options are also required.
          933  +# When compiling for UWP or the Windows 10 platform, some extra linker
          934  +# options are also required.
   880    935   #
   881         -!IF $(FOR_UAP)!=0
          936  +!IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0
   882    937   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
   883    938   LTLINKOPTS = $(LTLINKOPTS) mincore.lib
   884    939   !IFDEF PSDKLIBPATH
   885    940   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
   886    941   !ENDIF
   887    942   !ENDIF
          943  +
          944  +!IF $(FOR_WIN10)!=0
          945  +LTLINKOPTS = $(LTLINKOPTS) /guard:cf "/LIBPATH:$(UCRTLIBPATH)"
          946  +!IF $(DEBUG)>1
          947  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib
          948  +!ELSE
          949  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib
          950  +!ENDIF
          951  +!ENDIF
   888    952   
   889    953   # If either debugging or symbols are enabled, enable PDBs.
   890    954   #
   891    955   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   892    956   LDFLAGS = /DEBUG $(LDOPTS)
   893    957   !ELSE
   894    958   LDFLAGS = $(LDOPTS)
   895    959   !ENDIF
   896    960   
          961  +# <<mark>>
   897    962   # Start with the Tcl related linker options.
   898    963   #
   899    964   !IF $(NO_TCL)==0
   900    965   LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
   901    966   LTLIBS = $(LIBTCL)
   902    967   !ENDIF
   903    968   
   904    969   # If ICU support is enabled, add the linker options for it.
   905    970   #
   906    971   !IF $(USE_ICU)!=0
   907    972   LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR)
   908    973   LTLIBS = $(LTLIBS) $(LIBICU)
   909    974   !ENDIF
          975  +# <</mark>>
   910    976   
   911    977   # You should not have to change anything below this line
   912    978   ###############################################################################
   913    979   
          980  +# <<mark>>
   914    981   # Object files for the SQLite library (non-amalgamation).
   915    982   #
   916    983   LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
   917    984            backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
   918    985            callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
   919    986            expr.lo fault.lo fkey.lo \
   920    987            fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
   921    988            fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
   922    989            fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
   923    990            fts5.lo \
   924    991            func.lo global.lo hash.lo \
   925         -         icu.lo insert.lo journal.lo legacy.lo loadext.lo \
          992  +         icu.lo insert.lo legacy.lo loadext.lo \
   926    993            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   927    994            memjournal.lo \
   928    995            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   929    996            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   930    997            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   931         -         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
          998  +         random.lo resolve.lo rowset.lo rtree.lo \
          999  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
   932   1000            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   933   1001            update.lo util.lo vacuum.lo \
   934   1002            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   935   1003            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   936   1004            utf.lo vtab.lo
         1005  +# <</mark>>
   937   1006   
   938   1007   # Object files for the amalgamation.
   939   1008   #
   940   1009   LIBOBJS1 = sqlite3.lo
   941   1010   
   942   1011   # Determine the real value of LIBOBJ based on the 'configure' script
   943   1012   #
         1013  +# <<mark>>
   944   1014   !IF $(USE_AMALGAMATION)==0
   945   1015   LIBOBJ = $(LIBOBJS0)
   946   1016   !ELSE
         1017  +# <</mark>>
   947   1018   LIBOBJ = $(LIBOBJS1)
         1019  +# <<mark>>
   948   1020   !ENDIF
         1021  +# <</mark>>
   949   1022   
   950   1023   # Determine if embedded resource compilation and usage are enabled.
   951   1024   #
   952   1025   !IF $(USE_RC)!=0
   953   1026   LIBRESOBJS = sqlite3res.lo
   954   1027   !ELSE
   955   1028   LIBRESOBJS =
   956   1029   !ENDIF
   957   1030   
   958         -# All of the source code files.
         1031  +# <<mark>>
         1032  +# Core source code files, part 1.
   959   1033   #
   960         -SRC1 = \
         1034  +SRC00 = \
   961   1035     $(TOP)\src\alter.c \
   962   1036     $(TOP)\src\analyze.c \
   963   1037     $(TOP)\src\attach.c \
   964   1038     $(TOP)\src\auth.c \
   965   1039     $(TOP)\src\backup.c \
   966   1040     $(TOP)\src\bitvec.c \
   967   1041     $(TOP)\src\btmutex.c \
   968   1042     $(TOP)\src\btree.c \
   969         -  $(TOP)\src\btree.h \
   970         -  $(TOP)\src\btreeInt.h \
   971   1043     $(TOP)\src\build.c \
   972   1044     $(TOP)\src\callback.c \
   973   1045     $(TOP)\src\complete.c \
   974   1046     $(TOP)\src\ctime.c \
   975   1047     $(TOP)\src\date.c \
   976   1048     $(TOP)\src\dbstat.c \
   977   1049     $(TOP)\src\delete.c \
   978   1050     $(TOP)\src\expr.c \
   979   1051     $(TOP)\src\fault.c \
   980   1052     $(TOP)\src\fkey.c \
   981   1053     $(TOP)\src\func.c \
   982   1054     $(TOP)\src\global.c \
   983   1055     $(TOP)\src\hash.c \
   984         -  $(TOP)\src\hash.h \
   985         -  $(TOP)\src\hwtime.h \
   986   1056     $(TOP)\src\insert.c \
   987         -  $(TOP)\src\journal.c \
   988   1057     $(TOP)\src\legacy.c \
   989   1058     $(TOP)\src\loadext.c \
   990   1059     $(TOP)\src\main.c \
   991   1060     $(TOP)\src\malloc.c \
   992   1061     $(TOP)\src\mem0.c \
   993   1062     $(TOP)\src\mem1.c \
   994   1063     $(TOP)\src\mem2.c \
   995   1064     $(TOP)\src\mem3.c \
   996   1065     $(TOP)\src\mem5.c \
   997   1066     $(TOP)\src\memjournal.c \
   998         -  $(TOP)\src\msvc.h \
   999   1067     $(TOP)\src\mutex.c \
  1000         -  $(TOP)\src\mutex.h \
  1001   1068     $(TOP)\src\mutex_noop.c \
  1002   1069     $(TOP)\src\mutex_unix.c \
  1003   1070     $(TOP)\src\mutex_w32.c \
  1004   1071     $(TOP)\src\notify.c \
  1005   1072     $(TOP)\src\os.c \
  1006         -  $(TOP)\src\os.h \
  1007         -  $(TOP)\src\os_common.h \
  1008         -  $(TOP)\src\os_setup.h \
  1009   1073     $(TOP)\src\os_unix.c \
  1010         -  $(TOP)\src\os_win.c \
  1011         -  $(TOP)\src\os_win.h
  1012         -SRC2 = \
         1074  +  $(TOP)\src\os_win.c
         1075  +
         1076  +# Core source code files, part 2.
         1077  +#
         1078  +SRC01 = \
  1013   1079     $(TOP)\src\pager.c \
  1014         -  $(TOP)\src\pager.h \
  1015         -  $(TOP)\src\parse.y \
  1016   1080     $(TOP)\src\pcache.c \
  1017         -  $(TOP)\src\pcache.h \
  1018   1081     $(TOP)\src\pcache1.c \
  1019   1082     $(TOP)\src\pragma.c \
  1020         -  $(TOP)\src\pragma.h \
  1021   1083     $(TOP)\src\prepare.c \
  1022   1084     $(TOP)\src\printf.c \
  1023   1085     $(TOP)\src\random.c \
  1024   1086     $(TOP)\src\resolve.c \
  1025   1087     $(TOP)\src\rowset.c \
  1026   1088     $(TOP)\src\select.c \
  1027   1089     $(TOP)\src\status.c \
  1028         -  $(TOP)\src\shell.c \
  1029         -  $(TOP)\src\sqlite.h.in \
  1030         -  $(TOP)\src\sqlite3ext.h \
  1031         -  $(TOP)\src\sqliteInt.h \
  1032         -  $(TOP)\src\sqliteLimit.h \
  1033   1090     $(TOP)\src\table.c \
  1034   1091     $(TOP)\src\threads.c \
  1035   1092     $(TOP)\src\tclsqlite.c \
  1036   1093     $(TOP)\src\tokenize.c \
  1037   1094     $(TOP)\src\treeview.c \
  1038   1095     $(TOP)\src\trigger.c \
  1039   1096     $(TOP)\src\utf.c \
  1040   1097     $(TOP)\src\update.c \
  1041   1098     $(TOP)\src\util.c \
  1042   1099     $(TOP)\src\vacuum.c \
  1043   1100     $(TOP)\src\vdbe.c \
  1044         -  $(TOP)\src\vdbe.h \
  1045   1101     $(TOP)\src\vdbeapi.c \
  1046   1102     $(TOP)\src\vdbeaux.c \
  1047   1103     $(TOP)\src\vdbeblob.c \
  1048   1104     $(TOP)\src\vdbemem.c \
  1049   1105     $(TOP)\src\vdbesort.c \
  1050   1106     $(TOP)\src\vdbetrace.c \
  1051         -  $(TOP)\src\vdbeInt.h \
  1052   1107     $(TOP)\src\vtab.c \
  1053         -  $(TOP)\src\vxworks.h \
  1054   1108     $(TOP)\src\wal.c \
  1055         -  $(TOP)\src\wal.h \
  1056   1109     $(TOP)\src\walker.c \
  1057   1110     $(TOP)\src\where.c \
  1058   1111     $(TOP)\src\wherecode.c \
  1059         -  $(TOP)\src\whereexpr.c \
         1112  +  $(TOP)\src\whereexpr.c
         1113  +
         1114  +# Shell source code files.
         1115  +#
         1116  +SRC02 = \
         1117  +  $(TOP)\src\shell.c
         1118  +
         1119  +# Core miscellaneous files.
         1120  +#
         1121  +SRC03 = \
         1122  +  $(TOP)\src\parse.y
         1123  +
         1124  +# Core header files, part 1.
         1125  +#
         1126  +SRC04 = \
         1127  +  $(TOP)\src\btree.h \
         1128  +  $(TOP)\src\btreeInt.h \
         1129  +  $(TOP)\src\hash.h \
         1130  +  $(TOP)\src\hwtime.h \
         1131  +  $(TOP)\src\msvc.h \
         1132  +  $(TOP)\src\mutex.h \
         1133  +  $(TOP)\src\os.h \
         1134  +  $(TOP)\src\os_common.h \
         1135  +  $(TOP)\src\os_setup.h \
         1136  +  $(TOP)\src\os_win.h
         1137  +
         1138  +# Core header files, part 2.
         1139  +#
         1140  +SRC05 = \
         1141  +  $(TOP)\src\pager.h \
         1142  +  $(TOP)\src\pcache.h \
         1143  +  $(TOP)\src\pragma.h \
         1144  +  $(TOP)\src\sqlite.h.in \
         1145  +  $(TOP)\src\sqlite3ext.h \
         1146  +  $(TOP)\src\sqliteInt.h \
         1147  +  $(TOP)\src\sqliteLimit.h \
         1148  +  $(TOP)\src\vdbe.h \
         1149  +  $(TOP)\src\vdbeInt.h \
         1150  +  $(TOP)\src\vxworks.h \
         1151  +  $(TOP)\src\wal.h \
  1060   1152     $(TOP)\src\whereInt.h
  1061   1153   
  1062         -# Source code for extensions
         1154  +# Extension source code files, part 1.
  1063   1155   #
  1064         -SRC3 = \
         1156  +SRC06 = \
  1065   1157     $(TOP)\ext\fts1\fts1.c \
  1066         -  $(TOP)\ext\fts1\fts1.h \
  1067   1158     $(TOP)\ext\fts1\fts1_hash.c \
  1068         -  $(TOP)\ext\fts1\fts1_hash.h \
  1069   1159     $(TOP)\ext\fts1\fts1_porter.c \
  1070         -  $(TOP)\ext\fts1\fts1_tokenizer.h \
  1071   1160     $(TOP)\ext\fts1\fts1_tokenizer1.c \
  1072   1161     $(TOP)\ext\fts2\fts2.c \
  1073         -  $(TOP)\ext\fts2\fts2.h \
  1074   1162     $(TOP)\ext\fts2\fts2_hash.c \
  1075         -  $(TOP)\ext\fts2\fts2_hash.h \
  1076   1163     $(TOP)\ext\fts2\fts2_icu.c \
  1077   1164     $(TOP)\ext\fts2\fts2_porter.c \
  1078         -  $(TOP)\ext\fts2\fts2_tokenizer.h \
  1079   1165     $(TOP)\ext\fts2\fts2_tokenizer.c \
  1080   1166     $(TOP)\ext\fts2\fts2_tokenizer1.c
  1081         -SRC4 = \
         1167  +
         1168  +# Extension source code files, part 2.
         1169  +#
         1170  +SRC07 = \
  1082   1171     $(TOP)\ext\fts3\fts3.c \
  1083         -  $(TOP)\ext\fts3\fts3.h \
  1084         -  $(TOP)\ext\fts3\fts3Int.h \
  1085   1172     $(TOP)\ext\fts3\fts3_aux.c \
  1086   1173     $(TOP)\ext\fts3\fts3_expr.c \
  1087   1174     $(TOP)\ext\fts3\fts3_hash.c \
  1088         -  $(TOP)\ext\fts3\fts3_hash.h \
  1089   1175     $(TOP)\ext\fts3\fts3_icu.c \
  1090   1176     $(TOP)\ext\fts3\fts3_porter.c \
  1091   1177     $(TOP)\ext\fts3\fts3_snippet.c \
  1092         -  $(TOP)\ext\fts3\fts3_tokenizer.h \
  1093   1178     $(TOP)\ext\fts3\fts3_tokenizer.c \
  1094   1179     $(TOP)\ext\fts3\fts3_tokenizer1.c \
  1095   1180     $(TOP)\ext\fts3\fts3_tokenize_vtab.c \
  1096   1181     $(TOP)\ext\fts3\fts3_unicode.c \
  1097   1182     $(TOP)\ext\fts3\fts3_unicode2.c \
  1098   1183     $(TOP)\ext\fts3\fts3_write.c \
  1099         -  $(TOP)\ext\icu\sqliteicu.h \
  1100   1184     $(TOP)\ext\icu\icu.c \
  1101         -  $(TOP)\ext\rtree\rtree.h \
  1102   1185     $(TOP)\ext\rtree\rtree.c \
  1103         -  $(TOP)\ext\rbu\sqlite3rbu.h \
         1186  +  $(TOP)\ext\session\sqlite3session.c \
  1104   1187     $(TOP)\ext\rbu\sqlite3rbu.c \
  1105   1188     $(TOP)\ext\misc\json1.c
  1106   1189   
         1190  +# Extension header files, part 1.
         1191  +#
         1192  +SRC08 = \
         1193  +  $(TOP)\ext\fts1\fts1.h \
         1194  +  $(TOP)\ext\fts1\fts1_hash.h \
         1195  +  $(TOP)\ext\fts1\fts1_tokenizer.h \
         1196  +  $(TOP)\ext\fts2\fts2.h \
         1197  +  $(TOP)\ext\fts2\fts2_hash.h \
         1198  +  $(TOP)\ext\fts2\fts2_tokenizer.h
         1199  +
         1200  +# Extension header files, part 2.
         1201  +#
         1202  +SRC09 = \
         1203  +  $(TOP)\ext\fts3\fts3.h \
         1204  +  $(TOP)\ext\fts3\fts3Int.h \
         1205  +  $(TOP)\ext\fts3\fts3_hash.h \
         1206  +  $(TOP)\ext\fts3\fts3_tokenizer.h \
         1207  +  $(TOP)\ext\icu\sqliteicu.h \
         1208  +  $(TOP)\ext\rtree\rtree.h \
         1209  +  $(TOP)\ext\rbu\sqlite3rbu.h \
         1210  +  $(TOP)\ext\session\sqlite3session.h
  1107   1211   
  1108   1212   # Generated source code files
  1109   1213   #
  1110         -SRC5 = \
  1111         -  keywordhash.h \
         1214  +SRC10 = \
  1112   1215     opcodes.c \
         1216  +  parse.c
         1217  +
         1218  +# Generated header files
         1219  +#
         1220  +SRC11 = \
         1221  +  keywordhash.h \
  1113   1222     opcodes.h \
  1114         -  parse.c \
  1115   1223     parse.h \
  1116   1224     $(SQLITE3H)
  1117   1225   
  1118   1226   # All source code files.
  1119   1227   #
  1120         -SRC = $(SRC1) $(SRC2) $(SRC3) $(SRC4) $(SRC5)
         1228  +SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
  1121   1229   
  1122   1230   # Source code to the test files.
  1123   1231   #
  1124   1232   TESTSRC = \
  1125   1233     $(TOP)\src\test1.c \
  1126   1234     $(TOP)\src\test2.c \
  1127   1235     $(TOP)\src\test3.c \
................................................................................
  1130   1238     $(TOP)\src\test6.c \
  1131   1239     $(TOP)\src\test7.c \
  1132   1240     $(TOP)\src\test8.c \
  1133   1241     $(TOP)\src\test9.c \
  1134   1242     $(TOP)\src\test_autoext.c \
  1135   1243     $(TOP)\src\test_async.c \
  1136   1244     $(TOP)\src\test_backup.c \
         1245  +  $(TOP)\src\test_bestindex.c \
  1137   1246     $(TOP)\src\test_blob.c \
  1138   1247     $(TOP)\src\test_btree.c \
  1139   1248     $(TOP)\src\test_config.c \
  1140   1249     $(TOP)\src\test_demovfs.c \
  1141   1250     $(TOP)\src\test_devsym.c \
  1142   1251     $(TOP)\src\test_fs.c \
  1143   1252     $(TOP)\src\test_func.c \
................................................................................
  1160   1269     $(TOP)\src\test_tclvar.c \
  1161   1270     $(TOP)\src\test_thread.c \
  1162   1271     $(TOP)\src\test_vfs.c \
  1163   1272     $(TOP)\src\test_windirent.c \
  1164   1273     $(TOP)\src\test_wsd.c \
  1165   1274     $(TOP)\ext\fts3\fts3_term.c \
  1166   1275     $(TOP)\ext\fts3\fts3_test.c \
  1167         -  $(TOP)\ext\rbu\test_rbu.c
         1276  +  $(TOP)\ext\rbu\test_rbu.c \
         1277  +  $(TOP)\ext\session\test_session.c
  1168   1278   
  1169         -# Statically linked extensions
         1279  +# Statically linked extensions.
  1170   1280   #
  1171   1281   TESTEXT = \
  1172   1282     $(TOP)\ext\misc\amatch.c \
  1173   1283     $(TOP)\ext\misc\closure.c \
  1174   1284     $(TOP)\ext\misc\eval.c \
  1175   1285     $(TOP)\ext\misc\fileio.c \
  1176   1286     $(TOP)\ext\misc\fuzzer.c \
................................................................................
  1182   1292     $(TOP)\ext\misc\percentile.c \
  1183   1293     $(TOP)\ext\misc\regexp.c \
  1184   1294     $(TOP)\ext\misc\series.c \
  1185   1295     $(TOP)\ext\misc\spellfix.c \
  1186   1296     $(TOP)\ext\misc\totype.c \
  1187   1297     $(TOP)\ext\misc\wholenumber.c
  1188   1298   
  1189         -
  1190   1299   # Source code to the library files needed by the test fixture
         1300  +# (non-amalgamation)
  1191   1301   #
  1192   1302   TESTSRC2 = \
  1193         -  $(TOP)\src\attach.c \
  1194         -  $(TOP)\src\backup.c \
  1195         -  $(TOP)\src\bitvec.c \
  1196         -  $(TOP)\src\btree.c \
  1197         -  $(TOP)\src\build.c \
  1198         -  $(TOP)\src\ctime.c \
  1199         -  $(TOP)\src\date.c \
  1200         -  $(TOP)\src\dbstat.c \
  1201         -  $(TOP)\src\expr.c \
  1202         -  $(TOP)\src\func.c \
  1203         -  $(TOP)\src\insert.c \
  1204         -  $(TOP)\src\wal.c \
  1205         -  $(TOP)\src\main.c \
  1206         -  $(TOP)\src\mem5.c \
  1207         -  $(TOP)\src\os.c \
  1208         -  $(TOP)\src\os_unix.c \
  1209         -  $(TOP)\src\os_win.c \
  1210         -  $(TOP)\src\pager.c \
  1211         -  $(TOP)\src\pragma.c \
  1212         -  $(TOP)\src\prepare.c \
  1213         -  $(TOP)\src\printf.c \
  1214         -  $(TOP)\src\random.c \
  1215         -  $(TOP)\src\pcache.c \
  1216         -  $(TOP)\src\pcache1.c \
  1217         -  $(TOP)\src\select.c \
  1218         -  $(TOP)\src\tokenize.c \
  1219         -  $(TOP)\src\utf.c \
  1220         -  $(TOP)\src\util.c \
  1221         -  $(TOP)\src\vdbeapi.c \
  1222         -  $(TOP)\src\vdbeaux.c \
  1223         -  $(TOP)\src\vdbe.c \
  1224         -  $(TOP)\src\vdbemem.c \
  1225         -  $(TOP)\src\vdbesort.c \
  1226         -  $(TOP)\src\vdbetrace.c \
  1227         -  $(TOP)\src\where.c \
  1228         -  $(TOP)\src\wherecode.c \
  1229         -  $(TOP)\src\whereexpr.c \
  1230         -  parse.c \
  1231         -  $(TOP)\ext\fts3\fts3.c \
  1232         -  $(TOP)\ext\fts3\fts3_aux.c \
  1233         -  $(TOP)\ext\fts3\fts3_expr.c \
  1234         -  $(TOP)\ext\fts3\fts3_tokenizer.c \
  1235         -  $(TOP)\ext\fts3\fts3_tokenize_vtab.c \
  1236         -  $(TOP)\ext\fts3\fts3_unicode.c \
  1237         -  $(TOP)\ext\fts3\fts3_unicode2.c \
  1238         -  $(TOP)\ext\fts3\fts3_write.c \
         1303  +  $(SRC00) \
         1304  +  $(SRC01) \
         1305  +  $(SRC06) \
         1306  +  $(SRC07) \
         1307  +  $(SRC10) \
  1239   1308     $(TOP)\ext\async\sqlite3async.c
  1240   1309   
  1241   1310   # Header files used by all library source files.
  1242   1311   #
  1243   1312   HDR = \
  1244   1313      $(TOP)\src\btree.h \
  1245   1314      $(TOP)\src\btreeInt.h \
................................................................................
  1283   1352     $(TOP)\ext\fts3\fts3_tokenizer.h
  1284   1353   EXTHDR = $(EXTHDR) \
  1285   1354     $(TOP)\ext\rtree\rtree.h
  1286   1355   EXTHDR = $(EXTHDR) \
  1287   1356     $(TOP)\ext\icu\sqliteicu.h
  1288   1357   EXTHDR = $(EXTHDR) \
  1289   1358     $(TOP)\ext\rtree\sqlite3rtree.h
         1359  +EXTHDR = $(EXTHDR) \
         1360  +  $(TOP)\ext\session\sqlite3session.h
  1290   1361   
  1291   1362   # executables needed for testing
  1292   1363   #
  1293   1364   TESTPROGS = \
  1294   1365     testfixture.exe \
  1295   1366     $(SQLITE3EXE) \
  1296   1367     sqlite3_analyzer.exe \
................................................................................
  1299   1370   # Databases containing fuzzer test cases
  1300   1371   #
  1301   1372   FUZZDATA = \
  1302   1373     $(TOP)\test\fuzzdata1.db \
  1303   1374     $(TOP)\test\fuzzdata2.db \
  1304   1375     $(TOP)\test\fuzzdata3.db \
  1305   1376     $(TOP)\test\fuzzdata4.db
         1377  +# <</mark>>
  1306   1378   
  1307   1379   # Additional compiler options for the shell.  These are only effective
  1308   1380   # when the shell is not being dynamically linked.
  1309   1381   #
  1310   1382   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1311         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
         1383  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
  1312   1384   !ENDIF
  1313   1385   
         1386  +# <<mark>>
  1314   1387   # Extra compiler options for various test tools.
  1315   1388   #
  1316   1389   MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
  1317   1390   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
  1318   1391   FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
  1319   1392   
  1320   1393   # Standard options to testfixture.
  1321   1394   #
  1322   1395   TESTOPTS = --verbose=file --output=test-out.txt
         1396  +
         1397  +# Extra targets for the "all" target that require Tcl.
         1398  +#
         1399  +!IF $(NO_TCL)==0
         1400  +ALL_TCL_TARGETS = libtclsqlite3.lib
         1401  +!ELSE
         1402  +ALL_TCL_TARGETS =
         1403  +!ENDIF
         1404  +# <</mark>>
  1323   1405   
  1324   1406   # This is the default Makefile target.  The objects listed here
  1325   1407   # are what get build when you type just "make" with no arguments.
  1326   1408   #
  1327         -all:	dll libsqlite3.lib shell libtclsqlite3.lib
         1409  +all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
  1328   1410   
  1329   1411   # Dynamic link library section.
  1330   1412   #
  1331         -dll: $(SQLITE3DLL)
         1413  +dll:	$(SQLITE3DLL)
  1332   1414   
  1333   1415   # Shell executable.
  1334   1416   #
  1335         -shell: $(SQLITE3EXE)
         1417  +shell:	$(SQLITE3EXE)
  1336   1418   
         1419  +# <<mark>>
  1337   1420   libsqlite3.lib:	$(LIBOBJ)
  1338   1421   	$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)
  1339   1422   
  1340   1423   libtclsqlite3.lib:	tclsqlite.lo libsqlite3.lib
  1341   1424   	$(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS)
         1425  +# <</mark>>
  1342   1426   
  1343         -$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
         1427  +$(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
  1344   1428   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1345   1429   
  1346         -sqlite3.def: libsqlite3.lib
         1430  +# <<block2>>
         1431  +sqlite3.def:	libsqlite3.lib
  1347   1432   	echo EXPORTS > sqlite3.def
  1348   1433   	dumpbin /all libsqlite3.lib \
  1349         -		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \
         1434  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \
  1350   1435   		| sort >> sqlite3.def
         1436  +# <</block2>>
  1351   1437   
  1352   1438   $(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
  1353   1439   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
  1354   1440   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1355   1441   
         1442  +# <<mark>>
  1356   1443   sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
  1357   1444   	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1445  +
         1446  +srcck1.exe:	$(TOP)\tool\srcck1.c
         1447  +	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
         1448  +
         1449  +sourcetest:	srcck1.exe sqlite3.c
         1450  +	srcck1.exe sqlite3.c
  1358   1451   
  1359   1452   fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
  1360   1453   	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1361   1454   
  1362   1455   fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
  1363   1456   	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1364   1457   
................................................................................
  1384   1477   # build on the target system.  Some of the C source code and header
  1385   1478   # files are automatically generated.  This target takes care of
  1386   1479   # all that automatic generation.
  1387   1480   #
  1388   1481   .target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c
  1389   1482   	-rmdir /Q/S tsrc 2>NUL
  1390   1483   	-mkdir tsrc
  1391         -	for %i in ($(SRC1)) do copy /Y %i tsrc
  1392         -	for %i in ($(SRC2)) do copy /Y %i tsrc
  1393         -	for %i in ($(SRC3)) do copy /Y %i tsrc
  1394         -	for %i in ($(SRC4)) do copy /Y %i tsrc
  1395         -	for %i in ($(SRC5)) do copy /Y %i tsrc
         1484  +	for %i in ($(SRC00)) do copy /Y %i tsrc
         1485  +	for %i in ($(SRC01)) do copy /Y %i tsrc
         1486  +	for %i in ($(SRC02)) do copy /Y %i tsrc
         1487  +	for %i in ($(SRC03)) do copy /Y %i tsrc
         1488  +	for %i in ($(SRC04)) do copy /Y %i tsrc
         1489  +	for %i in ($(SRC05)) do copy /Y %i tsrc
         1490  +	for %i in ($(SRC06)) do copy /Y %i tsrc
         1491  +	for %i in ($(SRC07)) do copy /Y %i tsrc
         1492  +	for %i in ($(SRC08)) do copy /Y %i tsrc
         1493  +	for %i in ($(SRC09)) do copy /Y %i tsrc
         1494  +	for %i in ($(SRC10)) do copy /Y %i tsrc
         1495  +	for %i in ($(SRC11)) do copy /Y %i tsrc
  1396   1496   	copy /Y fts5.c tsrc
  1397   1497   	copy /Y fts5.h tsrc
  1398   1498   	del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL
  1399   1499   	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new
  1400   1500   	move vdbe.new tsrc\vdbe.c
  1401   1501   	echo > .target_source
  1402   1502   
  1403   1503   sqlite3.c:	.target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl
  1404   1504   	$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS)
  1405   1505   	copy tsrc\shell.c .
         1506  +	copy $(TOP)\ext\session\sqlite3session.h .
  1406   1507   
  1407   1508   sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
  1408   1509   	$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
         1510  +# <</mark>>
  1409   1511   
  1410   1512   # Rule to build the amalgamation
  1411   1513   #
  1412   1514   sqlite3.lo:	$(SQLITE3C)
  1413   1515   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)
  1414   1516   
         1517  +# <<mark>>
  1415   1518   # Rules to build the LEMON compiler generator
  1416   1519   #
  1417   1520   lempar.c:	$(TOP)\tool\lempar.c
  1418   1521   	copy $(TOP)\tool\lempar.c .
  1419   1522   
  1420   1523   lemon.exe:	$(TOP)\tool\lemon.c lempar.c
  1421   1524   	$(BCC) $(NO_WARN) -Daccess=_access \
................................................................................
  1428   1531   #     opcodes.lo
  1429   1532   #
  1430   1533   parse.lo:	parse.c $(HDR)
  1431   1534   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c parse.c
  1432   1535   
  1433   1536   opcodes.lo:	opcodes.c
  1434   1537   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c opcodes.c
         1538  +# <</mark>>
  1435   1539   
  1436   1540   # Rule to build the Win32 resources object file.
  1437   1541   #
  1438   1542   !IF $(USE_RC)!=0
         1543  +# <<block1>>
  1439   1544   $(LIBRESOBJS):	$(TOP)\src\sqlite3.rc $(SQLITE3H)
  1440   1545   	echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h
  1441   1546   	for /F %%V in ('type "$(TOP)\VERSION"') do ( \
  1442   1547   		echo #define SQLITE_RESOURCE_VERSION %%V \
  1443   1548   			| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \
  1444   1549   	)
  1445   1550   	echo #endif >> sqlite3rc.h
  1446   1551   	$(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc
         1552  +# <</block1>>
  1447   1553   !ENDIF
  1448   1554   
         1555  +# <<mark>>
  1449   1556   # Rules to build individual *.lo files from files in the src directory.
  1450   1557   #
  1451   1558   alter.lo:	$(TOP)\src\alter.c $(HDR)
  1452   1559   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\alter.c
  1453   1560   
  1454   1561   analyze.lo:	$(TOP)\src\analyze.c $(HDR)
  1455   1562   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\analyze.c
................................................................................
  1510   1617   
  1511   1618   hash.lo:	$(TOP)\src\hash.c $(HDR)
  1512   1619   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\hash.c
  1513   1620   
  1514   1621   insert.lo:	$(TOP)\src\insert.c $(HDR)
  1515   1622   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\insert.c
  1516   1623   
  1517         -journal.lo:	$(TOP)\src\journal.c $(HDR)
  1518         -	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\journal.c
  1519         -
  1520   1624   legacy.lo:	$(TOP)\src\legacy.c $(HDR)
  1521   1625   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c
  1522   1626   
  1523   1627   loadext.lo:	$(TOP)\src\loadext.c $(HDR)
  1524   1628   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\loadext.c
  1525   1629   
  1526   1630   main.lo:	$(TOP)\src\main.c $(HDR)
................................................................................
  1697   1801   	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y
  1698   1802   	move parse.h parse.h.temp
  1699   1803   	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h
  1700   1804   
  1701   1805   $(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
  1702   1806   	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H)
  1703   1807   
  1704         -sqlite3ext.h: .target_source
         1808  +sqlite3ext.h:	.target_source
  1705   1809   	copy tsrc\sqlite3ext.h .
  1706   1810   
  1707   1811   mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
  1708   1812   	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \
  1709   1813   		$(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
  1710   1814   
  1711   1815   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
................................................................................
  1774   1878   
  1775   1879   fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
  1776   1880   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
  1777   1881   
  1778   1882   rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
  1779   1883   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
  1780   1884   
         1885  +sqlite3session.lo:	$(TOP)\ext\session\sqlite3sesion.c $(HDR) $(EXTHDR)
         1886  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c
         1887  +
  1781   1888   # FTS5 things
  1782   1889   #
  1783   1890   FTS5_SRC = \
  1784   1891      $(TOP)\ext\fts5\fts5.h \
  1785   1892      $(TOP)\ext\fts5\fts5Int.h \
  1786   1893      $(TOP)\ext\fts5\fts5_aux.c \
  1787   1894      $(TOP)\ext\fts5\fts5_buffer.c \
................................................................................
  1798   1905      $(TOP)\ext\fts5\fts5_vocab.c
  1799   1906   
  1800   1907   fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
  1801   1908   	copy $(TOP)\ext\fts5\fts5parse.y .
  1802   1909   	del /Q fts5parse.h 2>NUL
  1803   1910   	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y
  1804   1911   
  1805         -fts5parse.h: fts5parse.c
         1912  +fts5parse.h:	fts5parse.c
  1806   1913   
  1807         -fts5.c: $(FTS5_SRC)
         1914  +fts5.c:	$(FTS5_SRC)
  1808   1915   	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
  1809   1916   	copy $(TOP)\ext\fts5\fts5.h .
  1810   1917   
  1811   1918   fts5.lo:	fts5.c $(HDR) $(EXTHDR)
  1812   1919   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
  1813   1920   
  1814   1921   fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
................................................................................
  1826   1933   # fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
  1827   1934   # necessary because the test fixture requires non-API symbols which are
  1828   1935   # hidden when the library is built via the amalgamation).
  1829   1936   #
  1830   1937   TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1831   1938   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  1832   1939   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
         1940  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
         1941  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
  1833   1942   
  1834         -TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) $(SHELL_CORE_DEP)
         1943  +TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  1835   1944   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  1836   1945   !IF $(USE_AMALGAMATION)==0
  1837   1946   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  1838   1947   !ELSE
  1839   1948   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)
  1840   1949   !ENDIF
  1841   1950   
  1842   1951   testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR)
  1843   1952   	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
  1844   1953   		-DBUILD_sqlite -I$(TCLINCDIR) \
  1845   1954   		$(TESTFIXTURE_SRC) \
  1846   1955   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1847   1956   
  1848         -extensiontest: testfixture.exe testloadext.dll
         1957  +extensiontest:	testfixture.exe testloadext.dll
  1849   1958   	@set PATH=$(LIBTCLPATH);$(PATH)
  1850   1959   	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
  1851   1960   
  1852   1961   fulltest:	$(TESTPROGS) fuzztest
  1853   1962   	@set PATH=$(LIBTCLPATH);$(PATH)
  1854   1963   	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
  1855   1964   
................................................................................
  1869   1978   	.\fuzzcheck.exe $(FUZZDATA)
  1870   1979   
  1871   1980   fastfuzztest:	fuzzcheck.exe
  1872   1981   	.\fuzzcheck.exe --limit-mem 100M $(FUZZDATA)
  1873   1982   
  1874   1983   # Minimal testing that runs in less than 3 minutes (on a fast machine)
  1875   1984   #
  1876         -quicktest:	testfixture.exe
         1985  +quicktest:	testfixture.exe sourcetest
  1877   1986   	@set PATH=$(LIBTCLPATH);$(PATH)
  1878   1987   	.\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS)
  1879   1988   
  1880   1989   # This is the common case.  Run many tests that do not take too long,
  1881   1990   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1882   1991   #
  1883         -test:	$(TESTPROGS) fastfuzztest
         1992  +test:	$(TESTPROGS) sourcetest fastfuzztest
  1884   1993   	@set PATH=$(LIBTCLPATH);$(PATH)
  1885   1994   	.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)
  1886   1995   
  1887   1996   smoketest:	$(TESTPROGS)
  1888   1997   	@set PATH=$(LIBTCLPATH);$(PATH)
  1889   1998   	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
  1890   1999   
  1891         -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
         2000  +sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
  1892   2001   	echo #define TCLSH 2 > $@
  1893   2002   	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
  1894   2003   	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
  1895   2004   	echo static const char *tclsh_main_loop(void){ >> $@
  1896   2005   	echo static const char *zMainloop = >> $@
  1897   2006   	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
  1898   2007   	echo ; return zMainloop; } >> $@
................................................................................
  1900   2009   sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
  1901   2010   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
  1902   2011   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1903   2012   
  1904   2013   testloadext.lo:	$(TOP)\src\test_loadext.c
  1905   2014   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
  1906   2015   
  1907         -testloadext.dll: testloadext.lo
         2016  +testloadext.dll:	testloadext.lo
  1908   2017   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  1909   2018   
  1910   2019   showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
  1911   2020   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1912   2021   		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1913   2022   
  1914   2023   showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
................................................................................
  1919   2028   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1920   2029   		$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1921   2030   
  1922   2031   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  1923   2032   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1924   2033   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1925   2034   
         2035  +changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C)
         2036  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2037  +		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2038  +
  1926   2039   fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
  1927   2040   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1928   2041   		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1929   2042   
  1930   2043   rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
  1931   2044   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1932   2045   		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
................................................................................
  1938   2051   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1939   2052   		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1940   2053   
  1941   2054   speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H)
  1942   2055   	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1943   2056   		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1944   2057   
  1945         -rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
         2058  +rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
  1946   2059   	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \
  1947   2060   		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1948   2061   
         2062  +moreclean:	clean
         2063  +	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
         2064  +# <</mark>>
         2065  +
  1949   2066   clean:
  1950   2067   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  1951         -	del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
  1952         -	del /Q $(SQLITE3C) $(SQLITE3H) opcodes.c opcodes.h 2>NUL
         2068  +	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
         2069  +	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
         2070  +# <<mark>>
         2071  +	del /Q sqlite3.c sqlite3.h 2>NUL
         2072  +	del /Q opcodes.c opcodes.h 2>NUL
  1953   2073   	del /Q lemon.* lempar.c parse.* 2>NUL
  1954   2074   	del /Q mkkeywordhash.* keywordhash.h 2>NUL
  1955   2075   	del /Q notasharedlib.* 2>NUL
  1956   2076   	-rmdir /Q/S .deps 2>NUL
  1957   2077   	-rmdir /Q/S .libs 2>NUL
  1958   2078   	-rmdir /Q/S quota2a 2>NUL
  1959   2079   	-rmdir /Q/S quota2b 2>NUL
................................................................................
  1960   2080   	-rmdir /Q/S quota2c 2>NUL
  1961   2081   	-rmdir /Q/S tsrc 2>NUL
  1962   2082   	del /Q .target_source 2>NUL
  1963   2083   	del /Q tclsqlite3.exe 2>NUL
  1964   2084   	del /Q testloadext.dll 2>NUL
  1965   2085   	del /Q testfixture.exe test.db 2>NUL
  1966   2086   	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
         2087  +	del /Q changeset.exe 2>NUL
  1967   2088   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  1968         -	del /Q mptester.exe wordcount.exe rbu.exe 2>NUL
  1969         -	del /Q $(SQLITE3EXE) $(SQLITE3DLL) sqlite3.def 2>NUL
         2089  +	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  1970   2090   	del /Q sqlite3.c sqlite3-*.c 2>NUL
  1971   2091   	del /Q sqlite3rc.h 2>NUL
  1972         -	del /Q shell.c sqlite3ext.h 2>NUL
         2092  +	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
  1973   2093   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  1974   2094   	del /Q sqlite-*-output.vsix 2>NUL
  1975   2095   	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
  1976   2096   	del /Q fts5.* fts5parse.* 2>NUL
         2097  +# <</mark>>

Changes to VERSION.

     1         -3.11.0
            1  +3.13.0

Changes to autoconf/Makefile.am.

     6      6   libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
     7      7   
     8      8   bin_PROGRAMS = sqlite3
     9      9   sqlite3_SOURCES = shell.c sqlite3.h
    10     10   EXTRA_sqlite3_SOURCES = sqlite3.c
    11     11   sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
    12     12   sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
    13         -sqlite3_CFLAGS = $(AM_CFLAGS)
           13  +sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
    14     14   
    15     15   include_HEADERS = sqlite3.h sqlite3ext.h
    16     16   
    17         -EXTRA_DIST = sqlite3.1 tea
           17  +EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
    18     18   pkgconfigdir = ${libdir}/pkgconfig
    19     19   pkgconfig_DATA = sqlite3.pc
    20     20   
    21     21   man_MANS = sqlite3.1

Added autoconf/Makefile.msc.

            1  +#### DO NOT EDIT ####
            2  +# This makefile is automatically generated from the Makefile.msc at
            3  +# the root of the canonical SQLite source tree (not the
            4  +# amalgamation tarball) using the tool/mkmsvcmin.tcl
            5  +# script.
            6  +#
            7  +
            8  +#
            9  +# nmake Makefile for SQLite
           10  +#
           11  +###############################################################################
           12  +############################## START OF OPTIONS ###############################
           13  +###############################################################################
           14  +
           15  +# The toplevel directory of the source tree.  This is the directory
           16  +# that contains this "Makefile.msc".
           17  +#
           18  +TOP = .
           19  +
           20  +
           21  +# Set this non-0 to enable full warnings (-W4, etc) when compiling.
           22  +#
           23  +!IFNDEF USE_FULLWARN
           24  +USE_FULLWARN = 0
           25  +!ENDIF
           26  +
           27  +# Set this non-0 to use "stdcall" calling convention for the core library
           28  +# and shell executable.
           29  +#
           30  +!IFNDEF USE_STDCALL
           31  +USE_STDCALL = 0
           32  +!ENDIF
           33  +
           34  +# Set this non-0 to have the shell executable link against the core dynamic
           35  +# link library.
           36  +#
           37  +!IFNDEF DYNAMIC_SHELL
           38  +DYNAMIC_SHELL = 0
           39  +!ENDIF
           40  +
           41  +# Set this non-0 to enable extra code that attempts to detect misuse of the
           42  +# SQLite API.
           43  +#
           44  +!IFNDEF API_ARMOR
           45  +API_ARMOR = 0
           46  +!ENDIF
           47  +
           48  +# If necessary, create a list of harmless compiler warnings to disable when
           49  +# compiling the various tools.  For the SQLite source code itself, warnings,
           50  +# if any, will be disabled from within it.
           51  +#
           52  +!IFNDEF NO_WARN
           53  +!IF $(USE_FULLWARN)!=0
           54  +NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206
           55  +NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706
           56  +!ENDIF
           57  +!ENDIF
           58  +
           59  +# Set this non-0 to use the library paths and other options necessary for
           60  +# Windows Phone 8.1.
           61  +#
           62  +!IFNDEF USE_WP81_OPTS
           63  +USE_WP81_OPTS = 0
           64  +!ENDIF
           65  +
           66  +# Set this non-0 to split the SQLite amalgamation file into chunks to
           67  +# be used for debugging with Visual Studio.
           68  +#
           69  +!IFNDEF SPLIT_AMALGAMATION
           70  +SPLIT_AMALGAMATION = 0
           71  +!ENDIF
           72  +
           73  +
           74  +# Set this non-0 to dynamically link to the MSVC runtime library.
           75  +#
           76  +!IFNDEF USE_CRT_DLL
           77  +USE_CRT_DLL = 0
           78  +!ENDIF
           79  +
           80  +# Set this non-0 to link to the RPCRT4 library.
           81  +#
           82  +!IFNDEF USE_RPCRT4_LIB
           83  +USE_RPCRT4_LIB = 0
           84  +!ENDIF
           85  +
           86  +# Set this non-0 to generate assembly code listings for the source code
           87  +# files.
           88  +#
           89  +!IFNDEF USE_LISTINGS
           90  +USE_LISTINGS = 0
           91  +!ENDIF
           92  +
           93  +# Set this non-0 to attempt setting the native compiler automatically
           94  +# for cross-compiling the command line tools needed during the compilation
           95  +# process.
           96  +#
           97  +!IFNDEF XCOMPILE
           98  +XCOMPILE = 0
           99  +!ENDIF
          100  +
          101  +# Set this non-0 to use the native libraries paths for cross-compiling
          102  +# the command line tools needed during the compilation process.
          103  +#
          104  +!IFNDEF USE_NATIVE_LIBPATHS
          105  +USE_NATIVE_LIBPATHS = 0
          106  +!ENDIF
          107  +
          108  +# Set this 0 to skip the compiling and embedding of version resources.
          109  +#
          110  +!IFNDEF USE_RC
          111  +USE_RC = 1
          112  +!ENDIF
          113  +
          114  +# Set this non-0 to compile binaries suitable for the WinRT environment.
          115  +# This setting does not apply to any binaries that require Tcl to operate
          116  +# properly (i.e. the text fixture, etc).
          117  +#
          118  +!IFNDEF FOR_WINRT
          119  +FOR_WINRT = 0
          120  +!ENDIF
          121  +
          122  +# Set this non-0 to compile binaries suitable for the UWP environment.
          123  +# This setting does not apply to any binaries that require Tcl to operate
          124  +# properly (i.e. the text fixture, etc).
          125  +#
          126  +!IFNDEF FOR_UWP
          127  +FOR_UWP = 0
          128  +!ENDIF
          129  +
          130  +# Set this non-0 to compile binaries suitable for the Windows 10 platform.
          131  +#
          132  +!IFNDEF FOR_WIN10
          133  +FOR_WIN10 = 0
          134  +!ENDIF
          135  +
          136  +
          137  +# Set this to non-0 to create and use PDBs.
          138  +#
          139  +!IFNDEF SYMBOLS
          140  +SYMBOLS = 1
          141  +!ENDIF
          142  +
          143  +# Set this to non-0 to use the SQLite debugging heap subsystem.
          144  +#
          145  +!IFNDEF MEMDEBUG
          146  +MEMDEBUG = 0
          147  +!ENDIF
          148  +
          149  +# Set this to non-0 to use the Win32 native heap subsystem.
          150  +#
          151  +!IFNDEF WIN32HEAP
          152  +WIN32HEAP = 0
          153  +!ENDIF
          154  +
          155  +# Set this to non-0 to enable OSTRACE() macros, which can be useful when
          156  +# debugging.
          157  +#
          158  +!IFNDEF OSTRACE
          159  +OSTRACE = 0
          160  +!ENDIF
          161  +
          162  +# Set this to one of the following values to enable various debugging
          163  +# features.  Each level includes the debugging options from the previous
          164  +# levels.  Currently, the recognized values for DEBUG are:
          165  +#
          166  +# 0 == NDEBUG: Disables assert() and other runtime diagnostics.
          167  +# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API.
          168  +# 2 == Disables NDEBUG and all optimizations and then enables PDBs.
          169  +# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code.
          170  +# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call.
          171  +# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros.
          172  +# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros.
          173  +#
          174  +!IFNDEF DEBUG
          175  +DEBUG = 0
          176  +!ENDIF
          177  +
          178  +# Enable use of available compiler optimizations?  Normally, this should be
          179  +# non-zero.  Setting this to zero, thus disabling all compiler optimizations,
          180  +# can be useful for testing.
          181  +#
          182  +!IFNDEF OPTIMIZATIONS
          183  +OPTIMIZATIONS = 2
          184  +!ENDIF
          185  +
          186  +# Set the source code file to be used by executables and libraries when
          187  +# they need the amalgamation.
          188  +#
          189  +!IFNDEF SQLITE3C
          190  +!IF $(SPLIT_AMALGAMATION)!=0
          191  +SQLITE3C = sqlite3-all.c
          192  +!ELSE
          193  +SQLITE3C = sqlite3.c
          194  +!ENDIF
          195  +!ENDIF
          196  +
          197  +# Set the include code file to be used by executables and libraries when
          198  +# they need SQLite.
          199  +#
          200  +!IFNDEF SQLITE3H
          201  +SQLITE3H = sqlite3.h
          202  +!ENDIF
          203  +
          204  +# This is the name to use for the SQLite dynamic link library (DLL).
          205  +#
          206  +!IFNDEF SQLITE3DLL
          207  +!IF $(FOR_WIN10)!=0
          208  +SQLITE3DLL = winsqlite3.dll
          209  +!ELSE
          210  +SQLITE3DLL = sqlite3.dll
          211  +!ENDIF
          212  +!ENDIF
          213  +
          214  +# This is the name to use for the SQLite import library (LIB).
          215  +#
          216  +!IFNDEF SQLITE3LIB
          217  +!IF $(FOR_WIN10)!=0
          218  +SQLITE3LIB = winsqlite3.lib
          219  +!ELSE
          220  +SQLITE3LIB = sqlite3.lib
          221  +!ENDIF
          222  +!ENDIF
          223  +
          224  +# This is the name to use for the SQLite shell executable (EXE).
          225  +#
          226  +!IFNDEF SQLITE3EXE
          227  +!IF $(FOR_WIN10)!=0
          228  +SQLITE3EXE = winsqlite3shell.exe
          229  +!ELSE
          230  +SQLITE3EXE = sqlite3.exe
          231  +!ENDIF
          232  +!ENDIF
          233  +
          234  +# This is the argument used to set the program database (PDB) file for the
          235  +# SQLite shell executable (EXE).
          236  +#
          237  +!IFNDEF SQLITE3EXEPDB
          238  +!IF $(FOR_WIN10)!=0
          239  +SQLITE3EXEPDB =
          240  +!ELSE
          241  +SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
          242  +!ENDIF
          243  +!ENDIF
          244  +
          245  +# These are the "standard" SQLite compilation options used when compiling for
          246  +# the Windows platform.
          247  +#
          248  +!IFNDEF OPT_FEATURE_FLAGS
          249  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
          250  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
          251  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
          252  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
          253  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
          254  +!ENDIF
          255  +
          256  +# These are the "extended" SQLite compilation options used when compiling for
          257  +# the Windows 10 platform.
          258  +#
          259  +!IFNDEF EXT_FEATURE_FLAGS
          260  +!IF $(FOR_WIN10)!=0
          261  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
          262  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1
          263  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1
          264  +!ELSE
          265  +EXT_FEATURE_FLAGS =
          266  +!ENDIF
          267  +!ENDIF
          268  +
          269  +###############################################################################
          270  +############################### END OF OPTIONS ################################
          271  +###############################################################################
          272  +
          273  +# When compiling for the Windows 10 platform, the PLATFORM macro must be set
          274  +# to an appropriate value (e.g. x86, x64, arm, arm64, etc).
          275  +#
          276  +!IF $(FOR_WIN10)!=0
          277  +!IFNDEF PLATFORM
          278  +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM.
          279  +!ENDIF
          280  +!ENDIF
          281  +
          282  +# This assumes that MSVC is always installed in 32-bit Program Files directory
          283  +# and sets the variable for use in locating other 32-bit installs accordingly.
          284  +#
          285  +PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
          286  +PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
          287  +
          288  +# Check for the predefined command macro CC.  This should point to the compiler
          289  +# binary for the target platform.  If it is not defined, simply define it to
          290  +# the legacy default value 'cl.exe'.
          291  +#
          292  +!IFNDEF CC
          293  +CC = cl.exe
          294  +!ENDIF
          295  +
          296  +# Check for the predefined command macro CSC.  This should point to a working
          297  +# C Sharp compiler binary.  If it is not defined, simply define it to the
          298  +# legacy default value 'csc.exe'.
          299  +#
          300  +!IFNDEF CSC
          301  +CSC = csc.exe
          302  +!ENDIF
          303  +
          304  +# Check for the command macro LD.  This should point to the linker binary for
          305  +# the target platform.  If it is not defined, simply define it to the legacy
          306  +# default value 'link.exe'.
          307  +#
          308  +!IFNDEF LD
          309  +LD = link.exe
          310  +!ENDIF
          311  +
          312  +# Check for the predefined command macro RC.  This should point to the resource
          313  +# compiler binary for the target platform.  If it is not defined, simply define
          314  +# it to the legacy default value 'rc.exe'.
          315  +#
          316  +!IFNDEF RC
          317  +RC = rc.exe
          318  +!ENDIF
          319  +
          320  +# Check for the MSVC runtime library path macro.  Otherwise, this value will
          321  +# default to the 'lib' directory underneath the MSVC installation directory.
          322  +#
          323  +!IFNDEF CRTLIBPATH
          324  +CRTLIBPATH = $(VCINSTALLDIR)\lib
          325  +!ENDIF
          326  +
          327  +CRTLIBPATH = $(CRTLIBPATH:\\=\)
          328  +
          329  +# Check for the command macro NCC.  This should point to the compiler binary
          330  +# for the platform the compilation process is taking place on.  If it is not
          331  +# defined, simply define it to have the same value as the CC macro.  When
          332  +# cross-compiling, it is suggested that this macro be modified via the command
          333  +# line (since nmake itself does not provide a built-in method to guess it).
          334  +# For example, to use the x86 compiler when cross-compiling for x64, a command
          335  +# line similar to the following could be used (all on one line):
          336  +#
          337  +#     nmake /f Makefile.msc sqlite3.dll
          338  +#           XCOMPILE=1 USE_NATIVE_LIBPATHS=1
          339  +#
          340  +# Alternatively, the full path and file name to the compiler binary for the
          341  +# platform the compilation process is taking place may be specified (all on
          342  +# one line):
          343  +#
          344  +#     nmake /f Makefile.msc sqlite3.dll
          345  +#           "NCC=""%VCINSTALLDIR%\bin\cl.exe"""
          346  +#           USE_NATIVE_LIBPATHS=1
          347  +#
          348  +!IFDEF NCC
          349  +NCC = $(NCC:\\=\)
          350  +!ELSEIF $(XCOMPILE)!=0
          351  +NCC = "$(VCINSTALLDIR)\bin\$(CC)"
          352  +NCC = $(NCC:\\=\)
          353  +!ELSE
          354  +NCC = $(CC)
          355  +!ENDIF
          356  +
          357  +# Check for the MSVC native runtime library path macro.  Otherwise,
          358  +# this value will default to the 'lib' directory underneath the MSVC
          359  +# installation directory.
          360  +#
          361  +!IFNDEF NCRTLIBPATH
          362  +NCRTLIBPATH = $(VCINSTALLDIR)\lib
          363  +!ENDIF
          364  +
          365  +NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
          366  +
          367  +# Check for the Platform SDK library path macro.  Otherwise, this
          368  +# value will default to the 'lib' directory underneath the Windows
          369  +# SDK installation directory (the environment variable used appears
          370  +# to be available when using Visual C++ 2008 or later via the
          371  +# command line).
          372  +#
          373  +!IFNDEF NSDKLIBPATH
          374  +NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
          375  +!ENDIF
          376  +
          377  +NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
          378  +
          379  +# Check for the UCRT library path macro.  Otherwise, this value will
          380  +# default to the version-specific, platform-specific 'lib' directory
          381  +# underneath the Windows SDK installation directory.
          382  +#
          383  +!IFNDEF UCRTLIBPATH
          384  +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM)
          385  +!ENDIF
          386  +
          387  +UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
          388  +
          389  +# C compiler and options for use in building executables that
          390  +# will run on the platform that is doing the build.
          391  +#
          392  +!IF $(USE_FULLWARN)!=0
          393  +BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
          394  +!ELSE
          395  +BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS)
          396  +!ENDIF
          397  +
          398  +# Check if assembly code listings should be generated for the source
          399  +# code files to be compiled.
          400  +#
          401  +!IF $(USE_LISTINGS)!=0
          402  +BCC = $(BCC) -FAcs
          403  +!ENDIF
          404  +
          405  +# Check if the native library paths should be used when compiling
          406  +# the command line tools used during the compilation process.  If
          407  +# so, set the necessary macro now.
          408  +#
          409  +!IF $(USE_NATIVE_LIBPATHS)!=0
          410  +NLTLIBPATHS = "/LIBPATH:$(NCRTLIBPATH)" "/LIBPATH:$(NSDKLIBPATH)"
          411  +
          412  +!IFDEF NUCRTLIBPATH
          413  +NUCRTLIBPATH = $(NUCRTLIBPATH:\\=\)
          414  +NLTLIBPATHS = $(NLTLIBPATHS) "/LIBPATH:$(NUCRTLIBPATH)"
          415  +!ENDIF
          416  +!ENDIF
          417  +
          418  +# C compiler and options for use in building executables that
          419  +# will run on the target platform.  (BCC and TCC are usually the
          420  +# same unless your are cross-compiling.)
          421  +#
          422  +!IF $(USE_FULLWARN)!=0
          423  +TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
          424  +!ELSE
          425  +TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
          426  +!ENDIF
          427  +
          428  +TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
          429  +RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
          430  +
          431  +# Check if we want to use the "stdcall" calling convention when compiling.
          432  +# This is not supported by the compilers for non-x86 platforms.  It should
          433  +# also be noted here that building any target with these "stdcall" options
          434  +# will most likely fail if the Tcl library is also required.  This is due
          435  +# to how the Tcl library functions are declared and exported (i.e. without
          436  +# an explicit calling convention, which results in "cdecl").
          437  +#
          438  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
          439  +!IF "$(PLATFORM)"=="x86"
          440  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          441  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          442  +!ELSE
          443  +!IFNDEF PLATFORM
          444  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          445  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          446  +!ELSE
          447  +CORE_CCONV_OPTS =
          448  +SHELL_CCONV_OPTS =
          449  +!ENDIF
          450  +!ENDIF
          451  +!ELSE
          452  +CORE_CCONV_OPTS =
          453  +SHELL_CCONV_OPTS =
          454  +!ENDIF
          455  +
          456  +# These are additional compiler options used for the core library.
          457  +#
          458  +!IFNDEF CORE_COMPILE_OPTS
          459  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          460  +CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport)
          461  +!ELSE
          462  +CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
          463  +!ENDIF
          464  +!ENDIF
          465  +
          466  +# These are the additional targets that the core library should depend on
          467  +# when linking.
          468  +#
          469  +!IFNDEF CORE_LINK_DEP
          470  +!IF $(DYNAMIC_SHELL)!=0
          471  +CORE_LINK_DEP =
          472  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
          473  +CORE_LINK_DEP = sqlite3.def
          474  +!ELSE
          475  +CORE_LINK_DEP =
          476  +!ENDIF
          477  +!ENDIF
          478  +
          479  +# These are additional linker options used for the core library.
          480  +#
          481  +!IFNDEF CORE_LINK_OPTS
          482  +!IF $(DYNAMIC_SHELL)!=0
          483  +CORE_LINK_OPTS =
          484  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
          485  +CORE_LINK_OPTS = /DEF:sqlite3.def
          486  +!ELSE
          487  +CORE_LINK_OPTS =
          488  +!ENDIF
          489  +!ENDIF
          490  +
          491  +# These are additional compiler options used for the shell executable.
          492  +#
          493  +!IFNDEF SHELL_COMPILE_OPTS
          494  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          495  +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
          496  +!ELSE
          497  +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
          498  +!ENDIF
          499  +!ENDIF
          500  +
          501  +# This is the source code that the shell executable should be compiled
          502  +# with.
          503  +#
          504  +!IFNDEF SHELL_CORE_SRC
          505  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          506  +SHELL_CORE_SRC =
          507  +!ELSE
          508  +SHELL_CORE_SRC = $(SQLITE3C)
          509  +!ENDIF
          510  +!ENDIF
          511  +
          512  +# This is the core library that the shell executable should depend on.
          513  +#
          514  +!IFNDEF SHELL_CORE_DEP
          515  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          516  +SHELL_CORE_DEP = $(SQLITE3DLL)
          517  +!ELSE
          518  +SHELL_CORE_DEP =
          519  +!ENDIF
          520  +!ENDIF
          521  +
          522  +# This is the core library that the shell executable should link with.
          523  +#
          524  +!IFNDEF SHELL_CORE_LIB
          525  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          526  +SHELL_CORE_LIB = $(SQLITE3LIB)
          527  +!ELSE
          528  +SHELL_CORE_LIB =
          529  +!ENDIF
          530  +!ENDIF
          531  +
          532  +# These are additional linker options used for the shell executable.
          533  +#
          534  +!IFNDEF SHELL_LINK_OPTS
          535  +SHELL_LINK_OPTS = $(SHELL_CORE_LIB)
          536  +!ENDIF
          537  +
          538  +# Check if assembly code listings should be generated for the source
          539  +# code files to be compiled.
          540  +#
          541  +!IF $(USE_LISTINGS)!=0
          542  +TCC = $(TCC) -FAcs
          543  +!ENDIF
          544  +
          545  +# When compiling the library for use in the WinRT environment,
          546  +# the following compile-time options must be used as well to
          547  +# disable use of Win32 APIs that are not available and to enable
          548  +# use of Win32 APIs that are specific to Windows 8 and/or WinRT.
          549  +#
          550  +!IF $(FOR_WINRT)!=0
          551  +TCC = $(TCC) -DSQLITE_OS_WINRT=1
          552  +RCC = $(RCC) -DSQLITE_OS_WINRT=1
          553  +TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
          554  +RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
          555  +!ENDIF
          556  +
          557  +# C compiler options for the Windows 10 platform (needs MSVC 2015).
          558  +#
          559  +!IF $(FOR_WIN10)!=0
          560  +TCC = $(TCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          561  +BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          562  +!ENDIF
          563  +
          564  +# Also, we need to dynamically link to the correct MSVC runtime
          565  +# when compiling for WinRT (e.g. debug or release) OR if the
          566  +# USE_CRT_DLL option is set to force dynamically linking to the
          567  +# MSVC runtime library.
          568  +#
          569  +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
          570  +!IF $(DEBUG)>1
          571  +TCC = $(TCC) -MDd
          572  +BCC = $(BCC) -MDd
          573  +!ELSE
          574  +TCC = $(TCC) -MD
          575  +BCC = $(BCC) -MD
          576  +!ENDIF
          577  +!ELSE
          578  +!IF $(DEBUG)>1
          579  +TCC = $(TCC) -MTd
          580  +BCC = $(BCC) -MTd
          581  +!ELSE
          582  +TCC = $(TCC) -MT
          583  +BCC = $(BCC) -MT
          584  +!ENDIF
          585  +!ENDIF
          586  +
          587  +
          588  +# Define -DNDEBUG to compile without debugging (i.e., for production usage)
          589  +# Omitting the define will cause extra debugging code to be inserted and
          590  +# includes extra comments when "EXPLAIN stmt" is used.
          591  +#
          592  +!IF $(DEBUG)==0
          593  +TCC = $(TCC) -DNDEBUG
          594  +BCC = $(BCC) -DNDEBUG
          595  +RCC = $(RCC) -DNDEBUG
          596  +!ENDIF
          597  +
          598  +!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0
          599  +TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
          600  +RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
          601  +!ENDIF
          602  +
          603  +!IF $(DEBUG)>2
          604  +TCC = $(TCC) -DSQLITE_DEBUG=1
          605  +RCC = $(RCC) -DSQLITE_DEBUG=1
          606  +!ENDIF
          607  +
          608  +!IF $(DEBUG)>4 || $(OSTRACE)!=0
          609  +TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
          610  +RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
          611  +!ENDIF
          612  +
          613  +!IF $(DEBUG)>5
          614  +TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE=1
          615  +RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE=1
          616  +!ENDIF
          617  +
          618  +# Prevent warnings about "insecure" MSVC runtime library functions
          619  +# being used.
          620  +#
          621  +TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
          622  +BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
          623  +RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
          624  +
          625  +# Prevent warnings about "deprecated" POSIX functions being used.
          626  +#
          627  +TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
          628  +BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
          629  +RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
          630  +
          631  +# Use the SQLite debugging heap subsystem?
          632  +#
          633  +!IF $(MEMDEBUG)!=0
          634  +TCC = $(TCC) -DSQLITE_MEMDEBUG=1
          635  +RCC = $(RCC) -DSQLITE_MEMDEBUG=1
          636  +
          637  +# Use native Win32 heap subsystem instead of malloc/free?
          638  +#
          639  +!ELSEIF $(WIN32HEAP)!=0
          640  +TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1
          641  +RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1
          642  +
          643  +# Validate the heap on every call into the native Win32 heap subsystem?
          644  +#
          645  +!IF $(DEBUG)>3
          646  +TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
          647  +RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
          648  +!ENDIF
          649  +!ENDIF
          650  +
          651  +
          652  +# Compiler options needed for programs that use the readline() library.
          653  +#
          654  +!IFNDEF READLINE_FLAGS
          655  +READLINE_FLAGS = -DHAVE_READLINE=0
          656  +!ENDIF
          657  +
          658  +# The library that programs using readline() must link against.
          659  +#
          660  +!IFNDEF LIBREADLINE
          661  +LIBREADLINE =
          662  +!ENDIF
          663  +
          664  +# Should the database engine be compiled threadsafe
          665  +#
          666  +TCC = $(TCC) -DSQLITE_THREADSAFE=1
          667  +RCC = $(RCC) -DSQLITE_THREADSAFE=1
          668  +
          669  +# Do threads override each others locks by default (1), or do we test (-1)
          670  +#
          671  +TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
          672  +RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
          673  +
          674  +# Any target libraries which libsqlite must be linked against
          675  +#
          676  +!IFNDEF TLIBS
          677  +TLIBS =
          678  +!ENDIF
          679  +
          680  +# Flags controlling use of the in memory btree implementation
          681  +#
          682  +# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to
          683  +# default to file, 2 to default to memory, and 3 to force temporary
          684  +# tables to always be in memory.
          685  +#
          686  +TCC = $(TCC) -DSQLITE_TEMP_STORE=1
          687  +RCC = $(RCC) -DSQLITE_TEMP_STORE=1
          688  +
          689  +# Enable/disable loadable extensions, and other optional features
          690  +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
          691  +# The same set of OMIT and ENABLE flags should be passed to the
          692  +# LEMON parser generator and the mkkeywordhash tool as well.
          693  +
          694  +# These are the required SQLite compilation options used when compiling for
          695  +# the Windows platform.
          696  +#
          697  +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100
          698  +
          699  +# If we are linking to the RPCRT4 library, enable features that need it.
          700  +#
          701  +!IF $(USE_RPCRT4_LIB)!=0
          702  +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1
          703  +!ENDIF
          704  +
          705  +# Add the required and optional SQLite compilation options into the command
          706  +# lines used to invoke the MSVC code and resource compilers.
          707  +#
          708  +TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
          709  +RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
          710  +
          711  +# Add in any optional parameters specified on the commane line, e.g.
          712  +# nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1"
          713  +#
          714  +TCC = $(TCC) $(OPTS)
          715  +RCC = $(RCC) $(OPTS)
          716  +
          717  +# If compiling for debugging, add some defines.
          718  +#
          719  +!IF $(DEBUG)>1
          720  +TCC = $(TCC) -D_DEBUG
          721  +BCC = $(BCC) -D_DEBUG
          722  +RCC = $(RCC) -D_DEBUG
          723  +!ENDIF
          724  +
          725  +# If optimizations are enabled or disabled (either implicitly or
          726  +# explicitly), add the necessary flags.
          727  +#
          728  +!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
          729  +TCC = $(TCC) -Od
          730  +BCC = $(BCC) -Od
          731  +!ELSEIF $(OPTIMIZATIONS)>=3
          732  +TCC = $(TCC) -Ox
          733  +BCC = $(BCC) -Ox
          734  +!ELSEIF $(OPTIMIZATIONS)==2
          735  +TCC = $(TCC) -O2
          736  +BCC = $(BCC) -O2
          737  +!ELSEIF $(OPTIMIZATIONS)==1
          738  +TCC = $(TCC) -O1
          739  +BCC = $(BCC) -O1
          740  +!ENDIF
          741  +
          742  +# If symbols are enabled (or compiling for debugging), enable PDBs.
          743  +#
          744  +!IF $(DEBUG)>1 || $(SYMBOLS)!=0
          745  +TCC = $(TCC) -Zi
          746  +BCC = $(BCC) -Zi
          747  +!ENDIF
          748  +
          749  +
          750  +# Command line prefixes for compiling code, compiling resources,
          751  +# linking, etc.
          752  +#
          753  +LTCOMPILE = $(TCC) -Fo$@
          754  +LTRCOMPILE = $(RCC) -r
          755  +LTLIB = lib.exe
          756  +LTLINK = $(TCC) -Fe$@
          757  +
          758  +# If requested, link to the RPCRT4 library.
          759  +#
          760  +!IF $(USE_RPCRT4_LIB)!=0
          761  +LTLINK = $(LTLINK) rpcrt4.lib
          762  +!ENDIF
          763  +
          764  +# If a platform was set, force the linker to target that.
          765  +# Note that the vcvars*.bat family of batch files typically
          766  +# set this for you.  Otherwise, the linker will attempt
          767  +# to deduce the binary type based on the object files.
          768  +!IFDEF PLATFORM
          769  +LTLINKOPTS = /NOLOGO /MACHINE:$(PLATFORM)
          770  +LTLIBOPTS = /NOLOGO /MACHINE:$(PLATFORM)
          771  +!ELSE
          772  +LTLINKOPTS = /NOLOGO
          773  +LTLIBOPTS = /NOLOGO
          774  +!ENDIF
          775  +
          776  +# When compiling for use in the WinRT environment, the following
          777  +# linker option must be used to mark the executable as runnable
          778  +# only in the context of an application container.
          779  +#
          780  +!IF $(FOR_WINRT)!=0
          781  +LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER
          782  +!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0"
          783  +!IFNDEF STORELIBPATH
          784  +!IF "$(PLATFORM)"=="x86"
          785  +STORELIBPATH = $(CRTLIBPATH)\store
          786  +!ELSEIF "$(PLATFORM)"=="x64"
          787  +STORELIBPATH = $(CRTLIBPATH)\store\amd64
          788  +!ELSEIF "$(PLATFORM)"=="ARM"
          789  +STORELIBPATH = $(CRTLIBPATH)\store\arm
          790  +!ELSE
          791  +STORELIBPATH = $(CRTLIBPATH)\store
          792  +!ENDIF
          793  +!ENDIF
          794  +STORELIBPATH = $(STORELIBPATH:\\=\)
          795  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)"
          796  +!ENDIF
          797  +!ENDIF
          798  +
          799  +# When compiling for Windows Phone 8.1, an extra library path is
          800  +# required.
          801  +#
          802  +!IF $(USE_WP81_OPTS)!=0
          803  +!IFNDEF WP81LIBPATH
          804  +!IF "$(PLATFORM)"=="x86"
          805  +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
          806  +!ELSEIF "$(PLATFORM)"=="ARM"
          807  +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM
          808  +!ELSE
          809  +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
          810  +!ENDIF
          811  +!ENDIF
          812  +!ENDIF
          813  +
          814  +# When compiling for Windows Phone 8.1, some extra linker options
          815  +# are also required.
          816  +#
          817  +!IF $(USE_WP81_OPTS)!=0
          818  +!IFDEF WP81LIBPATH
          819  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
          820  +!ENDIF
          821  +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
          822  +LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
          823  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
          824  +!ENDIF
          825  +
          826  +# When compiling for UWP or the Windows 10 platform, some extra linker
          827  +# options are also required.
          828  +#
          829  +!IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0
          830  +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
          831  +LTLINKOPTS = $(LTLINKOPTS) mincore.lib
          832  +!IFDEF PSDKLIBPATH
          833  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
          834  +!ENDIF
          835  +!ENDIF
          836  +
          837  +!IF $(FOR_WIN10)!=0
          838  +LTLINKOPTS = $(LTLINKOPTS) /guard:cf "/LIBPATH:$(UCRTLIBPATH)"
          839  +!IF $(DEBUG)>1
          840  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib
          841  +!ELSE
          842  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib
          843  +!ENDIF
          844  +!ENDIF
          845  +
          846  +# If either debugging or symbols are enabled, enable PDBs.
          847  +#
          848  +!IF $(DEBUG)>1 || $(SYMBOLS)!=0
          849  +LDFLAGS = /DEBUG $(LDOPTS)
          850  +!ELSE
          851  +LDFLAGS = $(LDOPTS)
          852  +!ENDIF
          853  +
          854  +
          855  +# You should not have to change anything below this line
          856  +###############################################################################
          857  +
          858  +
          859  +# Object files for the amalgamation.
          860  +#
          861  +LIBOBJS1 = sqlite3.lo
          862  +
          863  +# Determine the real value of LIBOBJ based on the 'configure' script
          864  +#
          865  +LIBOBJ = $(LIBOBJS1)
          866  +
          867  +# Determine if embedded resource compilation and usage are enabled.
          868  +#
          869  +!IF $(USE_RC)!=0
          870  +LIBRESOBJS = sqlite3res.lo
          871  +!ELSE
          872  +LIBRESOBJS =
          873  +!ENDIF
          874  +
          875  +
          876  +# Additional compiler options for the shell.  These are only effective
          877  +# when the shell is not being dynamically linked.
          878  +#
          879  +!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
          880  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
          881  +!ENDIF
          882  +
          883  +
          884  +# This is the default Makefile target.  The objects listed here
          885  +# are what get build when you type just "make" with no arguments.
          886  +#
          887  +all:	dll shell
          888  +
          889  +# Dynamic link library section.
          890  +#
          891  +dll:	$(SQLITE3DLL)
          892  +
          893  +# Shell executable.
          894  +#
          895  +shell:	$(SQLITE3EXE)
          896  +
          897  +
          898  +$(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
          899  +	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
          900  +
          901  +Replace.exe:
          902  +	$(CSC) /target:exe $(TOP)\Replace.cs
          903  +
          904  +sqlite3.def:	Replace.exe $(LIBOBJ)
          905  +	echo EXPORTS > sqlite3.def
          906  +	dumpbin /all $(LIBOBJ) \
          907  +		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
          908  +		| sort >> sqlite3.def
          909  +
          910  +$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
          911  +	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
          912  +		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
          913  +
          914  +
          915  +# Rule to build the amalgamation
          916  +#
          917  +sqlite3.lo:	$(SQLITE3C)
          918  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)
          919  +
          920  +
          921  +# Rule to build the Win32 resources object file.
          922  +#
          923  +!IF $(USE_RC)!=0
          924  +_HASHCHAR=^#
          925  +!IF ![echo !IFNDEF VERSION > rcver.vc] && \
          926  +    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
          927  +    ![echo !ENDIF >> rcver.vc]
          928  +!INCLUDE rcver.vc
          929  +!ENDIF
          930  +
          931  +RESOURCE_VERSION = $(VERSION:^#=)
          932  +RESOURCE_VERSION = $(RESOURCE_VERSION:define=)
          933  +RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=)
          934  +RESOURCE_VERSION = $(RESOURCE_VERSION:"=)
          935  +RESOURCE_VERSION = $(RESOURCE_VERSION:.=,)
          936  +
          937  +$(LIBRESOBJS):	$(TOP)\sqlite3.rc rcver.vc $(SQLITE3H)
          938  +	echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h
          939  +	echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h
          940  +	echo #endif >> sqlite3rc.h
          941  +	$(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc
          942  +!ENDIF
          943  +
          944  +
          945  +clean:
          946  +	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
          947  +	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
          948  +	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL

Name change from autoconf/README to autoconf/README.txt.

     1         -
     2      1   This package contains:
     3      2   
     4         - * the SQLite library amalgamation (single file) source code distribution,
     5         - * the shell.c file used to build the sqlite3 shell too, and
     6         - * the sqlite3.h and sqlite3ext.h header files required to link programs
     7         -   and sqlite extensions against the installed libary.
     8         - * autoconf/automake installation infrastucture.
            3  + * the SQLite library amalgamation source code file: sqlite3.c
            4  + * the sqlite3.h and sqlite3ext.h header files that define the C-language
            5  +   interface to the sqlite3.c library file
            6  + * the shell.c file used to build the sqlite3 command-line shell program
            7  + * autoconf/automake installation infrastucture for building on POSIX
            8  +   compliant systems
            9  + * a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft
           10  +   Visual C++ on Windows
           11  +
           12  +SUMMARY OF HOW TO BUILD
           13  +=======================
           14  +
           15  +  Unix:      ./configure; make
           16  +  Windows:   nmake /f Makefile.msc
           17  +
           18  +BUILDING ON POSIX
           19  +=================
     9     20   
    10     21   The generic installation instructions for autoconf/automake are found
    11     22   in the INSTALL file.
    12     23   
    13     24   The following SQLite specific boolean options are supported:
    14     25   
    15     26     --enable-readline           use readline in shell tool   [default=yes]
    16     27     --enable-threadsafe         build a thread-safe library  [default=yes]
    17     28     --enable-dynamic-extensions support loadable extensions  [default=yes]
    18     29   
    19         -The default value for the CFLAGS variable (options passed to the C 
           30  +The default value for the CFLAGS variable (options passed to the C
    20     31   compiler) includes debugging symbols in the build, resulting in larger
    21     32   binaries than are necessary. Override it on the configure command
    22     33   line like this:
    23     34   
    24     35     $ CFLAGS="-Os" ./configure
    25     36   
    26     37   to produce a smaller installation footprint.
    27     38   
    28     39   Other SQLite compilation parameters can also be set using CFLAGS. For
    29     40   example:
    30     41   
    31         -  $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure
           42  +  $ CFLAGS="-Os -DSQLITE_THREADSAFE=0" ./configure
           43  +
           44  +
           45  +BUILDING WITH MICROSOFT VISUAL C++
           46  +==================================
           47  +
           48  +To compile for Windows using Microsoft Visual C++:
           49  +
           50  +  $ nmake /f Makefile.msc
           51  +
           52  +Using Microsoft Visual C++ 2005 (or later) is recommended.  Several Windows
           53  +platform variants may be built by adding additional macros to the NMAKE
           54  +command line.
           55  +
           56  +Building for WinRT 8.0
           57  +----------------------
           58  +
           59  +  FOR_WINRT=1
           60  +
           61  +Using Microsoft Visual C++ 2012 (or later) is required.  When using the
           62  +above, something like the following macro will need to be added to the
           63  +NMAKE command line as well:
           64  +
           65  +  "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86"
           66  +
           67  +Building for WinRT 8.1
           68  +----------------------
           69  +
           70  +  FOR_WINRT=1
           71  +
           72  +Using Microsoft Visual C++ 2013 (or later) is required.  When using the
           73  +above, something like the following macro will need to be added to the
           74  +NMAKE command line as well:
           75  +
           76  +  "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86"
           77  +
           78  +Building for UWP 10.0
           79  +---------------------
           80  +
           81  +  FOR_WINRT=1 FOR_UWP=1
           82  +
           83  +Using Microsoft Visual C++ 2015 (or later) is required.  When using the
           84  +above, something like the following macros will need to be added to the
           85  +NMAKE command line as well:
           86  +
           87  +  "NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
           88  +  "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
           89  +  "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86"
           90  +
           91  +Building for the Windows 10 SDK
           92  +-------------------------------
           93  +
           94  +  FOR_WIN10=1
           95  +
           96  +Using Microsoft Visual C++ 2015 (or later) is required.  When using the
           97  +above, no other macros should be needed on the NMAKE command line.
           98  +
           99  +Other preprocessor defines
          100  +--------------------------
          101  +
          102  +Additionally, preprocessor defines may be specified by using the OPTS macro
          103  +on the NMAKE command line.  However, not all possible preprocessor defines
          104  +may be specified in this manner as some require the amalgamation to be built
          105  +with them enabled (see http://www.sqlite.org/compile.html). For example, the
          106  +following will work:
          107  +
          108  +  "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1"
          109  +
          110  +However, the following will not compile unless the amalgamation was built
          111  +with it enabled:
    32    112   
          113  +  "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1"

Changes to autoconf/configure.ac.

    71     71   #
    72     72   AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
    73     73     [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
    74     74     [], [enable_threadsafe=yes])
    75     75   THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
    76     76   if test x"$enable_threadsafe" != "xno"; then
    77     77     THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
           78  +  AC_SEARCH_LIBS(pthread_create, pthread)
    78     79     AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
    79     80   fi
    80     81   AC_SUBST(THREADSAFE_FLAGS)
    81     82   #-----------------------------------------------------------------------
    82     83   
    83     84   #-----------------------------------------------------------------------
    84     85   #   --enable-dynamic-extensions
................................................................................
   125    126   #   --enable-static-shell
   126    127   #
   127    128   AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
   128    129     [--enable-static-shell], 
   129    130     [statically link libsqlite3 into shell tool [default=yes]])], 
   130    131     [], [enable_static_shell=yes])
   131    132   if test x"$enable_static_shell" == "xyes"; then
   132         -  EXTRA_SHELL_OBJ=sqlite3.$OBJEXT
          133  +  EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT
   133    134   else
   134    135     EXTRA_SHELL_OBJ=libsqlite3.la
   135    136   fi
   136    137   AC_SUBST(EXTRA_SHELL_OBJ)
   137    138   #-----------------------------------------------------------------------
   138    139   
   139    140   AC_CHECK_FUNCS(posix_fallocate)

Changes to config.h.in.

    40     40   #undef HAVE_MALLOC_H
    41     41   
    42     42   /* Define to 1 if you have the `malloc_usable_size' function. */
    43     43   #undef HAVE_MALLOC_USABLE_SIZE
    44     44   
    45     45   /* Define to 1 if you have the <memory.h> header file. */
    46     46   #undef HAVE_MEMORY_H
           47  +
           48  +/* Define to 1 if you have the pread() function. */
           49  +#undef HAVE_PREAD
           50  +
           51  +/* Define to 1 if you have the pread64() function. */
           52  +#undef HAVE_PREAD64
           53  +
           54  +/* Define to 1 if you have the pwrite() function. */
           55  +#undef HAVE_PWRITE
           56  +
           57  +/* Define to 1 if you have the pwrite64() function. */
           58  +#undef HAVE_PWRITE64
    47     59   
    48     60   /* Define to 1 if you have the <stdint.h> header file. */
    49     61   #undef HAVE_STDINT_H
    50     62   
    51     63   /* Define to 1 if you have the <stdlib.h> header file. */
    52     64   #undef HAVE_STDLIB_H
    53     65   

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.11.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.13.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.11.0'
   730         -PACKAGE_STRING='sqlite 3.11.0'
          729  +PACKAGE_VERSION='3.13.0'
          730  +PACKAGE_STRING='sqlite 3.13.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1456   1456   #
  1457   1457   # Report the --help message.
  1458   1458   #
  1459   1459   if test "$ac_init_help" = "long"; then
  1460   1460     # Omit some internal or obsolete options to make the list less imposing.
  1461   1461     # This message is too long to be a string in the A/UX 3.1 sh.
  1462   1462     cat <<_ACEOF
  1463         -\`configure' configures sqlite 3.11.0 to adapt to many kinds of systems.
         1463  +\`configure' configures sqlite 3.13.0 to adapt to many kinds of systems.
  1464   1464   
  1465   1465   Usage: $0 [OPTION]... [VAR=VALUE]...
  1466   1466   
  1467   1467   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1468   1468   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1469   1469   
  1470   1470   Defaults for the options are specified in brackets.
................................................................................
  1521   1521     --build=BUILD     configure for building on BUILD [guessed]
  1522   1522     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1523   1523   _ACEOF
  1524   1524   fi
  1525   1525   
  1526   1526   if test -n "$ac_init_help"; then
  1527   1527     case $ac_init_help in
  1528         -     short | recursive ) echo "Configuration of sqlite 3.11.0:";;
         1528  +     short | recursive ) echo "Configuration of sqlite 3.13.0:";;
  1529   1529      esac
  1530   1530     cat <<\_ACEOF
  1531   1531   
  1532   1532   Optional Features:
  1533   1533     --disable-option-checking  ignore unrecognized --enable/--with options
  1534   1534     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1535   1535     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1642   1642       cd "$ac_pwd" || { ac_status=$?; break; }
  1643   1643     done
  1644   1644   fi
  1645   1645   
  1646   1646   test -n "$ac_init_help" && exit $ac_status
  1647   1647   if $ac_init_version; then
  1648   1648     cat <<\_ACEOF
  1649         -sqlite configure 3.11.0
         1649  +sqlite configure 3.13.0
  1650   1650   generated by GNU Autoconf 2.69
  1651   1651   
  1652   1652   Copyright (C) 2012 Free Software Foundation, Inc.
  1653   1653   This configure script is free software; the Free Software Foundation
  1654   1654   gives unlimited permission to copy, distribute and modify it.
  1655   1655   _ACEOF
  1656   1656     exit
................................................................................
  2061   2061     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2062   2062   
  2063   2063   } # ac_fn_c_check_header_mongrel
  2064   2064   cat >config.log <<_ACEOF
  2065   2065   This file contains any messages produced by compilers while
  2066   2066   running configure, to aid debugging if configure makes a mistake.
  2067   2067   
  2068         -It was created by sqlite $as_me 3.11.0, which was
         2068  +It was created by sqlite $as_me 3.13.0, which was
  2069   2069   generated by GNU Autoconf 2.69.  Invocation command line was
  2070   2070   
  2071   2071     $ $0 $@
  2072   2072   
  2073   2073   _ACEOF
  2074   2074   exec 5>>config.log
  2075   2075   {
................................................................................
 10269  10269   
 10270  10270   done
 10271  10271   
 10272  10272   
 10273  10273   #########
 10274  10274   # Figure out whether or not we have these functions
 10275  10275   #
 10276         -for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime
        10276  +for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64
 10277  10277   do :
 10278  10278     as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 10279  10279   ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
 10280  10280   if eval test \"x\$"$as_ac_var"\" = x"yes"; then :
 10281  10281     cat >>confdefs.h <<_ACEOF
 10282  10282   #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1
 10283  10283   _ACEOF
................................................................................
 10460  10460     SQLITE_THREADSAFE=1
 10461  10461     { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 10462  10462   $as_echo "yes" >&6; }
 10463  10463   fi
 10464  10464   
 10465  10465   
 10466  10466   if test "$SQLITE_THREADSAFE" = "1"; then
        10467  +  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5
        10468  +$as_echo_n "checking for library containing pthread_create... " >&6; }
        10469  +if ${ac_cv_search_pthread_create+:} false; then :
        10470  +  $as_echo_n "(cached) " >&6
        10471  +else
        10472  +  ac_func_search_save_LIBS=$LIBS
        10473  +cat confdefs.h - <<_ACEOF >conftest.$ac_ext
        10474  +/* end confdefs.h.  */
        10475  +
        10476  +/* Override any GCC internal prototype to avoid an error.
        10477  +   Use char because int might match the return type of a GCC
        10478  +   builtin and then its argument prototype would still apply.  */
        10479  +#ifdef __cplusplus
        10480  +extern "C"
        10481  +#endif
        10482  +char pthread_create ();
        10483  +int
        10484  +main ()
        10485  +{
        10486  +return pthread_create ();
        10487  +  ;
        10488  +  return 0;
        10489  +}
        10490  +_ACEOF
        10491  +for ac_lib in '' pthread; do
        10492  +  if test -z "$ac_lib"; then
        10493  +    ac_res="none required"
        10494  +  else
        10495  +    ac_res=-l$ac_lib
        10496  +    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
        10497  +  fi
        10498  +  if ac_fn_c_try_link "$LINENO"; then :
        10499  +  ac_cv_search_pthread_create=$ac_res
        10500  +fi
        10501  +rm -f core conftest.err conftest.$ac_objext \
        10502  +    conftest$ac_exeext
        10503  +  if ${ac_cv_search_pthread_create+:} false; then :
        10504  +  break
        10505  +fi
        10506  +done
        10507  +if ${ac_cv_search_pthread_create+:} false; then :
        10508  +
        10509  +else
        10510  +  ac_cv_search_pthread_create=no
        10511  +fi
        10512  +rm conftest.$ac_ext
        10513  +LIBS=$ac_func_search_save_LIBS
        10514  +fi
        10515  +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5
        10516  +$as_echo "$ac_cv_search_pthread_create" >&6; }
        10517  +ac_res=$ac_cv_search_pthread_create
        10518  +if test "$ac_res" != no; then :
        10519  +  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
        10520  +
        10521  +fi
        10522  +
 10467  10523     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5
 10468  10524   $as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; }
 10469  10525   if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
 10470  10526     $as_echo_n "(cached) " >&6
 10471  10527   else
 10472  10528     ac_func_search_save_LIBS=$LIBS
 10473  10529   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
................................................................................
 12019  12075   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12020  12076   
 12021  12077   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12022  12078   # Save the log message, to keep $0 and so on meaningful, and to
 12023  12079   # report actual input values of CONFIG_FILES etc. instead of their
 12024  12080   # values after options handling.
 12025  12081   ac_log="
 12026         -This file was extended by sqlite $as_me 3.11.0, which was
        12082  +This file was extended by sqlite $as_me 3.13.0, which was
 12027  12083   generated by GNU Autoconf 2.69.  Invocation command line was
 12028  12084   
 12029  12085     CONFIG_FILES    = $CONFIG_FILES
 12030  12086     CONFIG_HEADERS  = $CONFIG_HEADERS
 12031  12087     CONFIG_LINKS    = $CONFIG_LINKS
 12032  12088     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12033  12089     $ $0 $@
................................................................................
 12085  12141   
 12086  12142   Report bugs to the package provider."
 12087  12143   
 12088  12144   _ACEOF
 12089  12145   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12090  12146   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12091  12147   ac_cs_version="\\
 12092         -sqlite config.status 3.11.0
        12148  +sqlite config.status 3.13.0
 12093  12149   configured by $0, generated by GNU Autoconf 2.69,
 12094  12150     with options \\"\$ac_cs_config\\"
 12095  12151   
 12096  12152   Copyright (C) 2012 Free Software Foundation, Inc.
 12097  12153   This config.status script is free software; the Free Software Foundation
 12098  12154   gives unlimited permission to copy, distribute and modify it."
 12099  12155   

Changes to configure.ac.

   104    104   #########
   105    105   # Check for needed/wanted headers
   106    106   AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h])
   107    107   
   108    108   #########
   109    109   # Figure out whether or not we have these functions
   110    110   #
   111         -AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime])
          111  +AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64])
   112    112   
   113    113   #########
   114    114   # By default, we use the amalgamation (this may be changed below...)
   115    115   #
   116    116   USE_AMALGAMATION=1
   117    117   
   118    118   #########
................................................................................
   190    190   else
   191    191     SQLITE_THREADSAFE=1
   192    192     AC_MSG_RESULT([yes])
   193    193   fi
   194    194   AC_SUBST(SQLITE_THREADSAFE)
   195    195   
   196    196   if test "$SQLITE_THREADSAFE" = "1"; then
          197  +  AC_SEARCH_LIBS(pthread_create, pthread)
   197    198     AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
   198    199   fi
   199    200   
   200    201   ##########
   201    202   # Do we want to support release
   202    203   #
   203    204   AC_ARG_ENABLE(releasemode, 

Changes to doc/lemon.html.

     1      1   <html>
     2      2   <head>
     3      3   <title>The Lemon Parser Generator</title>
     4      4   </head>
     5      5   <body bgcolor=white>
     6      6   <h1 align=center>The Lemon Parser Generator</h1>
     7      7   
     8         -<p>Lemon is an LALR(1) parser generator for C or C++.  
     9         -It does the same job as ``bison'' and ``yacc''.
    10         -But lemon is not another bison or yacc clone.  It
            8  +<p>Lemon is an LALR(1) parser generator for C.
            9  +It does the same job as "bison" and "yacc".
           10  +But lemon is not a bison or yacc clone.  Lemon
    11     11   uses a different grammar syntax which is designed to
    12         -reduce the number of coding errors.  Lemon also uses a more
    13         -sophisticated parsing engine that is faster than yacc and
    14         -bison and which is both reentrant and thread-safe.
    15         -Furthermore, Lemon implements features that can be used
           12  +reduce the number of coding errors.  Lemon also uses a
           13  +parsing engine that is faster than yacc and
           14  +bison and which is both reentrant and threadsafe.
           15  +(Update: Since the previous sentence was written, bison
           16  +has also been updated so that it too can generate a
           17  +reentrant and threadsafe parser.)
           18  +Lemon also implements features that can be used
    16     19   to eliminate resource leaks, making is suitable for use
    17     20   in long-running programs such as graphical user interfaces
    18     21   or embedded controllers.</p>
    19     22   
    20     23   <p>This document is an introduction to the Lemon
    21     24   parser generator.</p>
    22     25   
................................................................................
    40     43   <ul>
    41     44   <li>C code to implement the parser.
    42     45   <li>A header file defining an integer ID for each terminal symbol.
    43     46   <li>An information file that describes the states of the generated parser
    44     47       automaton.
    45     48   </ul>
    46     49   By default, all three of these output files are generated.
    47         -The header file is suppressed if the ``-m'' command-line option is
    48         -used and the report file is omitted when ``-q'' is selected.</p>
           50  +The header file is suppressed if the "-m" command-line option is
           51  +used and the report file is omitted when "-q" is selected.</p>
    49     52   
    50         -<p>The grammar specification file uses a ``.y'' suffix, by convention.
           53  +<p>The grammar specification file uses a ".y" suffix, by convention.
    51     54   In the examples used in this document, we'll assume the name of the
    52         -grammar file is ``gram.y''.  A typical use of Lemon would be the
           55  +grammar file is "gram.y".  A typical use of Lemon would be the
    53     56   following command:
    54     57   <pre>
    55     58      lemon gram.y
    56     59   </pre>
    57         -This command will generate three output files named ``gram.c'',
    58         -``gram.h'' and ``gram.out''.
           60  +This command will generate three output files named "gram.c",
           61  +"gram.h" and "gram.out".
    59     62   The first is C code to implement the parser.  The second
    60     63   is the header file that defines numerical values for all
    61     64   terminal symbols, and the last is the report that explains
    62     65   the states used by the parser automaton.</p>
    63     66   
    64     67   <h3>Command Line Options</h3>
    65     68   
................................................................................
    67     70   You can obtain a list of the available command-line options together
    68     71   with a brief explanation of what each does by typing
    69     72   <pre>
    70     73      lemon -?
    71     74   </pre>
    72     75   As of this writing, the following command-line options are supported:
    73     76   <ul>
    74         -<li><tt>-b</tt>
    75         -<li><tt>-c</tt>
    76         -<li><tt>-g</tt>
    77         -<li><tt>-m</tt>
    78         -<li><tt>-q</tt>
    79         -<li><tt>-s</tt>
    80         -<li><tt>-x</tt>
    81         -</ul>
    82         -The ``-b'' option reduces the amount of text in the report file by
    83         -printing only the basis of each parser state, rather than the full
    84         -configuration.
    85         -The ``-c'' option suppresses action table compression.  Using -c
    86         -will make the parser a little larger and slower but it will detect
    87         -syntax errors sooner.
    88         -The ``-g'' option causes no output files to be generated at all.
    89         -Instead, the input grammar file is printed on standard output but
    90         -with all comments, actions and other extraneous text deleted.  This
    91         -is a useful way to get a quick summary of a grammar.
    92         -The ``-m'' option causes the output C source file to be compatible
    93         -with the ``makeheaders'' program.
    94         -Makeheaders is a program that automatically generates header files
    95         -from C source code.  When the ``-m'' option is used, the header
    96         -file is not output since the makeheaders program will take care
    97         -of generated all header files automatically.
    98         -The ``-q'' option suppresses the report file.
    99         -Using ``-s'' causes a brief summary of parser statistics to be
   100         -printed.  Like this:
   101         -<pre>
   102         -   Parser statistics: 74 terminals, 70 nonterminals, 179 rules
   103         -                      340 states, 2026 parser table entries, 0 conflicts
   104         -</pre>
   105         -Finally, the ``-x'' option causes Lemon to print its version number
   106         -and then stops without attempting to read the grammar or generate a parser.</p>
           77  +<li><b>-b</b>
           78  +Show only the basis for each parser state in the report file.
           79  +<li><b>-c</b>
           80  +Do not compress the generated action tables.
           81  +<li><b>-D<i>name</i></b>
           82  +Define C preprocessor macro <i>name</i>.  This macro is useable by
           83  +"%ifdef" lines in the grammar file.
           84  +<li><b>-g</b>
           85  +Do not generate a parser.  Instead write the input grammar to standard
           86  +output with all comments, actions, and other extraneous text removed.
           87  +<li><b>-l</b>
           88  +Omit "#line" directives int the generated parser C code.
           89  +<li><b>-m</b>
           90  +Cause the output C source code to be compatible with the "makeheaders"
           91  +program. 
           92  +<li><b>-p</b>
           93  +Display all conflicts that are resolved by 
           94  +<a href='#precrules'>precedence rules</a>.
           95  +<li><b>-q</b>
           96  +Suppress generation of the report file.
           97  +<li><b>-r</b>
           98  +Do not sort or renumber the parser states as part of optimization.
           99  +<li><b>-s</b>
          100  +Show parser statistics before existing.
          101  +<li><b>-T<i>file</i></b>
          102  +Use <i>file</i> as the template for the generated C-code parser implementation.
          103  +<li><b>-x</b>
          104  +Print the Lemon version number.
          105  +</ul>
   107    106   
   108    107   <h3>The Parser Interface</h3>
   109    108   
   110    109   <p>Lemon doesn't generate a complete, working program.  It only generates
   111    110   a few subroutines that implement a parser.  This section describes
   112    111   the interface to those subroutines.  It is up to the programmer to
   113    112   call these subroutines in an appropriate way in order to produce a
................................................................................
   117    116   must first create the parser.
   118    117   A new parser is created as follows:
   119    118   <pre>
   120    119      void *pParser = ParseAlloc( malloc );
   121    120   </pre>
   122    121   The ParseAlloc() routine allocates and initializes a new parser and
   123    122   returns a pointer to it.
   124         -The actual data structure used to represent a parser is opaque --
          123  +The actual data structure used to represent a parser is opaque &mdash;
   125    124   its internal structure is not visible or usable by the calling routine.
   126    125   For this reason, the ParseAlloc() routine returns a pointer to void
   127    126   rather than a pointer to some particular structure.
   128    127   The sole argument to the ParseAlloc() routine is a pointer to the
   129         -subroutine used to allocate memory.  Typically this means ``malloc()''.</p>
          128  +subroutine used to allocate memory.  Typically this means malloc().</p>
   130    129   
   131    130   <p>After a program is finished using a parser, it can reclaim all
   132    131   memory allocated by that parser by calling
   133    132   <pre>
   134    133      ParseFree(pParser, free);
   135    134   </pre>
   136    135   The first argument is the same pointer returned by ParseAlloc().  The
................................................................................
   147    146   The first argument to the Parse() routine is the pointer returned by
   148    147   ParseAlloc().
   149    148   The second argument is a small positive integer that tells the parse the
   150    149   type of the next token in the data stream.
   151    150   There is one token type for each terminal symbol in the grammar.
   152    151   The gram.h file generated by Lemon contains #define statements that
   153    152   map symbolic terminal symbol names into appropriate integer values.
   154         -(A value of 0 for the second argument is a special flag to the
   155         -parser to indicate that the end of input has been reached.)
          153  +A value of 0 for the second argument is a special flag to the
          154  +parser to indicate that the end of input has been reached.
   156    155   The third argument is the value of the given token.  By default,
   157    156   the type of the third argument is integer, but the grammar will
   158    157   usually redefine this type to be some kind of structure.
   159    158   Typically the second argument will be a broad category of tokens
   160         -such as ``identifier'' or ``number'' and the third argument will
          159  +such as "identifier" or "number" and the third argument will
   161    160   be the name of the identifier or the value of the number.</p>
   162    161   
   163    162   <p>The Parse() function may have either three or four arguments,
   164         -depending on the grammar.  If the grammar specification file request
   165         -it, the Parse() function will have a fourth parameter that can be
          163  +depending on the grammar.  If the grammar specification file requests
          164  +it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
          165  +the Parse() function will have a fourth parameter that can be
   166    166   of any type chosen by the programmer.  The parser doesn't do anything
   167    167   with this argument except to pass it through to action routines.
   168    168   This is a convenient mechanism for passing state information down
   169    169   to the action routines without having to use global variables.</p>
   170    170   
   171    171   <p>A typical use of a Lemon parser might look something like the
   172    172   following:
................................................................................
   188    188      15    ParseFree(pParser, free );
   189    189      16    TokenizerFree(pTokenizer);
   190    190      17    return sState.treeRoot;
   191    191      18 }
   192    192   </pre>
   193    193   This example shows a user-written routine that parses a file of
   194    194   text and returns a pointer to the parse tree.
   195         -(We've omitted all error-handling from this example to keep it
          195  +(All error-handling code is omitted from this example to keep it
   196    196   simple.)
   197    197   We assume the existence of some kind of tokenizer which is created
   198    198   using TokenizerCreate() on line 8 and deleted by TokenizerFree()
   199    199   on line 16.  The GetNextToken() function on line 11 retrieves the
   200    200   next token from the input file and puts its type in the 
   201    201   integer variable hTokenId.  The sToken variable is assumed to be
   202    202   some kind of structure that contains details about each token,
................................................................................
   259    259       and bison do not.
   260    260   </ul>
   261    261   These differences may cause some initial confusion for programmers
   262    262   with prior yacc and bison experience.
   263    263   But after years of experience using Lemon, I firmly
   264    264   believe that the Lemon way of doing things is better.</p>
   265    265   
          266  +<p><i>Updated as of 2016-02-16:</i>
          267  +The text above was written in the 1990s.
          268  +We are told that Bison has lately been enhanced to support the
          269  +tokenizer-calls-parser paradigm used by Lemon, and to obviate the
          270  +need for global variables.</p>
          271  +
   266    272   <h2>Input File Syntax</h2>
   267    273   
   268    274   <p>The main purpose of the grammar specification file for Lemon is
   269    275   to define the grammar for the parser.  But the input file also
   270    276   specifies additional information Lemon requires to do its job.
   271    277   Most of the work in using Lemon is in writing an appropriate
   272    278   grammar file.</p>
................................................................................
   276    282   declaration can occur at any point in the file.
   277    283   Lemon ignores whitespace (except where it is needed to separate
   278    284   tokens) and it honors the same commenting conventions as C and C++.</p>
   279    285   
   280    286   <h3>Terminals and Nonterminals</h3>
   281    287   
   282    288   <p>A terminal symbol (token) is any string of alphanumeric
   283         -and underscore characters
          289  +and/or underscore characters
   284    290   that begins with an upper case letter.
   285    291   A terminal can contain lowercase letters after the first character,
   286    292   but the usual convention is to make terminals all upper case.
   287    293   A nonterminal, on the other hand, is any string of alphanumeric
   288    294   and underscore characters than begins with a lower case letter.
   289    295   Again, the usual convention is to make nonterminals use all lower
   290    296   case letters.</p>
................................................................................
   303    309   must have alphanumeric names.</p>
   304    310   
   305    311   <h3>Grammar Rules</h3>
   306    312   
   307    313   <p>The main component of a Lemon grammar file is a sequence of grammar
   308    314   rules.
   309    315   Each grammar rule consists of a nonterminal symbol followed by
   310         -the special symbol ``::='' and then a list of terminals and/or nonterminals.
          316  +the special symbol "::=" and then a list of terminals and/or nonterminals.
   311    317   The rule is terminated by a period.
   312    318   The list of terminals and nonterminals on the right-hand side of the
   313    319   rule can be empty.
   314    320   Rules can occur in any order, except that the left-hand side of the
   315    321   first rule is assumed to be the start symbol for the grammar (unless
   316    322   specified otherwise using the <tt>%start</tt> directive described below.)
   317    323   A typical sequence of grammar rules might look something like this:
................................................................................
   319    325     expr ::= expr PLUS expr.
   320    326     expr ::= expr TIMES expr.
   321    327     expr ::= LPAREN expr RPAREN.
   322    328     expr ::= VALUE.
   323    329   </pre>
   324    330   </p>
   325    331   
   326         -<p>There is one non-terminal in this example, ``expr'', and five
   327         -terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'',
   328         -``RPAREN'' and ``VALUE''.</p>
          332  +<p>There is one non-terminal in this example, "expr", and five
          333  +terminal symbols or tokens: "PLUS", "TIMES", "LPAREN",
          334  +"RPAREN" and "VALUE".</p>
   329    335   
   330    336   <p>Like yacc and bison, Lemon allows the grammar to specify a block
   331    337   of C code that will be executed whenever a grammar rule is reduced
   332    338   by the parser.
   333    339   In Lemon, this action is specified by putting the C code (contained
   334    340   within curly braces <tt>{...}</tt>) immediately after the
   335    341   period that closes the rule.
................................................................................
   337    343   <pre>
   338    344     expr ::= expr PLUS expr.   { printf("Doing an addition...\n"); }
   339    345   </pre>
   340    346   </p>
   341    347   
   342    348   <p>In order to be useful, grammar actions must normally be linked to
   343    349   their associated grammar rules.
   344         -In yacc and bison, this is accomplished by embedding a ``$$'' in the
          350  +In yacc and bison, this is accomplished by embedding a "$$" in the
   345    351   action to stand for the value of the left-hand side of the rule and
   346         -symbols ``$1'', ``$2'', and so forth to stand for the value of
          352  +symbols "$1", "$2", and so forth to stand for the value of
   347    353   the terminal or nonterminal at position 1, 2 and so forth on the
   348    354   right-hand side of the rule.
   349    355   This idea is very powerful, but it is also very error-prone.  The
   350    356   single most common source of errors in a yacc or bison grammar is
   351    357   to miscount the number of symbols on the right-hand side of a grammar
   352         -rule and say ``$7'' when you really mean ``$8''.</p>
          358  +rule and say "$7" when you really mean "$8".</p>
   353    359   
   354    360   <p>Lemon avoids the need to count grammar symbols by assigning symbolic
   355    361   names to each symbol in a grammar rule and then using those symbolic
   356    362   names in the action.
   357    363   In yacc or bison, one would write this:
   358    364   <pre>
   359    365     expr -> expr PLUS expr  { $$ = $1 + $3; };
................................................................................
   375    381   includes a linking symbol in parentheses but that linking symbol
   376    382   is not actually used in the reduce action, then an error message
   377    383   is generated.
   378    384   For example, the rule
   379    385   <pre>
   380    386     expr(A) ::= expr(B) PLUS expr(C).  { A = B; }
   381    387   </pre>
   382         -will generate an error because the linking symbol ``C'' is used
          388  +will generate an error because the linking symbol "C" is used
   383    389   in the grammar rule but not in the reduce action.</p>
   384    390   
   385    391   <p>The Lemon notation for linking grammar rules to reduce actions
   386    392   also facilitates the use of destructors for reclaiming memory
   387    393   allocated by the values of terminals and nonterminals on the
   388    394   right-hand side of a rule.</p>
   389    395   
          396  +<a name='precrules'></a>
   390    397   <h3>Precedence Rules</h3>
   391    398   
   392    399   <p>Lemon resolves parsing ambiguities in exactly the same way as
   393    400   yacc and bison.  A shift-reduce conflict is resolved in favor
   394    401   of the shift, and a reduce-reduce conflict is resolved by reducing
   395    402   whichever rule comes first in the grammar file.</p>
   396    403   
   397    404   <p>Just like in
   398    405   yacc and bison, Lemon allows a measure of control 
   399    406   over the resolution of paring conflicts using precedence rules.
   400    407   A precedence value can be assigned to any terminal symbol
   401         -using the %left, %right or %nonassoc directives.  Terminal symbols
          408  +using the 
          409  +<a href='#pleft'>%left</a>,
          410  +<a href='#pright'>%right</a> or
          411  +<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
   402    412   mentioned in earlier directives have a lower precedence that
   403    413   terminal symbols mentioned in later directives.  For example:</p>
   404    414   
   405    415   <p><pre>
   406    416      %left AND.
   407    417      %left OR.
   408    418      %nonassoc EQ NE GT GE LT LE.
................................................................................
   514    524   
   515    525   <p>Lemon supports the following special directives:
   516    526   <ul>
   517    527   <li><tt>%code</tt>
   518    528   <li><tt>%default_destructor</tt>
   519    529   <li><tt>%default_type</tt>
   520    530   <li><tt>%destructor</tt>
          531  +<li><tt>%endif</tt>
   521    532   <li><tt>%extra_argument</tt>
          533  +<li><tt>%fallback</tt>
          534  +<li><tt>%ifdef</tt>
          535  +<li><tt>%ifndef</tt>
   522    536   <li><tt>%include</tt>
   523    537   <li><tt>%left</tt>
   524    538   <li><tt>%name</tt>
   525    539   <li><tt>%nonassoc</tt>
   526    540   <li><tt>%parse_accept</tt>
   527    541   <li><tt>%parse_failure </tt>
   528    542   <li><tt>%right</tt>
   529    543   <li><tt>%stack_overflow</tt>
   530    544   <li><tt>%stack_size</tt>
   531    545   <li><tt>%start_symbol</tt>
   532    546   <li><tt>%syntax_error</tt>
          547  +<li><tt>%token_class</tt>
   533    548   <li><tt>%token_destructor</tt>
   534    549   <li><tt>%token_prefix</tt>
   535    550   <li><tt>%token_type</tt>
   536    551   <li><tt>%type</tt>
          552  +<li><tt>%wildcard</tt>
   537    553   </ul>
   538    554   Each of these directives will be described separately in the
   539    555   following sections:</p>
   540    556   
          557  +<a name='pcode'></a>
   541    558   <h4>The <tt>%code</tt> directive</h4>
   542    559   
   543         -<p>The %code directive is used to specify addition C/C++ code that
          560  +<p>The %code directive is used to specify addition C code that
   544    561   is added to the end of the main output file.  This is similar to
   545         -the %include directive except that %include is inserted at the
   546         -beginning of the main output file.</p>
          562  +the <a href='#pinclude'>%include</a> directive except that %include
          563  +is inserted at the beginning of the main output file.</p>
   547    564   
   548    565   <p>%code is typically used to include some action routines or perhaps
   549         -a tokenizer as part of the output file.</p>
          566  +a tokenizer or even the "main()" function 
          567  +as part of the output file.</p>
   550    568   
          569  +<a name='default_destructor'></a>
   551    570   <h4>The <tt>%default_destructor</tt> directive</h4>
   552    571   
   553    572   <p>The %default_destructor directive specifies a destructor to 
   554    573   use for non-terminals that do not have their own destructor
   555    574   specified by a separate %destructor directive.  See the documentation
   556         -on the %destructor directive below for additional information.</p>
          575  +on the <a name='#destructor'>%destructor</a> directive below for
          576  +additional information.</p>
   557    577   
   558    578   <p>In some grammers, many different non-terminal symbols have the
   559    579   same datatype and hence the same destructor.  This directive is
   560    580   a convenience way to specify the same destructor for all those
   561    581   non-terminals using a single statement.</p>
   562    582   
          583  +<a name='default_type'></a>
   563    584   <h4>The <tt>%default_type</tt> directive</h4>
   564    585   
   565    586   <p>The %default_type directive specifies the datatype of non-terminal
   566    587   symbols that do no have their own datatype defined using a separate
   567         -%type directive.  See the documentation on %type below for addition
   568         -information.</p>
          588  +<a href='#ptype'>%type</a> directive.  
          589  +</p>
   569    590   
          591  +<a name='destructor'></a>
   570    592   <h4>The <tt>%destructor</tt> directive</h4>
   571    593   
   572    594   <p>The %destructor directive is used to specify a destructor for
   573    595   a non-terminal symbol.
   574         -(See also the %token_destructor directive which is used to
   575         -specify a destructor for terminal symbols.)</p>
          596  +(See also the <a href='#token_destructor'>%token_destructor</a>
          597  +directive which is used to specify a destructor for terminal symbols.)</p>
   576    598   
   577    599   <p>A non-terminal's destructor is called to dispose of the
   578    600   non-terminal's value whenever the non-terminal is popped from
   579    601   the stack.  This includes all of the following circumstances:
   580    602   <ul>
   581    603   <li> When a rule reduces and the value of a non-terminal on
   582    604        the right-hand side is not linked to C code.
................................................................................
   591    613   <pre>
   592    614      %type nt {void*}
   593    615      %destructor nt { free($$); }
   594    616      nt(A) ::= ID NUM.   { A = malloc( 100 ); }
   595    617   </pre>
   596    618   This example is a bit contrived but it serves to illustrate how
   597    619   destructors work.  The example shows a non-terminal named
   598         -``nt'' that holds values of type ``void*''.  When the rule for
   599         -an ``nt'' reduces, it sets the value of the non-terminal to
          620  +"nt" that holds values of type "void*".  When the rule for
          621  +an "nt" reduces, it sets the value of the non-terminal to
   600    622   space obtained from malloc().  Later, when the nt non-terminal
   601    623   is popped from the stack, the destructor will fire and call
   602    624   free() on this malloced space, thus avoiding a memory leak.
   603         -(Note that the symbol ``$$'' in the destructor code is replaced
          625  +(Note that the symbol "$$" in the destructor code is replaced
   604    626   by the value of the non-terminal.)</p>
   605    627   
   606    628   <p>It is important to note that the value of a non-terminal is passed
   607    629   to the destructor whenever the non-terminal is removed from the
   608    630   stack, unless the non-terminal is used in a C-code action.  If
   609    631   the non-terminal is used by C-code, then it is assumed that the
   610         -C-code will take care of destroying it if it should really
   611         -be destroyed.  More commonly, the value is used to build some
          632  +C-code will take care of destroying it.
          633  +More commonly, the value is used to build some
   612    634   larger structure and we don't want to destroy it, which is why
   613    635   the destructor is not called in this circumstance.</p>
   614    636   
   615         -<p>By appropriate use of destructors, it is possible to
   616         -build a parser using Lemon that can be used within a long-running
   617         -program, such as a GUI, that will not leak memory or other resources.
          637  +<p>Destructors help avoid memory leaks by automatically freeing
          638  +allocated objects when they go out of scope.
   618    639   To do the same using yacc or bison is much more difficult.</p>
   619    640   
          641  +<a name="extraarg"></a>
   620    642   <h4>The <tt>%extra_argument</tt> directive</h4>
   621    643   
   622    644   The %extra_argument directive instructs Lemon to add a 4th parameter
   623    645   to the parameter list of the Parse() function it generates.  Lemon
   624    646   doesn't do anything itself with this extra argument, but it does
   625    647   make the argument available to C-code action routines, destructors,
   626    648   and so forth.  For example, if the grammar file contains:</p>
   627    649   
   628    650   <p><pre>
   629    651       %extra_argument { MyStruct *pAbc }
   630    652   </pre></p>
   631    653   
   632    654   <p>Then the Parse() function generated will have an 4th parameter
   633         -of type ``MyStruct*'' and all action routines will have access to
   634         -a variable named ``pAbc'' that is the value of the 4th parameter
          655  +of type "MyStruct*" and all action routines will have access to
          656  +a variable named "pAbc" that is the value of the 4th parameter
   635    657   in the most recent call to Parse().</p>
   636    658   
          659  +<a name='pfallback'></a>
          660  +<h4>The <tt>%fallback</tt> directive</h4>
          661  +
          662  +<p>The %fallback directive specifies an alternative meaning for one
          663  +or more tokens.  The alternative meaning is tried if the original token
          664  +would have generated a syntax error.
          665  +
          666  +<p>The %fallback directive was added to support robust parsing of SQL
          667  +syntax in <a href="https://www.sqlite.org/">SQLite</a>.
          668  +The SQL language contains a large assortment of keywords, each of which
          669  +appears as a different token to the language parser.  SQL contains so
          670  +many keywords, that it can be difficult for programmers to keep up with
          671  +them all.  Programmers will, therefore, sometimes mistakenly use an
          672  +obscure language keyword for an identifier.  The %fallback directive
          673  +provides a mechanism to tell the parser:  "If you are unable to parse
          674  +this keyword, try treating it as an identifier instead."
          675  +
          676  +<p>The syntax of %fallback is as follows:
          677  +
          678  +<blockquote>
          679  +<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
          680  +</blockquote>
          681  +
          682  +<p>In words, the %fallback directive is followed by a list of token names
          683  +terminated by a period.  The first token name is the fallback token - the
          684  +token to which all the other tokens fall back to.  The second and subsequent
          685  +arguments are tokens which fall back to the token identified by the first
          686  +argument.
          687  +
          688  +<a name='pifdef'></a>
          689  +<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
          690  +
          691  +<p>The %ifdef, %ifndef, and %endif directives are similar to
          692  +#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
          693  +Each of these directives must begin at the left margin.  No whitespace
          694  +is allowed between the "%" and the directive name.
          695  +
          696  +<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
          697  +ignored unless the "-DMACRO" command-line option is used.  Grammar text
          698  +betwen "%ifndef MACRO" and the next nested "%endif" is included except when
          699  +the "-DMACRO" command-line option is used.
          700  +
          701  +<p>Note that the argument to %ifdef and %ifndef must be a single 
          702  +preprocessor symbol name, not a general expression.  There is no "%else"
          703  +directive.
          704  +
          705  +
          706  +<a name='pinclude'></a>
   637    707   <h4>The <tt>%include</tt> directive</h4>
   638    708   
   639    709   <p>The %include directive specifies C code that is included at the
   640    710   top of the generated parser.  You can include any text you want --
   641    711   the Lemon parser generator copies it blindly.  If you have multiple
   642         -%include directives in your grammar file the value of the last
   643         -%include directive overwrites all the others.</p.
          712  +%include directives in your grammar file, their values are concatenated
          713  +so that all %include code ultimately appears near the top of the
          714  +generated parser, in the same order as it appeared in the grammer.</p>
   644    715   
   645    716   <p>The %include directive is very handy for getting some extra #include
   646    717   preprocessor statements at the beginning of the generated parser.
   647    718   For example:</p>
   648    719   
   649    720   <p><pre>
   650    721      %include {#include &lt;unistd.h&gt;}
   651    722   </pre></p>
   652    723   
   653    724   <p>This might be needed, for example, if some of the C actions in the
   654    725   grammar call functions that are prototyed in unistd.h.</p>
   655    726   
          727  +<a name='pleft'></a>
   656    728   <h4>The <tt>%left</tt> directive</h4>
   657    729   
   658         -The %left directive is used (along with the %right and
   659         -%nonassoc directives) to declare precedences of terminal
   660         -symbols.  Every terminal symbol whose name appears after
   661         -a %left directive but before the next period (``.'') is
          730  +The %left directive is used (along with the <a href='#pright'>%right</a> and
          731  +<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
          732  +terminal symbols.  Every terminal symbol whose name appears after
          733  +a %left directive but before the next period (".") is
   662    734   given the same left-associative precedence value.  Subsequent
   663    735   %left directives have higher precedence.  For example:</p>
   664    736   
   665    737   <p><pre>
   666    738      %left AND.
   667    739      %left OR.
   668    740      %nonassoc EQ NE GT GE LT LE.
................................................................................
   675    747   directive.</p>
   676    748   
   677    749   <p>LALR(1) grammars can get into a situation where they require
   678    750   a large amount of stack space if you make heavy use or right-associative
   679    751   operators.  For this reason, it is recommended that you use %left
   680    752   rather than %right whenever possible.</p>
   681    753   
          754  +<a name='pname'></a>
   682    755   <h4>The <tt>%name</tt> directive</h4>
   683    756   
   684    757   <p>By default, the functions generated by Lemon all begin with the
   685         -five-character string ``Parse''.  You can change this string to something
          758  +five-character string "Parse".  You can change this string to something
   686    759   different using the %name directive.  For instance:</p>
   687    760   
   688    761   <p><pre>
   689    762      %name Abcde
   690    763   </pre></p>
   691    764   
   692    765   <p>Putting this directive in the grammar file will cause Lemon to generate
................................................................................
   697    770   <li> AbcdeTrace(), and
   698    771   <li> Abcde().
   699    772   </ul>
   700    773   The %name directive allows you to generator two or more different
   701    774   parsers and link them all into the same executable.
   702    775   </p>
   703    776   
          777  +<a name='pnonassoc'></a>
   704    778   <h4>The <tt>%nonassoc</tt> directive</h4>
   705    779   
   706    780   <p>This directive is used to assign non-associative precedence to
   707         -one or more terminal symbols.  See the section on precedence rules
   708         -or on the %left directive for additional information.</p>
          781  +one or more terminal symbols.  See the section on 
          782  +<a href='#precrules'>precedence rules</a>
          783  +or on the <a href='#pleft'>%left</a> directive for additional information.</p>
   709    784   
          785  +<a name='parse_accept'></a>
   710    786   <h4>The <tt>%parse_accept</tt> directive</h4>
   711    787   
   712    788   <p>The %parse_accept directive specifies a block of C code that is
   713         -executed whenever the parser accepts its input string.  To ``accept''
          789  +executed whenever the parser accepts its input string.  To "accept"
   714    790   an input string means that the parser was able to process all tokens
   715    791   without error.</p>
   716    792   
   717    793   <p>For example:</p>
   718    794   
   719    795   <p><pre>
   720    796      %parse_accept {
   721    797         printf("parsing complete!\n");
   722    798      }
   723    799   </pre></p>
   724    800   
   725         -
          801  +<a name='parse_failure'></a>
   726    802   <h4>The <tt>%parse_failure</tt> directive</h4>
   727    803   
   728    804   <p>The %parse_failure directive specifies a block of C code that
   729    805   is executed whenever the parser fails complete.  This code is not
   730    806   executed until the parser has tried and failed to resolve an input
   731    807   error using is usual error recovery strategy.  The routine is
   732    808   only invoked when parsing is unable to continue.</p>
................................................................................
   733    809   
   734    810   <p><pre>
   735    811      %parse_failure {
   736    812        fprintf(stderr,"Giving up.  Parser is hopelessly lost...\n");
   737    813      }
   738    814   </pre></p>
   739    815   
          816  +<a name='pright'></a>
   740    817   <h4>The <tt>%right</tt> directive</h4>
   741    818   
   742    819   <p>This directive is used to assign right-associative precedence to
   743         -one or more terminal symbols.  See the section on precedence rules
   744         -or on the %left directive for additional information.</p>
          820  +one or more terminal symbols.  See the section on 
          821  +<a href='#precrules'>precedence rules</a>
          822  +or on the <a href='#pleft'>%left</a> directive for additional information.</p>
   745    823   
          824  +<a name='stack_overflow'></a>
   746    825   <h4>The <tt>%stack_overflow</tt> directive</h4>
   747    826   
   748    827   <p>The %stack_overflow directive specifies a block of C code that
   749    828   is executed if the parser's internal stack ever overflows.  Typically
   750    829   this just prints an error message.  After a stack overflow, the parser
   751    830   will be unable to continue and must be reset.</p>
   752    831   
................................................................................
   767    846   </pre>
   768    847   Not like this:
   769    848   <pre>
   770    849      list ::= element list.      // right-recursion.  Bad!
   771    850      list ::= .
   772    851   </pre>
   773    852   
          853  +<a name='stack_size'></a>
   774    854   <h4>The <tt>%stack_size</tt> directive</h4>
   775    855   
   776    856   <p>If stack overflow is a problem and you can't resolve the trouble
   777    857   by using left-recursion, then you might want to increase the size
   778    858   of the parser's stack using this directive.  Put an positive integer
   779    859   after the %stack_size directive and Lemon will generate a parse
   780    860   with a stack of the requested size.  The default value is 100.</p>
   781    861   
   782    862   <p><pre>
   783    863      %stack_size 2000
   784    864   </pre></p>
   785    865   
          866  +<a name='start_symbol'></a>
   786    867   <h4>The <tt>%start_symbol</tt> directive</h4>
   787    868   
   788    869   <p>By default, the start-symbol for the grammar that Lemon generates
   789    870   is the first non-terminal that appears in the grammar file.  But you
   790    871   can choose a different start-symbol using the %start_symbol directive.</p>
   791    872   
   792    873   <p><pre>
   793    874      %start_symbol  prog
   794    875   </pre></p>
   795    876   
          877  +<a name='token_destructor'></a>
   796    878   <h4>The <tt>%token_destructor</tt> directive</h4>
   797    879   
   798    880   <p>The %destructor directive assigns a destructor to a non-terminal
   799    881   symbol.  (See the description of the %destructor directive above.)
   800    882   This directive does the same thing for all terminal symbols.</p>
   801    883   
   802    884   <p>Unlike non-terminal symbols which may each have a different data type
   803    885   for their values, terminals all use the same data type (defined by
   804    886   the %token_type directive) and so they use a common destructor.  Other
   805    887   than that, the token destructor works just like the non-terminal
   806    888   destructors.</p>
   807    889   
          890  +<a name='token_prefix'></a>
   808    891   <h4>The <tt>%token_prefix</tt> directive</h4>
   809    892   
   810    893   <p>Lemon generates #defines that assign small integer constants
   811    894   to each terminal symbol in the grammar.  If desired, Lemon will
   812    895   add a prefix specified by this directive
   813    896   to each of the #defines it generates.
   814    897   So if the default output of Lemon looked like this:
................................................................................
   826    909   <pre>
   827    910       #define TOKEN_AND        1
   828    911       #define TOKEN_MINUS      2
   829    912       #define TOKEN_OR         3
   830    913       #define TOKEN_PLUS       4
   831    914   </pre>
   832    915   
          916  +<a name='token_type'></a><a name='ptype'></a>
   833    917   <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
   834    918   
   835    919   <p>These directives are used to specify the data types for values
   836    920   on the parser's stack associated with terminal and non-terminal
   837    921   symbols.  The values of all terminal symbols must be of the same
   838    922   type.  This turns out to be the same data type as the 3rd parameter
   839    923   to the Parse() function generated by Lemon.  Typically, you will
................................................................................
   841    925   token structure.  Like this:</p>
   842    926   
   843    927   <p><pre>
   844    928      %token_type    {Token*}
   845    929   </pre></p>
   846    930   
   847    931   <p>If the data type of terminals is not specified, the default value
   848         -is ``int''.</p>
          932  +is "int".</p>
   849    933   
   850    934   <p>Non-terminal symbols can each have their own data types.  Typically
   851    935   the data type  of a non-terminal is a pointer to the root of a parse-tree
   852    936   structure that contains all information about that non-terminal.
   853    937   For example:</p>
   854    938   
   855    939   <p><pre>
................................................................................
   862    946   on what the corresponding non-terminal or terminal symbol is.  But
   863    947   the grammar designer should keep in mind that the size of the union
   864    948   will be the size of its largest element.  So if you have a single
   865    949   non-terminal whose data type requires 1K of storage, then your 100
   866    950   entry parser stack will require 100K of heap space.  If you are willing
   867    951   and able to pay that price, fine.  You just need to know.</p>
   868    952   
          953  +<a name='pwildcard'></a>
          954  +<h4>The <tt>%wildcard</tt> directive</h4>
          955  +
          956  +<p>The %wildcard directive is followed by a single token name and a
          957  +period.  This directive specifies that the identified token should 
          958  +match any input token.
          959  +
          960  +<p>When the generated parser has the choice of matching an input against
          961  +the wildcard token and some other token, the other token is always used.
          962  +The wildcard token is only matched if there are no other alternatives.
          963  +
   869    964   <h3>Error Processing</h3>
   870    965   
   871    966   <p>After extensive experimentation over several years, it has been
   872    967   discovered that the error recovery strategy used by yacc is about
   873    968   as good as it gets.  And so that is what Lemon uses.</p>
   874    969   
   875    970   <p>When a Lemon-generated parser encounters a syntax error, it
   876    971   first invokes the code specified by the %syntax_error directive, if
   877    972   any.  It then enters its error recovery strategy.  The error recovery
   878    973   strategy is to begin popping the parsers stack until it enters a
   879    974   state where it is permitted to shift a special non-terminal symbol
   880         -named ``error''.  It then shifts this non-terminal and continues
          975  +named "error".  It then shifts this non-terminal and continues
   881    976   parsing.  But the %syntax_error routine will not be called again
   882    977   until at least three new tokens have been successfully shifted.</p>
   883    978   
   884    979   <p>If the parser pops its stack until the stack is empty, and it still
   885    980   is unable to shift the error symbol, then the %parse_failed routine
   886    981   is invoked and the parser resets itself to its start state, ready
   887    982   to begin parsing a new file.  This is what will happen at the very
   888    983   first syntax error, of course, if there are no instances of the 
   889         -``error'' non-terminal in your grammar.</p>
          984  +"error" non-terminal in your grammar.</p>
   890    985   
   891    986   </body>
   892    987   </html>

Changes to ext/fts3/fts3Int.h.

    13     13   */
    14     14   #ifndef _FTSINT_H
    15     15   #define _FTSINT_H
    16     16   
    17     17   #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
    18     18   # define NDEBUG 1
    19     19   #endif
           20  +
           21  +/* FTS3/FTS4 require virtual tables */
           22  +#ifdef SQLITE_OMIT_VIRTUALTABLE
           23  +# undef SQLITE_ENABLE_FTS3
           24  +# undef SQLITE_ENABLE_FTS4
           25  +#endif
    20     26   
    21     27   /*
    22     28   ** FTS4 is really an extension for FTS3.  It is enabled using the
    23     29   ** SQLITE_ENABLE_FTS3 macro.  But to avoid confusion we also all
    24     30   ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
    25     31   */
    26     32   #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)

Changes to ext/fts3/fts3_test.c.

   522    522     Tcl_Interp *interp,
   523    523     int objc,
   524    524     Tcl_Obj *CONST objv[]
   525    525   ){
   526    526   #ifdef SQLITE_ENABLE_FTS3
   527    527     char aBuf[24];
   528    528     int rc;
   529         -  Tcl_WideInt w, w2;
          529  +  Tcl_WideInt w;
          530  +  sqlite3_int64 w2;
   530    531     int nByte, nByte2;
   531    532   
   532    533     if( objc!=2 ){
   533    534       Tcl_WrongNumArgs(interp, 1, objv, "INTEGER");
   534    535       return TCL_ERROR;
   535    536     }
   536    537   

Changes to ext/fts3/fts3_tokenizer.c.

    24     24   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    25     25   */
    26     26   #include "fts3Int.h"
    27     27   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    28     28   
    29     29   #include <assert.h>
    30     30   #include <string.h>
           31  +
           32  +/*
           33  +** Return true if the two-argument version of fts3_tokenizer()
           34  +** has been activated via a prior call to sqlite3_db_config(db,
           35  +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
           36  +*/
           37  +static int fts3TokenizerEnabled(sqlite3_context *context){
           38  +  sqlite3 *db = sqlite3_context_db_handle(context);
           39  +  int isEnabled = 0;
           40  +  sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled);
           41  +  return isEnabled;
           42  +}
    31     43   
    32     44   /*
    33     45   ** Implementation of the SQL scalar function for accessing the underlying 
    34     46   ** hash table. This function may be called as follows:
    35     47   **
    36     48   **   SELECT <function-name>(<key-name>);
    37     49   **   SELECT <function-name>(<key-name>, <pointer>);
................................................................................
    45     57   ** the string <key-name> must already exist in the has table. Otherwise,
    46     58   ** an error is returned.
    47     59   **
    48     60   ** Whether or not the <pointer> argument is specified, the value returned
    49     61   ** is a blob containing the pointer stored as the hash data corresponding
    50     62   ** to string <key-name> (after the hash-table is updated, if applicable).
    51     63   */
    52         -static void scalarFunc(
           64  +static void fts3TokenizerFunc(
    53     65     sqlite3_context *context,
    54     66     int argc,
    55     67     sqlite3_value **argv
    56     68   ){
    57     69     Fts3Hash *pHash;
    58     70     void *pPtr = 0;
    59     71     const unsigned char *zName;
................................................................................
    63     75   
    64     76     pHash = (Fts3Hash *)sqlite3_user_data(context);
    65     77   
    66     78     zName = sqlite3_value_text(argv[0]);
    67     79     nName = sqlite3_value_bytes(argv[0])+1;
    68     80   
    69     81     if( argc==2 ){
    70         -    void *pOld;
    71         -    int n = sqlite3_value_bytes(argv[1]);
    72         -    if( zName==0 || n!=sizeof(pPtr) ){
    73         -      sqlite3_result_error(context, "argument type mismatch", -1);
    74         -      return;
    75         -    }
    76         -    pPtr = *(void **)sqlite3_value_blob(argv[1]);
    77         -    pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
    78         -    if( pOld==pPtr ){
    79         -      sqlite3_result_error(context, "out of memory", -1);
           82  +    if( fts3TokenizerEnabled(context) ){
           83  +      void *pOld;
           84  +      int n = sqlite3_value_bytes(argv[1]);
           85  +      if( zName==0 || n!=sizeof(pPtr) ){
           86  +        sqlite3_result_error(context, "argument type mismatch", -1);
           87  +        return;
           88  +      }
           89  +      pPtr = *(void **)sqlite3_value_blob(argv[1]);
           90  +      pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
           91  +      if( pOld==pPtr ){
           92  +        sqlite3_result_error(context, "out of memory", -1);
           93  +      }
           94  +    }else{
           95  +      sqlite3_result_error(context, "fts3tokenize disabled", -1);
    80     96         return;
    81     97       }
    82     98     }else{
    83     99       if( zName ){
    84    100         pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
    85    101       }
    86    102       if( !pPtr ){
    87    103         char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
    88    104         sqlite3_result_error(context, zErr, -1);
    89    105         sqlite3_free(zErr);
    90    106         return;
    91    107       }
    92    108     }
    93         -
    94    109     sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
    95    110   }
    96    111   
    97    112   int sqlite3Fts3IsIdChar(char c){
    98    113     static const char isFtsIdChar[] = {
    99    114         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
   100    115         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
................................................................................
   345    360   
   346    361     sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
   347    362     sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
   348    363     sqlite3_step(pStmt);
   349    364   
   350    365     return sqlite3_finalize(pStmt);
   351    366   }
          367  +
   352    368   
   353    369   static
   354    370   int queryTokenizer(
   355    371     sqlite3 *db, 
   356    372     char *zName,  
   357    373     const sqlite3_tokenizer_module **pp
   358    374   ){
................................................................................
   416    432     assert( p1==p2 );
   417    433     rc = queryTokenizer(db, "nosuchtokenizer", &p2);
   418    434     assert( rc==SQLITE_ERROR );
   419    435     assert( p2==0 );
   420    436     assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
   421    437   
   422    438     /* Test the storage function */
   423         -  rc = registerTokenizer(db, "nosuchtokenizer", p1);
   424         -  assert( rc==SQLITE_OK );
   425         -  rc = queryTokenizer(db, "nosuchtokenizer", &p2);
   426         -  assert( rc==SQLITE_OK );
   427         -  assert( p2==p1 );
          439  +  if( fts3TokenizerEnabled(context) ){
          440  +    rc = registerTokenizer(db, "nosuchtokenizer", p1);
          441  +    assert( rc==SQLITE_OK );
          442  +    rc = queryTokenizer(db, "nosuchtokenizer", &p2);
          443  +    assert( rc==SQLITE_OK );
          444  +    assert( p2==p1 );
          445  +  }
   428    446   
   429    447     sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
   430    448   }
   431    449   
   432    450   #endif
   433    451   
   434    452   /*
................................................................................
   436    454   ** the hash table pointed to by argument pHash. The hash table must
   437    455   ** been initialized to use string keys, and to take a private copy 
   438    456   ** of the key when a value is inserted. i.e. by a call similar to:
   439    457   **
   440    458   **    sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
   441    459   **
   442    460   ** This function adds a scalar function (see header comment above
   443         -** scalarFunc() in this file for details) and, if ENABLE_TABLE is
          461  +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is
   444    462   ** defined at compilation time, a temporary virtual table (see header 
   445    463   ** comment above struct HashTableVtab) to the database schema. Both 
   446    464   ** provide read/write access to the contents of *pHash.
   447    465   **
   448    466   ** The third argument to this function, zName, is used as the name
   449    467   ** of both the scalar and, if created, the virtual table.
   450    468   */
................................................................................
   465    483     zTest2 = sqlite3_mprintf("%s_internal_test", zName);
   466    484     if( !zTest || !zTest2 ){
   467    485       rc = SQLITE_NOMEM;
   468    486     }
   469    487   #endif
   470    488   
   471    489     if( SQLITE_OK==rc ){
   472         -    rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
          490  +    rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0);
   473    491     }
   474    492     if( SQLITE_OK==rc ){
   475         -    rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
          493  +    rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0);
   476    494     }
   477    495   #ifdef SQLITE_TEST
   478    496     if( SQLITE_OK==rc ){
   479    497       rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0);
   480    498     }
   481    499     if( SQLITE_OK==rc ){
   482    500       rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);

Changes to ext/fts3/fts3_write.c.

   329    329   /* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'",
   330    330   
   331    331   /* This statement is used to determine which level to read the input from
   332    332   ** when performing an incremental merge. It returns the absolute level number
   333    333   ** of the oldest level in the db that contains at least ? segments. Or,
   334    334   ** if no level in the FTS index contains more than ? segments, the statement
   335    335   ** returns zero rows.  */
   336         -/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
          336  +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
          337  +         "  GROUP BY level HAVING cnt>=?"
   337    338            "  ORDER BY (level %% 1024) ASC LIMIT 1",
   338    339   
   339    340   /* Estimate the upper limit on the number of leaf nodes in a new segment
   340    341   ** created by merging the oldest :2 segments from absolute level :1. See 
   341    342   ** function sqlite3Fts3Incrmerge() for details.  */
   342    343   /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
   343    344            "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
................................................................................
  3190   3191     }
  3191   3192   
  3192   3193     if( iLevel==FTS3_SEGCURSOR_ALL ){
  3193   3194       /* This call is to merge all segments in the database to a single
  3194   3195       ** segment. The level of the new segment is equal to the numerically
  3195   3196       ** greatest segment level currently present in the database for this
  3196   3197       ** index. The idx of the new segment is always 0.  */
  3197         -    if( csr.nSegment==1 ){
         3198  +    if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){
  3198   3199         rc = SQLITE_DONE;
  3199   3200         goto finished;
  3200   3201       }
  3201   3202       iNewLevel = iMaxLevel;
  3202   3203       bIgnoreEmpty = 1;
  3203   3204   
  3204   3205     }else{
................................................................................
  4832   4833       /* Search the %_segdir table for the absolute level with the smallest
  4833   4834       ** relative level number that contains at least nMin segments, if any.
  4834   4835       ** If one is found, set iAbsLevel to the absolute level number and
  4835   4836       ** nSeg to nMin. If no level with at least nMin segments can be found, 
  4836   4837       ** set nSeg to -1.
  4837   4838       */
  4838   4839       rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
  4839         -    sqlite3_bind_int(pFindLevel, 1, nMin);
         4840  +    sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin));
  4840   4841       if( sqlite3_step(pFindLevel)==SQLITE_ROW ){
  4841   4842         iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
  4842         -      nSeg = nMin;
         4843  +      nSeg = sqlite3_column_int(pFindLevel, 1);
         4844  +      assert( nSeg>=2 );
  4843   4845       }else{
  4844   4846         nSeg = -1;
  4845   4847       }
  4846   4848       rc = sqlite3_reset(pFindLevel);
  4847   4849   
  4848   4850       /* If the hint read from the %_stat table is not empty, check if the
  4849   4851       ** last entry in it specifies a relative level smaller than or equal

Changes to ext/fts3/unicode/mkunicode.tcl.

   222    222     puts "** The results are undefined if the value passed to this function"
   223    223     puts "** is less than zero."
   224    224     puts "*/"
   225    225     puts "int ${zFunc}\(int c)\{"
   226    226     an_print_range_array $lRange
   227    227     an_print_ascii_bitmap $lRange
   228    228     puts {
   229         -  if( c<128 ){
          229  +  if( (unsigned int)c<128 ){
   230    230       return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   231         -  }else if( c<(1<<22) ){
          231  +  }else if( (unsigned int)c<(1<<22) ){
   232    232       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   233    233       int iRes = 0;
   234    234       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   235    235       int iLo = 0;
   236    236       while( iHi>=iLo ){
   237    237         int iTest = (iHi + iLo) / 2;
   238    238         if( key >= aEntry[iTest] ){

Changes to ext/fts5/fts5Int.h.

    22     22   #include <assert.h>
    23     23   
    24     24   #ifndef SQLITE_AMALGAMATION
    25     25   
    26     26   typedef unsigned char  u8;
    27     27   typedef unsigned int   u32;
    28     28   typedef unsigned short u16;
           29  +typedef short i16;
    29     30   typedef sqlite3_int64 i64;
    30     31   typedef sqlite3_uint64 u64;
    31     32   
    32         -#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
           33  +#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
    33     34   
    34     35   #define testcase(x)
    35     36   #define ALWAYS(x) 1
    36     37   #define NEVER(x) 0
    37     38   
    38     39   #define MIN(x,y) (((x) < (y)) ? (x) : (y))
    39     40   #define MAX(x,y) (((x) > (y)) ? (x) : (y))
................................................................................
    42     43   ** Constants for the largest and smallest possible 64-bit signed integers.
    43     44   */
    44     45   # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
    45     46   # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
    46     47   
    47     48   #endif
    48     49   
           50  +/* Truncate very long tokens to this many bytes. Hard limit is 
           51  +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
           52  +** field that occurs at the start of each leaf page (see fts5_index.c). */
           53  +#define FTS5_MAX_TOKEN_SIZE 32768
    49     54   
    50     55   /*
    51     56   ** Maximum number of prefix indexes on single FTS5 table. This must be
    52     57   ** less than 32. If it is set to anything large than that, an #error
    53     58   ** directive in fts5_index.c will cause the build to fail.
    54     59   */
    55     60   #define FTS5_MAX_PREFIX_INDEXES 31
................................................................................
    75     80   */
    76     81   #ifdef SQLITE_DEBUG
    77     82   extern int sqlite3_fts5_may_be_corrupt;
    78     83   # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
    79     84   #else
    80     85   # define assert_nc(x) assert(x)
    81     86   #endif
           87  +
           88  +/* Mark a function parameter as unused, to suppress nuisance compiler
           89  +** warnings. */
           90  +#ifndef UNUSED_PARAM
           91  +# define UNUSED_PARAM(X)  (void)(X)
           92  +#endif
           93  +
           94  +#ifndef UNUSED_PARAM2
           95  +# define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
           96  +#endif
    82     97   
    83     98   typedef struct Fts5Global Fts5Global;
    84     99   typedef struct Fts5Colset Fts5Colset;
    85    100   
    86    101   /* If a NEAR() clump or phrase may only match a specific set of columns, 
    87    102   ** then an object of the following type is used to record the set of columns.
    88    103   ** Each entry in the aiCol[] array is a column that may be matched.
................................................................................
   157    172     fts5_tokenizer *pTokApi;
   158    173   
   159    174     /* Values loaded from the %_config table */
   160    175     int iCookie;                    /* Incremented when %_config is modified */
   161    176     int pgsz;                       /* Approximate page size used in %_data */
   162    177     int nAutomerge;                 /* 'automerge' setting */
   163    178     int nCrisisMerge;               /* Maximum allowed segments per level */
          179  +  int nUsermerge;                 /* 'usermerge' setting */
   164    180     int nHashSize;                  /* Bytes of memory for in-memory hash */
   165    181     char *zRank;                    /* Name of rank function */
   166    182     char *zRankArgs;                /* Arguments to rank function */
   167    183   
   168    184     /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   169    185     char **pzErrmsg;
   170    186   
................................................................................
   221    237   
   222    238   /*
   223    239   ** Buffer object for the incremental building of string data.
   224    240   */
   225    241   typedef struct Fts5Buffer Fts5Buffer;
   226    242   struct Fts5Buffer {
   227    243     u8 *p;
   228         -  u32 n;
   229         -  u32 nSpace;
          244  +  int n;
          245  +  int nSpace;
   230    246   };
   231    247   
   232    248   int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
   233    249   void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
   234    250   void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
   235    251   void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
   236    252   void sqlite3Fts5BufferFree(Fts5Buffer*);
................................................................................
   243    259   #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
   244    260   #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
   245    261   #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
   246    262   #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
   247    263   #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
   248    264   
   249    265   #define fts5BufferGrow(pRc,pBuf,nn) ( \
   250         -  (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \
          266  +  (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
   251    267       sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
   252    268   )
   253    269   
   254    270   /* Write and decode big-endian 32-bit integer values */
   255    271   void sqlite3Fts5Put32(u8*, int);
   256    272   int sqlite3Fts5Get32(const u8*);
   257    273   
................................................................................
   278    294   int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
   279    295   
   280    296   typedef struct Fts5PoslistWriter Fts5PoslistWriter;
   281    297   struct Fts5PoslistWriter {
   282    298     i64 iPrev;
   283    299   };
   284    300   int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
          301  +void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
   285    302   
   286    303   int sqlite3Fts5PoslistNext64(
   287    304     const u8 *a, int n,             /* Buffer containing poslist */
   288    305     int *pi,                        /* IN/OUT: Offset within a[] */
   289    306     i64 *piOff                      /* IN/OUT: Current offset */
   290    307   );
   291    308   
................................................................................
   311    328   ** Interface to code in fts5_index.c. fts5_index.c contains contains code
   312    329   ** to access the data stored in the %_data table.
   313    330   */
   314    331   
   315    332   typedef struct Fts5Index Fts5Index;
   316    333   typedef struct Fts5IndexIter Fts5IndexIter;
   317    334   
          335  +struct Fts5IndexIter {
          336  +  i64 iRowid;
          337  +  const u8 *pData;
          338  +  int nData;
          339  +  u8 bEof;
          340  +};
          341  +
          342  +#define sqlite3Fts5IterEof(x) ((x)->bEof)
          343  +
   318    344   /*
   319    345   ** Values used as part of the flags argument passed to IndexQuery().
   320    346   */
   321    347   #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
   322    348   #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
   323    349   #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
   324    350   #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
   325    351   
          352  +/* The following are used internally by the fts5_index.c module. They are
          353  +** defined here only to make it easier to avoid clashes with the flags
          354  +** above. */
          355  +#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
          356  +#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
          357  +
   326    358   /*
   327    359   ** Create/destroy an Fts5Index object.
   328    360   */
   329    361   int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
   330    362   int sqlite3Fts5IndexClose(Fts5Index *p);
   331    363   
   332         -/*
   333         -** for(
   334         -**   sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
   335         -**   0==sqlite3Fts5IterEof(pIter);
   336         -**   sqlite3Fts5IterNext(pIter)
   337         -** ){
   338         -**   i64 iRowid = sqlite3Fts5IterRowid(pIter);
   339         -** }
   340         -*/
   341         -
   342    364   /*
   343    365   ** Return a simple checksum value based on the arguments.
   344    366   */
   345    367   u64 sqlite3Fts5IndexEntryCksum(
   346    368     i64 iRowid, 
   347    369     int iCol, 
   348    370     int iPos, 
................................................................................
   374    396     Fts5IndexIter **ppIter          /* OUT: New iterator object */
   375    397   );
   376    398   
   377    399   /*
   378    400   ** The various operations on open token or token prefix iterators opened
   379    401   ** using sqlite3Fts5IndexQuery().
   380    402   */
   381         -int sqlite3Fts5IterEof(Fts5IndexIter*);
   382    403   int sqlite3Fts5IterNext(Fts5IndexIter*);
   383    404   int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
   384         -i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
   385         -int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*);
   386         -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf);
   387    405   
   388    406   /*
   389    407   ** Close an iterator opened by sqlite3Fts5IndexQuery().
   390    408   */
   391    409   void sqlite3Fts5IterClose(Fts5IndexIter*);
   392    410   
   393    411   /*
................................................................................
   462    480   ** this connection since it was created.
   463    481   */
   464    482   int sqlite3Fts5IndexReads(Fts5Index *p);
   465    483   
   466    484   int sqlite3Fts5IndexReinit(Fts5Index *p);
   467    485   int sqlite3Fts5IndexOptimize(Fts5Index *p);
   468    486   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
          487  +int sqlite3Fts5IndexReset(Fts5Index *p);
   469    488   
   470    489   int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
   471    490   
   472         -int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);
   473         -
   474    491   /*
   475    492   ** End of interface to code in fts5_index.c.
   476    493   **************************************************************************/
   477    494   
   478    495   /**************************************************************************
   479    496   ** Interface to code in fts5_varint.c. 
   480    497   */
................................................................................
   606    623       Fts5Storage *p, const char*, sqlite3_value*, int
   607    624   );
   608    625   
   609    626   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   610    627   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
   611    628   int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   612    629   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
          630  +int sqlite3Fts5StorageReset(Fts5Storage *p);
   613    631   
   614    632   /*
   615    633   ** End of interface to code in fts5_storage.c.
   616    634   **************************************************************************/
   617    635   
   618    636   
   619    637   /**************************************************************************
................................................................................
   666    684   Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
   667    685   int sqlite3Fts5ExprPopulatePoslists(
   668    686       Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
   669    687   );
   670    688   void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
   671    689   void sqlite3Fts5ExprClearEof(Fts5Expr*);
   672    690   
   673         -int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
          691  +int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
   674    692   
   675    693   int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
   676    694   
   677    695   /*******************************************
   678    696   ** The fts5_expr.c API above this point is used by the other hand-written
   679    697   ** C code in this module. The interfaces below this point are called by
   680    698   ** the parser code in fts5parse.y.  */
................................................................................
   684    702   Fts5ExprNode *sqlite3Fts5ParseNode(
   685    703     Fts5Parse *pParse,
   686    704     int eType,
   687    705     Fts5ExprNode *pLeft,
   688    706     Fts5ExprNode *pRight,
   689    707     Fts5ExprNearset *pNear
   690    708   );
          709  +
          710  +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
          711  +  Fts5Parse *pParse,
          712  +  Fts5ExprNode *pLeft,
          713  +  Fts5ExprNode *pRight
          714  +);
   691    715   
   692    716   Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   693    717     Fts5Parse *pParse, 
   694    718     Fts5ExprPhrase *pPhrase, 
   695    719     Fts5Token *pToken,
   696    720     int bPrefix
   697    721   );

Changes to ext/fts5/fts5_aux.c.

   153    153     int nToken,                     /* Size of token in bytes */
   154    154     int iStartOff,                  /* Start offset of token */
   155    155     int iEndOff                     /* End offset of token */
   156    156   ){
   157    157     HighlightContext *p = (HighlightContext*)pContext;
   158    158     int rc = SQLITE_OK;
   159    159     int iPos;
          160  +
          161  +  UNUSED_PARAM2(pToken, nToken);
   160    162   
   161    163     if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
   162    164     iPos = p->iPos++;
   163    165   
   164    166     if( p->iRangeEnd>0 ){
   165    167       if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
   166    168       if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
................................................................................
   387    389   */
   388    390   static int fts5CountCb(
   389    391     const Fts5ExtensionApi *pApi, 
   390    392     Fts5Context *pFts,
   391    393     void *pUserData                 /* Pointer to sqlite3_int64 variable */
   392    394   ){
   393    395     sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
          396  +  UNUSED_PARAM2(pApi, pFts);
   394    397     (*pn)++;
   395    398     return SQLITE_OK;
   396    399   }
   397    400   
   398    401   /*
   399    402   ** Set *ppData to point to the Fts5Bm25Data object for the current query. 
   400    403   ** If the object has not already been allocated, allocate and populate it
................................................................................
   540    543       { "snippet",   0, fts5SnippetFunction, 0 },
   541    544       { "highlight", 0, fts5HighlightFunction, 0 },
   542    545       { "bm25",      0, fts5Bm25Function,    0 },
   543    546     };
   544    547     int rc = SQLITE_OK;             /* Return code */
   545    548     int i;                          /* To iterate through builtin functions */
   546    549   
   547         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
          550  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
   548    551       rc = pApi->xCreateFunction(pApi,
   549    552           aBuiltin[i].zFunc,
   550    553           aBuiltin[i].pUserData,
   551    554           aBuiltin[i].xFunc,
   552    555           aBuiltin[i].xDestroy
   553    556       );
   554    557     }
   555    558   
   556    559     return rc;
   557    560   }
   558    561   
   559    562   

Changes to ext/fts5/fts5_buffer.c.

    12     12   */
    13     13   
    14     14   
    15     15   
    16     16   #include "fts5Int.h"
    17     17   
    18     18   int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
    19         -  u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
    20         -  u8 *pNew;
    21         -  while( nNew<nByte ){
    22         -    nNew = nNew * 2;
    23         -  }
    24         -  pNew = sqlite3_realloc(pBuf->p, nNew);
    25         -  if( pNew==0 ){
    26         -    *pRc = SQLITE_NOMEM;
    27         -    return 1;
    28         -  }else{
    29         -    pBuf->nSpace = nNew;
    30         -    pBuf->p = pNew;
           19  +  if( (u32)pBuf->nSpace<nByte ){
           20  +    u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
           21  +    u8 *pNew;
           22  +    while( nNew<nByte ){
           23  +      nNew = nNew * 2;
           24  +    }
           25  +    pNew = sqlite3_realloc(pBuf->p, nNew);
           26  +    if( pNew==0 ){
           27  +      *pRc = SQLITE_NOMEM;
           28  +      return 1;
           29  +    }else{
           30  +      pBuf->nSpace = nNew;
           31  +      pBuf->p = pNew;
           32  +    }
    31     33     }
    32     34     return 0;
    33     35   }
    34     36   
    35     37   
    36     38   /*
    37     39   ** Encode value iVal as an SQLite varint and append it to the buffer object
................................................................................
   203    205   ){
   204    206     memset(pIter, 0, sizeof(*pIter));
   205    207     pIter->a = a;
   206    208     pIter->n = n;
   207    209     sqlite3Fts5PoslistReaderNext(pIter);
   208    210     return pIter->bEof;
   209    211   }
          212  +
          213  +/*
          214  +** Append position iPos to the position list being accumulated in buffer
          215  +** pBuf, which must be already be large enough to hold the new data.
          216  +** The previous position written to this list is *piPrev. *piPrev is set
          217  +** to iPos before returning.
          218  +*/
          219  +void sqlite3Fts5PoslistSafeAppend(
          220  +  Fts5Buffer *pBuf, 
          221  +  i64 *piPrev, 
          222  +  i64 iPos
          223  +){
          224  +  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
          225  +  if( (iPos & colmask) != (*piPrev & colmask) ){
          226  +    pBuf->p[pBuf->n++] = 1;
          227  +    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
          228  +    *piPrev = (iPos & colmask);
          229  +  }
          230  +  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
          231  +  *piPrev = iPos;
          232  +}
   210    233   
   211    234   int sqlite3Fts5PoslistWriterAppend(
   212    235     Fts5Buffer *pBuf, 
   213    236     Fts5PoslistWriter *pWriter,
   214    237     i64 iPos
   215    238   ){
   216         -  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
   217         -  int rc = SQLITE_OK;
   218         -  if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
   219         -    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
   220         -      pBuf->p[pBuf->n++] = 1;
   221         -      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
   222         -      pWriter->iPrev = (iPos & colmask);
   223         -    }
   224         -    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
   225         -    pWriter->iPrev = iPos;
   226         -  }
   227         -  return rc;
          239  +  int rc = 0;   /* Initialized only to suppress erroneous warning from Clang */
          240  +  if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
          241  +  sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
          242  +  return SQLITE_OK;
   228    243   }
   229    244   
   230    245   void *sqlite3Fts5MallocZero(int *pRc, int nByte){
   231    246     void *pRet = 0;
   232    247     if( *pRc==SQLITE_OK ){
   233    248       pRet = sqlite3_malloc(nByte);
   234    249       if( pRet==0 && nByte>0 ){
................................................................................
   318    333     const char *pTerm, int nTerm, 
   319    334     int *pbPresent
   320    335   ){
   321    336     int rc = SQLITE_OK;
   322    337     *pbPresent = 0;
   323    338     if( p ){
   324    339       int i;
   325         -    int hash = 13;
          340  +    u32 hash = 13;
   326    341       Fts5TermsetEntry *pEntry;
   327    342   
   328    343       /* Calculate a hash value for this term. This is the same hash checksum
   329    344       ** used by the fts5_hash.c module. This is not important for correct
   330    345       ** operation of the module, but is necessary to ensure that some tests
   331    346       ** designed to produce hash table collisions really do work.  */
   332    347       for(i=nTerm-1; i>=0; i--){
................................................................................
   335    350       hash = (hash << 3) ^ hash ^ iIdx;
   336    351       hash = hash % ArraySize(p->apHash);
   337    352   
   338    353       for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
   339    354         if( pEntry->iIdx==iIdx 
   340    355             && pEntry->nTerm==nTerm 
   341    356             && memcmp(pEntry->pTerm, pTerm, nTerm)==0 
   342         -        ){
          357  +      ){
   343    358           *pbPresent = 1;
   344    359           break;
   345    360         }
   346    361       }
   347    362   
   348    363       if( pEntry==0 ){
   349    364         pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
................................................................................
   359    374     }
   360    375   
   361    376     return rc;
   362    377   }
   363    378   
   364    379   void sqlite3Fts5TermsetFree(Fts5Termset *p){
   365    380     if( p ){
   366         -    int i;
          381  +    u32 i;
   367    382       for(i=0; i<ArraySize(p->apHash); i++){
   368    383         Fts5TermsetEntry *pEntry = p->apHash[i];
   369    384         while( pEntry ){
   370    385           Fts5TermsetEntry *pDel = pEntry;
   371    386           pEntry = pEntry->pNext;
   372    387           sqlite3_free(pDel);
   373    388         }
   374    389       }
   375    390       sqlite3_free(p);
   376    391     }
   377    392   }
   378         -
   379         -
   380         -

Changes to ext/fts5/fts5_config.c.

    14     14   */
    15     15   
    16     16   
    17     17   #include "fts5Int.h"
    18     18   
    19     19   #define FTS5_DEFAULT_PAGE_SIZE   4050
    20     20   #define FTS5_DEFAULT_AUTOMERGE      4
           21  +#define FTS5_DEFAULT_USERMERGE      4
    21     22   #define FTS5_DEFAULT_CRISISMERGE   16
    22     23   #define FTS5_DEFAULT_HASHSIZE    (1024*1024)
    23     24   
    24     25   /* Maximum allowed page size */
    25     26   #define FTS5_MAX_PAGE_SIZE (128*1024)
    26     27   
    27     28   static int fts5_iswhitespace(char x){
................................................................................
   202    203   typedef struct Fts5Enum Fts5Enum;
   203    204   
   204    205   static int fts5ConfigSetEnum(
   205    206     const Fts5Enum *aEnum, 
   206    207     const char *zEnum, 
   207    208     int *peVal
   208    209   ){
   209         -  int nEnum = strlen(zEnum);
          210  +  int nEnum = (int)strlen(zEnum);
   210    211     int i;
   211    212     int iVal = -1;
   212    213   
   213    214     for(i=0; aEnum[i].zName; i++){
   214    215       if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
   215    216         if( iVal>=0 ) return SQLITE_ERROR;
   216    217         iVal = aEnum[i].eVal;
................................................................................
   437    438       memcpy(zOut, zIn, nIn+1);
   438    439       if( fts5_isopenquote(zOut[0]) ){
   439    440         int ii = fts5Dequote(zOut);
   440    441         zRet = &zIn[ii];
   441    442         *pbQuoted = 1;
   442    443       }else{
   443    444         zRet = fts5ConfigSkipBareword(zIn);
   444         -      zOut[zRet-zIn] = '\0';
          445  +      if( zRet ){
          446  +        zOut[zRet-zIn] = '\0';
          447  +      }
   445    448       }
   446    449     }
   447    450   
   448    451     if( zRet==0 ){
   449    452       sqlite3_free(zOut);
   450    453     }else{
   451    454       *pzOut = zOut;
................................................................................
   852    855       if( nAutomerge<0 || nAutomerge>64 ){
   853    856         *pbBadkey = 1;
   854    857       }else{
   855    858         if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
   856    859         pConfig->nAutomerge = nAutomerge;
   857    860       }
   858    861     }
          862  +
          863  +  else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
          864  +    int nUsermerge = -1;
          865  +    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
          866  +      nUsermerge = sqlite3_value_int(pVal);
          867  +    }
          868  +    if( nUsermerge<2 || nUsermerge>16 ){
          869  +      *pbBadkey = 1;
          870  +    }else{
          871  +      pConfig->nUsermerge = nUsermerge;
          872  +    }
          873  +  }
   859    874   
   860    875     else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
   861    876       int nCrisisMerge = -1;
   862    877       if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
   863    878         nCrisisMerge = sqlite3_value_int(pVal);
   864    879       }
   865    880       if( nCrisisMerge<0 ){
................................................................................
   899    914     sqlite3_stmt *p = 0;
   900    915     int rc = SQLITE_OK;
   901    916     int iVersion = 0;
   902    917   
   903    918     /* Set default values */
   904    919     pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
   905    920     pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
          921  +  pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   906    922     pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   907    923     pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
   908    924   
   909    925     zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   910    926     if( zSql ){
   911    927       rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
   912    928       sqlite3_free(zSql);
................................................................................
   939    955     }
   940    956   
   941    957     if( rc==SQLITE_OK ){
   942    958       pConfig->iCookie = iCookie;
   943    959     }
   944    960     return rc;
   945    961   }
   946         -

Changes to ext/fts5/fts5_expr.c.

    58     58   **       FTS5_TERM                (pNear valid)
    59     59   */
    60     60   struct Fts5ExprNode {
    61     61     int eType;                      /* Node type */
    62     62     int bEof;                       /* True at EOF */
    63     63     int bNomatch;                   /* True if entry is not a match */
    64     64   
           65  +  /* Next method for this node. */
           66  +  int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
           67  +
    65     68     i64 iRowid;                     /* Current rowid */
    66     69     Fts5ExprNearset *pNear;         /* For FTS5_STRING - cluster of phrases */
    67     70   
    68     71     /* Child nodes. For a NOT node, this array always contains 2 entries. For 
    69     72     ** AND or OR nodes, it contains 2 or more entries.  */
    70     73     int nChild;                     /* Number of child nodes */
    71     74     Fts5ExprNode *apChild[1];       /* Array of child nodes */
    72     75   };
    73     76   
    74     77   #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING)
    75     78   
           79  +/*
           80  +** Invoke the xNext method of an Fts5ExprNode object. This macro should be
           81  +** used as if it has the same signature as the xNext() methods themselves.
           82  +*/
           83  +#define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d))
           84  +
    76     85   /*
    77     86   ** An instance of the following structure represents a single search term
    78     87   ** or term prefix.
    79     88   */
    80     89   struct Fts5ExprTerm {
    81     90     int bPrefix;                    /* True for a prefix term */
    82     91     char *zTerm;                    /* nul-terminated term */
................................................................................
   230    239     assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
   231    240     if( sParse.rc==SQLITE_OK ){
   232    241       *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
   233    242       if( pNew==0 ){
   234    243         sParse.rc = SQLITE_NOMEM;
   235    244         sqlite3Fts5ParseNodeFree(sParse.pExpr);
   236    245       }else{
   237         -      pNew->pRoot = sParse.pExpr;
          246  +      if( !sParse.pExpr ){
          247  +        const int nByte = sizeof(Fts5ExprNode);
          248  +        pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
          249  +        if( pNew->pRoot ){
          250  +          pNew->pRoot->bEof = 1;
          251  +        }
          252  +      }else{
          253  +        pNew->pRoot = sParse.pExpr;
          254  +      }
   238    255         pNew->pIndex = 0;
   239    256         pNew->pConfig = pConfig;
   240    257         pNew->apExprPhrase = sParse.apPhrase;
   241    258         pNew->nPhrase = sParse.nPhrase;
   242    259         sParse.apPhrase = 0;
   243    260       }
          261  +  }else{
          262  +    sqlite3Fts5ParseNodeFree(sParse.pExpr);
   244    263     }
   245    264   
   246    265     sqlite3_free(sParse.apPhrase);
   247    266     *pzErr = sParse.zErr;
   248    267     return sParse.rc;
   249    268   }
   250    269   
................................................................................
   282    301     int bRetValid = 0;
   283    302     Fts5ExprTerm *p;
   284    303   
   285    304     assert( pTerm->pSynonym );
   286    305     assert( bDesc==0 || bDesc==1 );
   287    306     for(p=pTerm; p; p=p->pSynonym){
   288    307       if( 0==sqlite3Fts5IterEof(p->pIter) ){
   289         -      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
          308  +      i64 iRowid = p->pIter->iRowid;
   290    309         if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){
   291    310           iRet = iRowid;
   292    311           bRetValid = 1;
   293    312         }
   294    313       }
   295    314     }
   296    315   
................................................................................
   299    318   }
   300    319   
   301    320   /*
   302    321   ** Argument pTerm must be a synonym iterator.
   303    322   */
   304    323   static int fts5ExprSynonymList(
   305    324     Fts5ExprTerm *pTerm, 
   306         -  int bCollist, 
   307         -  Fts5Colset *pColset,
   308    325     i64 iRowid,
   309         -  int *pbDel,                     /* OUT: Caller should sqlite3_free(*pa) */
          326  +  Fts5Buffer *pBuf,               /* Use this buffer for space if required */
   310    327     u8 **pa, int *pn
   311    328   ){
   312    329     Fts5PoslistReader aStatic[4];
   313    330     Fts5PoslistReader *aIter = aStatic;
   314    331     int nIter = 0;
   315    332     int nAlloc = 4;
   316    333     int rc = SQLITE_OK;
   317    334     Fts5ExprTerm *p;
   318    335   
   319    336     assert( pTerm->pSynonym );
   320    337     for(p=pTerm; p; p=p->pSynonym){
   321    338       Fts5IndexIter *pIter = p->pIter;
   322         -    if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
   323         -      const u8 *a;
   324         -      int n;
   325         -
   326         -      if( bCollist ){
   327         -        rc = sqlite3Fts5IterCollist(pIter, &a, &n);
   328         -      }else{
   329         -        i64 dummy;
   330         -        rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
   331         -      }
   332         -
   333         -      if( rc!=SQLITE_OK ) goto synonym_poslist_out;
   334         -      if( n==0 ) continue;
          339  +    if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){
          340  +      if( pIter->nData==0 ) continue;
   335    341         if( nIter==nAlloc ){
   336    342           int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
   337    343           Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   338    344           if( aNew==0 ){
   339    345             rc = SQLITE_NOMEM;
   340    346             goto synonym_poslist_out;
   341    347           }
   342    348           memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter);
   343    349           nAlloc = nAlloc*2;
   344    350           if( aIter!=aStatic ) sqlite3_free(aIter);
   345    351           aIter = aNew;
   346    352         }
   347         -      sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]);
          353  +      sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]);
   348    354         assert( aIter[nIter].bEof==0 );
   349    355         nIter++;
   350    356       }
   351    357     }
   352    358   
   353         -  assert( *pbDel==0 );
   354    359     if( nIter==1 ){
   355    360       *pa = (u8*)aIter[0].a;
   356    361       *pn = aIter[0].n;
   357    362     }else{
   358    363       Fts5PoslistWriter writer = {0};
   359         -    Fts5Buffer buf = {0,0,0};
   360    364       i64 iPrev = -1;
          365  +    fts5BufferZero(pBuf);
   361    366       while( 1 ){
   362    367         int i;
   363    368         i64 iMin = FTS5_LARGEST_INT64;
   364    369         for(i=0; i<nIter; i++){
   365    370           if( aIter[i].bEof==0 ){
   366    371             if( aIter[i].iPos==iPrev ){
   367    372               if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue;
................................................................................
   368    373             }
   369    374             if( aIter[i].iPos<iMin ){
   370    375               iMin = aIter[i].iPos;
   371    376             }
   372    377           }
   373    378         }
   374    379         if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break;
   375         -      rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin);
          380  +      rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin);
   376    381         iPrev = iMin;
   377    382       }
   378         -    if( rc ){
   379         -      sqlite3_free(buf.p);
   380         -    }else{
   381         -      *pa = buf.p;
   382         -      *pn = buf.n;
   383         -      *pbDel = 1;
          383  +    if( rc==SQLITE_OK ){
          384  +      *pa = pBuf->p;
          385  +      *pn = pBuf->n;
   384    386       }
   385    387     }
   386    388   
   387    389    synonym_poslist_out:
   388    390     if( aIter!=aStatic ) sqlite3_free(aIter);
   389    391     return rc;
   390    392   }
................................................................................
   399    401   **
   400    402   ** SQLITE_OK is returned if an error occurs, or an SQLite error code 
   401    403   ** otherwise. It is not considered an error code if the current rowid is 
   402    404   ** not a match.
   403    405   */
   404    406   static int fts5ExprPhraseIsMatch(
   405    407     Fts5ExprNode *pNode,            /* Node pPhrase belongs to */
   406         -  Fts5Colset *pColset,            /* Restrict matches to these columns */
   407    408     Fts5ExprPhrase *pPhrase,        /* Phrase object to initialize */
   408    409     int *pbMatch                    /* OUT: Set to true if really a match */
   409    410   ){
   410    411     Fts5PoslistWriter writer = {0};
   411    412     Fts5PoslistReader aStatic[4];
   412    413     Fts5PoslistReader *aIter = aStatic;
   413    414     int i;
   414    415     int rc = SQLITE_OK;
   415    416     
   416    417     fts5BufferZero(&pPhrase->poslist);
   417    418   
   418    419     /* If the aStatic[] array is not large enough, allocate a large array
   419    420     ** using sqlite3_malloc(). This approach could be improved upon. */
   420         -  if( pPhrase->nTerm>(int)ArraySize(aStatic) ){
          421  +  if( pPhrase->nTerm>ArraySize(aStatic) ){
   421    422       int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
   422    423       aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   423    424       if( !aIter ) return SQLITE_NOMEM;
   424    425     }
   425    426     memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
   426    427   
   427    428     /* Initialize a term iterator for each term in the phrase */
   428    429     for(i=0; i<pPhrase->nTerm; i++){
   429    430       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
   430         -    i64 dummy;
   431    431       int n = 0;
   432    432       int bFlag = 0;
   433         -    const u8 *a = 0;
          433  +    u8 *a = 0;
   434    434       if( pTerm->pSynonym ){
   435         -      rc = fts5ExprSynonymList(
   436         -          pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
   437         -      );
          435  +      Fts5Buffer buf = {0, 0, 0};
          436  +      rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n);
          437  +      if( rc ){
          438  +        sqlite3_free(a);
          439  +        goto ismatch_out;
          440  +      }
          441  +      if( a==buf.p ) bFlag = 1;
   438    442       }else{
   439         -      rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
          443  +      a = (u8*)pTerm->pIter->pData;
          444  +      n = pTerm->pIter->nData;
   440    445       }
   441         -    if( rc!=SQLITE_OK ) goto ismatch_out;
   442    446       sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
   443    447       aIter[i].bFlag = (u8)bFlag;
   444    448       if( aIter[i].bEof ) goto ismatch_out;
   445    449     }
   446    450   
   447    451     while( 1 ){
   448    452       int bMatch;
................................................................................
   506    510     memset(p, 0, sizeof(Fts5LookaheadReader));
   507    511     p->a = a;
   508    512     p->n = n;
   509    513     fts5LookaheadReaderNext(p);
   510    514     return fts5LookaheadReaderNext(p);
   511    515   }
   512    516   
   513         -#if 0
   514         -static int fts5LookaheadReaderEof(Fts5LookaheadReader *p){
   515         -  return (p->iPos==FTS5_LOOKAHEAD_EOF);
   516         -}
   517         -#endif
   518         -
   519    517   typedef struct Fts5NearTrimmer Fts5NearTrimmer;
   520    518   struct Fts5NearTrimmer {
   521    519     Fts5LookaheadReader reader;     /* Input iterator */
   522    520     Fts5PoslistWriter writer;       /* Writer context */
   523    521     Fts5Buffer *pOut;               /* Output poslist */
   524    522   };
   525    523   
................................................................................
   549    547     int rc = *pRc;
   550    548     int bMatch;
   551    549   
   552    550     assert( pNear->nPhrase>1 );
   553    551   
   554    552     /* If the aStatic[] array is not large enough, allocate a large array
   555    553     ** using sqlite3_malloc(). This approach could be improved upon. */
   556         -  if( pNear->nPhrase>(int)ArraySize(aStatic) ){
          554  +  if( pNear->nPhrase>ArraySize(aStatic) ){
   557    555       int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
   558    556       a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
   559    557     }else{
   560    558       memset(aStatic, 0, sizeof(aStatic));
   561    559     }
   562    560     if( rc!=SQLITE_OK ){
   563    561       *pRc = rc;
................................................................................
   626    624       int bRet = a[0].pOut->n>0;
   627    625       *pRc = rc;
   628    626       if( a!=aStatic ) sqlite3_free(a);
   629    627       return bRet;
   630    628     }
   631    629   }
   632    630   
   633         -/*
   634         -** Advance the first term iterator in the first phrase of pNear. Set output
   635         -** variable *pbEof to true if it reaches EOF or if an error occurs.
   636         -**
   637         -** Return SQLITE_OK if successful, or an SQLite error code if an error
   638         -** occurs.
   639         -*/
   640         -static int fts5ExprNearAdvanceFirst(
   641         -  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
   642         -  Fts5ExprNode *pNode,            /* FTS5_STRING or FTS5_TERM node */
   643         -  int bFromValid,
   644         -  i64 iFrom 
   645         -){
   646         -  Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
   647         -  int rc = SQLITE_OK;
   648         -
   649         -  if( pTerm->pSynonym ){
   650         -    int bEof = 1;
   651         -    Fts5ExprTerm *p;
   652         -
   653         -    /* Find the firstest rowid any synonym points to. */
   654         -    i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
   655         -
   656         -    /* Advance each iterator that currently points to iRowid. Or, if iFrom
   657         -    ** is valid - each iterator that points to a rowid before iFrom.  */
   658         -    for(p=pTerm; p; p=p->pSynonym){
   659         -      if( sqlite3Fts5IterEof(p->pIter)==0 ){
   660         -        i64 ii = sqlite3Fts5IterRowid(p->pIter);
   661         -        if( ii==iRowid 
   662         -         || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) 
   663         -        ){
   664         -          if( bFromValid ){
   665         -            rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
   666         -          }else{
   667         -            rc = sqlite3Fts5IterNext(p->pIter);
   668         -          }
   669         -          if( rc!=SQLITE_OK ) break;
   670         -          if( sqlite3Fts5IterEof(p->pIter)==0 ){
   671         -            bEof = 0;
   672         -          }
   673         -        }else{
   674         -          bEof = 0;
   675         -        }
   676         -      }
   677         -    }
   678         -
   679         -    /* Set the EOF flag if either all synonym iterators are at EOF or an
   680         -    ** error has occurred.  */
   681         -    pNode->bEof = (rc || bEof);
   682         -  }else{
   683         -    Fts5IndexIter *pIter = pTerm->pIter;
   684         -
   685         -    assert( Fts5NodeIsString(pNode) );
   686         -    if( bFromValid ){
   687         -      rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
   688         -    }else{
   689         -      rc = sqlite3Fts5IterNext(pIter);
   690         -    }
   691         -
   692         -    pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
   693         -  }
   694         -
   695         -  return rc;
   696         -}
   697         -
   698    631   /*
   699    632   ** Advance iterator pIter until it points to a value equal to or laster
   700    633   ** than the initial value of *piLast. If this means the iterator points
   701    634   ** to a value laster than *piLast, update *piLast to the new lastest value.
   702    635   **
   703    636   ** If the iterator reaches EOF, set *pbEof to true before returning. If
   704    637   ** an error occurs, set *pRc to an error code. If either *pbEof or *pRc
................................................................................
   710    643     i64 *piLast,                    /* IN/OUT: Lastest rowid seen so far */
   711    644     int *pRc,                       /* OUT: Error code */
   712    645     int *pbEof                      /* OUT: Set to true if EOF */
   713    646   ){
   714    647     i64 iLast = *piLast;
   715    648     i64 iRowid;
   716    649   
   717         -  iRowid = sqlite3Fts5IterRowid(pIter);
          650  +  iRowid = pIter->iRowid;
   718    651     if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
   719    652       int rc = sqlite3Fts5IterNextFrom(pIter, iLast);
   720    653       if( rc || sqlite3Fts5IterEof(pIter) ){
   721    654         *pRc = rc;
   722    655         *pbEof = 1;
   723    656         return 1;
   724    657       }
   725         -    iRowid = sqlite3Fts5IterRowid(pIter);
          658  +    iRowid = pIter->iRowid;
   726    659       assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) );
   727    660     }
   728    661     *piLast = iRowid;
   729    662   
   730    663     return 0;
   731    664   }
   732    665   
................................................................................
   739    672     int rc = SQLITE_OK;
   740    673     i64 iLast = *piLast;
   741    674     Fts5ExprTerm *p;
   742    675     int bEof = 0;
   743    676   
   744    677     for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
   745    678       if( sqlite3Fts5IterEof(p->pIter)==0 ){
   746         -      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
          679  +      i64 iRowid = p->pIter->iRowid;
   747    680         if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
   748    681           rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
   749    682         }
   750    683       }
   751    684     }
   752    685   
   753    686     if( rc!=SQLITE_OK ){
................................................................................
   771    704     if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){
   772    705       Fts5ExprTerm *pTerm;
   773    706       Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
   774    707       pPhrase->poslist.n = 0;
   775    708       for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
   776    709         Fts5IndexIter *pIter = pTerm->pIter;
   777    710         if( sqlite3Fts5IterEof(pIter)==0 ){
   778         -        int n;
   779         -        i64 iRowid;
   780         -        rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
   781         -        if( rc!=SQLITE_OK ){
   782         -          *pRc = rc;
   783         -          return 0;
   784         -        }else if( iRowid==pNode->iRowid && n>0 ){
          711  +        if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){
   785    712             pPhrase->poslist.n = 1;
   786    713           }
   787    714         }
   788    715       }
   789    716       return pPhrase->poslist.n;
   790    717     }else{
   791    718       int i;
................................................................................
   793    720       /* Check that each phrase in the nearset matches the current row.
   794    721       ** Populate the pPhrase->poslist buffers at the same time. If any
   795    722       ** phrase is not a match, break out of the loop early.  */
   796    723       for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   797    724         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   798    725         if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
   799    726           int bMatch = 0;
   800         -        rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
          727  +        rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
   801    728           if( bMatch==0 ) break;
   802    729         }else{
   803         -        rc = sqlite3Fts5IterPoslistBuffer(
   804         -            pPhrase->aTerm[0].pIter, &pPhrase->poslist
   805         -        );
          730  +        Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
          731  +        fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData);
   806    732         }
   807    733       }
   808    734   
   809    735       *pRc = rc;
   810    736       if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
   811    737         return 1;
   812    738       }
   813    739       return 0;
   814    740     }
   815    741   }
   816    742   
   817         -static int fts5ExprTokenTest(
   818         -  Fts5Expr *pExpr,                /* Expression that pNear is a part of */
   819         -  Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_TERM) */
   820         -){
   821         -  /* As this "NEAR" object is actually a single phrase that consists 
   822         -  ** of a single term only, grab pointers into the poslist managed by the
   823         -  ** fts5_index.c iterator object. This is much faster than synthesizing 
   824         -  ** a new poslist the way we have to for more complicated phrase or NEAR
   825         -  ** expressions.  */
   826         -  Fts5ExprNearset *pNear = pNode->pNear;
   827         -  Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
   828         -  Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
   829         -  Fts5Colset *pColset = pNear->pColset;
   830         -  int rc;
   831         -
   832         -  assert( pNode->eType==FTS5_TERM );
   833         -  assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
   834         -  assert( pPhrase->aTerm[0].pSynonym==0 );
   835         -
   836         -  rc = sqlite3Fts5IterPoslist(pIter, pColset, 
   837         -      (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid
   838         -  );
   839         -  pNode->bNomatch = (pPhrase->poslist.n==0);
   840         -  return rc;
   841         -}
   842         -
   843         -/*
   844         -** All individual term iterators in pNear are guaranteed to be valid when
   845         -** this function is called. This function checks if all term iterators
   846         -** point to the same rowid, and if not, advances them until they do.
   847         -** If an EOF is reached before this happens, *pbEof is set to true before
   848         -** returning.
   849         -**
   850         -** SQLITE_OK is returned if an error occurs, or an SQLite error code 
   851         -** otherwise. It is not considered an error code if an iterator reaches
   852         -** EOF.
   853         -*/
   854         -static int fts5ExprNearNextMatch(
   855         -  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
   856         -  Fts5ExprNode *pNode
   857         -){
   858         -  Fts5ExprNearset *pNear = pNode->pNear;
   859         -  Fts5ExprPhrase *pLeft = pNear->apPhrase[0];
   860         -  int rc = SQLITE_OK;
   861         -  i64 iLast;                      /* Lastest rowid any iterator points to */
   862         -  int i, j;                       /* Phrase and token index, respectively */
   863         -  int bMatch;                     /* True if all terms are at the same rowid */
   864         -  const int bDesc = pExpr->bDesc;
   865         -
   866         -  /* Check that this node should not be FTS5_TERM */
   867         -  assert( pNear->nPhrase>1 
   868         -       || pNear->apPhrase[0]->nTerm>1 
   869         -       || pNear->apPhrase[0]->aTerm[0].pSynonym
   870         -  );
   871         -
   872         -  /* Initialize iLast, the "lastest" rowid any iterator points to. If the
   873         -  ** iterator skips through rowids in the default ascending order, this means
   874         -  ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
   875         -  ** means the minimum rowid.  */
   876         -  if( pLeft->aTerm[0].pSynonym ){
   877         -    iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
   878         -  }else{
   879         -    iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
   880         -  }
   881         -
   882         -  do {
   883         -    bMatch = 1;
   884         -    for(i=0; i<pNear->nPhrase; i++){
   885         -      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   886         -      for(j=0; j<pPhrase->nTerm; j++){
   887         -        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
   888         -        if( pTerm->pSynonym ){
   889         -          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
   890         -          if( iRowid==iLast ) continue;
   891         -          bMatch = 0;
   892         -          if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
   893         -            pNode->bEof = 1;
   894         -            return rc;
   895         -          }
   896         -        }else{
   897         -          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
   898         -          i64 iRowid = sqlite3Fts5IterRowid(pIter);
   899         -          if( iRowid==iLast ) continue;
   900         -          bMatch = 0;
   901         -          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
   902         -            return rc;
   903         -          }
   904         -        }
   905         -      }
   906         -    }
   907         -  }while( bMatch==0 );
   908         -
   909         -  pNode->iRowid = iLast;
   910         -  pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
   911         -
   912         -  return rc;
   913         -}
   914    743   
   915    744   /*
   916    745   ** Initialize all term iterators in the pNear object. If any term is found
   917    746   ** to match no documents at all, return immediately without initializing any
   918    747   ** further iterators.
   919    748   */
   920    749   static int fts5ExprNearInitAll(
................................................................................
   921    750     Fts5Expr *pExpr,
   922    751     Fts5ExprNode *pNode
   923    752   ){
   924    753     Fts5ExprNearset *pNear = pNode->pNear;
   925    754     int i, j;
   926    755     int rc = SQLITE_OK;
   927    756   
          757  +  assert( pNode->bNomatch==0 );
   928    758     for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   929    759       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   930    760       for(j=0; j<pPhrase->nTerm; j++){
   931    761         Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
   932    762         Fts5ExprTerm *p;
   933    763         int bEof = 1;
   934    764   
................................................................................
   955    785           return rc;
   956    786         }
   957    787       }
   958    788     }
   959    789   
   960    790     return rc;
   961    791   }
   962         -
   963         -/* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */
   964         -static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*);
   965         -
   966    792   
   967    793   /*
   968    794   ** If pExpr is an ASC iterator, this function returns a value with the
   969    795   ** same sign as:
   970    796   **
   971    797   **   (iLhs - iRhs)
   972    798   **
................................................................................
   988    814       return (iLhs < iRhs);
   989    815     }
   990    816   }
   991    817   
   992    818   static void fts5ExprSetEof(Fts5ExprNode *pNode){
   993    819     int i;
   994    820     pNode->bEof = 1;
          821  +  pNode->bNomatch = 0;
   995    822     for(i=0; i<pNode->nChild; i++){
   996    823       fts5ExprSetEof(pNode->apChild[i]);
   997    824     }
   998    825   }
   999    826   
  1000    827   static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){
  1001    828     if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
................................................................................
  1010    837       for(i=0; i<pNode->nChild; i++){
  1011    838         fts5ExprNodeZeroPoslist(pNode->apChild[i]);
  1012    839       }
  1013    840     }
  1014    841   }
  1015    842   
  1016    843   
  1017         -static int fts5ExprNodeNext(Fts5Expr*, Fts5ExprNode*, int, i64);
          844  +
          845  +/*
          846  +** Compare the values currently indicated by the two nodes as follows:
          847  +**
          848  +**    res = (*p1) - (*p2)
          849  +**
          850  +** Nodes that point to values that come later in the iteration order are
          851  +** considered to be larger. Nodes at EOF are the largest of all.
          852  +**
          853  +** This means that if the iteration order is ASC, then numerically larger
          854  +** rowids are considered larger. Or if it is the default DESC, numerically
          855  +** smaller rowids are larger.
          856  +*/
          857  +static int fts5NodeCompare(
          858  +  Fts5Expr *pExpr,
          859  +  Fts5ExprNode *p1, 
          860  +  Fts5ExprNode *p2
          861  +){
          862  +  if( p2->bEof ) return -1;
          863  +  if( p1->bEof ) return +1;
          864  +  return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
          865  +}
          866  +
          867  +/*
          868  +** All individual term iterators in pNear are guaranteed to be valid when
          869  +** this function is called. This function checks if all term iterators
          870  +** point to the same rowid, and if not, advances them until they do.
          871  +** If an EOF is reached before this happens, *pbEof is set to true before
          872  +** returning.
          873  +**
          874  +** SQLITE_OK is returned if an error occurs, or an SQLite error code 
          875  +** otherwise. It is not considered an error code if an iterator reaches
          876  +** EOF.
          877  +*/
          878  +static int fts5ExprNodeTest_STRING(
          879  +  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
          880  +  Fts5ExprNode *pNode
          881  +){
          882  +  Fts5ExprNearset *pNear = pNode->pNear;
          883  +  Fts5ExprPhrase *pLeft = pNear->apPhrase[0];
          884  +  int rc = SQLITE_OK;
          885  +  i64 iLast;                      /* Lastest rowid any iterator points to */
          886  +  int i, j;                       /* Phrase and token index, respectively */
          887  +  int bMatch;                     /* True if all terms are at the same rowid */
          888  +  const int bDesc = pExpr->bDesc;
          889  +
          890  +  /* Check that this node should not be FTS5_TERM */
          891  +  assert( pNear->nPhrase>1 
          892  +       || pNear->apPhrase[0]->nTerm>1 
          893  +       || pNear->apPhrase[0]->aTerm[0].pSynonym
          894  +  );
          895  +
          896  +  /* Initialize iLast, the "lastest" rowid any iterator points to. If the
          897  +  ** iterator skips through rowids in the default ascending order, this means
          898  +  ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
          899  +  ** means the minimum rowid.  */
          900  +  if( pLeft->aTerm[0].pSynonym ){
          901  +    iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
          902  +  }else{
          903  +    iLast = pLeft->aTerm[0].pIter->iRowid;
          904  +  }
          905  +
          906  +  do {
          907  +    bMatch = 1;
          908  +    for(i=0; i<pNear->nPhrase; i++){
          909  +      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
          910  +      for(j=0; j<pPhrase->nTerm; j++){
          911  +        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
          912  +        if( pTerm->pSynonym ){
          913  +          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
          914  +          if( iRowid==iLast ) continue;
          915  +          bMatch = 0;
          916  +          if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
          917  +            pNode->bNomatch = 0;
          918  +            pNode->bEof = 1;
          919  +            return rc;
          920  +          }
          921  +        }else{
          922  +          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
          923  +          if( pIter->iRowid==iLast ) continue;
          924  +          bMatch = 0;
          925  +          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
          926  +            return rc;
          927  +          }
          928  +        }
          929  +      }
          930  +    }
          931  +  }while( bMatch==0 );
          932  +
          933  +  pNode->iRowid = iLast;
          934  +  pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK);
          935  +  assert( pNode->bEof==0 || pNode->bNomatch==0 );
          936  +
          937  +  return rc;
          938  +}
          939  +
          940  +/*
          941  +** Advance the first term iterator in the first phrase of pNear. Set output
          942  +** variable *pbEof to true if it reaches EOF or if an error occurs.
          943  +**
          944  +** Return SQLITE_OK if successful, or an SQLite error code if an error
          945  +** occurs.
          946  +*/
          947  +static int fts5ExprNodeNext_STRING(
          948  +  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
          949  +  Fts5ExprNode *pNode,            /* FTS5_STRING or FTS5_TERM node */
          950  +  int bFromValid,
          951  +  i64 iFrom 
          952  +){
          953  +  Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
          954  +  int rc = SQLITE_OK;
          955  +
          956  +  pNode->bNomatch = 0;
          957  +  if( pTerm->pSynonym ){
          958  +    int bEof = 1;
          959  +    Fts5ExprTerm *p;
          960  +
          961  +    /* Find the firstest rowid any synonym points to. */
          962  +    i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
          963  +
          964  +    /* Advance each iterator that currently points to iRowid. Or, if iFrom
          965  +    ** is valid - each iterator that points to a rowid before iFrom.  */
          966  +    for(p=pTerm; p; p=p->pSynonym){
          967  +      if( sqlite3Fts5IterEof(p->pIter)==0 ){
          968  +        i64 ii = p->pIter->iRowid;
          969  +        if( ii==iRowid 
          970  +         || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) 
          971  +        ){
          972  +          if( bFromValid ){
          973  +            rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
          974  +          }else{
          975  +            rc = sqlite3Fts5IterNext(p->pIter);
          976  +          }
          977  +          if( rc!=SQLITE_OK ) break;
          978  +          if( sqlite3Fts5IterEof(p->pIter)==0 ){
          979  +            bEof = 0;
          980  +          }
          981  +        }else{
          982  +          bEof = 0;
          983  +        }
          984  +      }
          985  +    }
          986  +
          987  +    /* Set the EOF flag if either all synonym iterators are at EOF or an
          988  +    ** error has occurred.  */
          989  +    pNode->bEof = (rc || bEof);
          990  +  }else{
          991  +    Fts5IndexIter *pIter = pTerm->pIter;
          992  +
          993  +    assert( Fts5NodeIsString(pNode) );
          994  +    if( bFromValid ){
          995  +      rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
          996  +    }else{
          997  +      rc = sqlite3Fts5IterNext(pIter);
          998  +    }
          999  +
         1000  +    pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
         1001  +  }
         1002  +
         1003  +  if( pNode->bEof==0 ){
         1004  +    assert( rc==SQLITE_OK );
         1005  +    rc = fts5ExprNodeTest_STRING(pExpr, pNode);
         1006  +  }
         1007  +
         1008  +  return rc;
         1009  +}
         1010  +
         1011  +
         1012  +static int fts5ExprNodeTest_TERM(
         1013  +  Fts5Expr *pExpr,                /* Expression that pNear is a part of */
         1014  +  Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_TERM) */
         1015  +){
         1016  +  /* As this "NEAR" object is actually a single phrase that consists 
         1017  +  ** of a single term only, grab pointers into the poslist managed by the
         1018  +  ** fts5_index.c iterator object. This is much faster than synthesizing 
         1019  +  ** a new poslist the way we have to for more complicated phrase or NEAR
         1020  +  ** expressions.  */
         1021  +  Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
         1022  +  Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
         1023  +
         1024  +  assert( pNode->eType==FTS5_TERM );
         1025  +  assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 );
         1026  +  assert( pPhrase->aTerm[0].pSynonym==0 );
         1027  +
         1028  +  pPhrase->poslist.n = pIter->nData;
         1029  +  if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){
         1030  +    pPhrase->poslist.p = (u8*)pIter->pData;
         1031  +  }
         1032  +  pNode->iRowid = pIter->iRowid;
         1033  +  pNode->bNomatch = (pPhrase->poslist.n==0);
         1034  +  return SQLITE_OK;
         1035  +}
         1036  +
         1037  +/*
         1038  +** xNext() method for a node of type FTS5_TERM.
         1039  +*/
         1040  +static int fts5ExprNodeNext_TERM(
         1041  +  Fts5Expr *pExpr, 
         1042  +  Fts5ExprNode *pNode,
         1043  +  int bFromValid,
         1044  +  i64 iFrom
         1045  +){
         1046  +  int rc;
         1047  +  Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
         1048  +
         1049  +  assert( pNode->bEof==0 );
         1050  +  if( bFromValid ){
         1051  +    rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
         1052  +  }else{
         1053  +    rc = sqlite3Fts5IterNext(pIter);
         1054  +  }
         1055  +  if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){
         1056  +    rc = fts5ExprNodeTest_TERM(pExpr, pNode);
         1057  +  }else{
         1058  +    pNode->bEof = 1;
         1059  +    pNode->bNomatch = 0;
         1060  +  }
         1061  +  return rc;
         1062  +}
         1063  +
         1064  +static void fts5ExprNodeTest_OR(
         1065  +  Fts5Expr *pExpr,                /* Expression of which pNode is a part */
         1066  +  Fts5ExprNode *pNode             /* Expression node to test */
         1067  +){
         1068  +  Fts5ExprNode *pNext = pNode->apChild[0];
         1069  +  int i;
         1070  +
         1071  +  for(i=1; i<pNode->nChild; i++){
         1072  +    Fts5ExprNode *pChild = pNode->apChild[i];
         1073  +    int cmp = fts5NodeCompare(pExpr, pNext, pChild);
         1074  +    if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){
         1075  +      pNext = pChild;
         1076  +    }
         1077  +  }
         1078  +  pNode->iRowid = pNext->iRowid;
         1079  +  pNode->bEof = pNext->bEof;
         1080  +  pNode->bNomatch = pNext->bNomatch;
         1081  +}
         1082  +
         1083  +static int fts5ExprNodeNext_OR(
         1084  +  Fts5Expr *pExpr, 
         1085  +  Fts5ExprNode *pNode,
         1086  +  int bFromValid,
         1087  +  i64 iFrom
         1088  +){
         1089  +  int i;
         1090  +  i64 iLast = pNode->iRowid;
         1091  +
         1092  +  for(i=0; i<pNode->nChild; i++){
         1093  +    Fts5ExprNode *p1 = pNode->apChild[i];
         1094  +    assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 );
         1095  +    if( p1->bEof==0 ){
         1096  +      if( (p1->iRowid==iLast) 
         1097  +       || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
         1098  +      ){
         1099  +        int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
         1100  +        if( rc!=SQLITE_OK ) return rc;
         1101  +      }
         1102  +    }
         1103  +  }
         1104  +
         1105  +  fts5ExprNodeTest_OR(pExpr, pNode);
         1106  +  return SQLITE_OK;
         1107  +}
  1018   1108   
  1019   1109   /*
  1020   1110   ** Argument pNode is an FTS5_AND node.
  1021   1111   */
  1022         -static int fts5ExprAndNextRowid(
         1112  +static int fts5ExprNodeTest_AND(
  1023   1113     Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
  1024   1114     Fts5ExprNode *pAnd              /* FTS5_AND node to advance */
  1025   1115   ){
  1026   1116     int iChild;
  1027   1117     i64 iLast = pAnd->iRowid;
  1028   1118     int rc = SQLITE_OK;
  1029   1119     int bMatch;
................................................................................
  1030   1120   
  1031   1121     assert( pAnd->bEof==0 );
  1032   1122     do {
  1033   1123       pAnd->bNomatch = 0;
  1034   1124       bMatch = 1;
  1035   1125       for(iChild=0; iChild<pAnd->nChild; iChild++){
  1036   1126         Fts5ExprNode *pChild = pAnd->apChild[iChild];
  1037         -      if( 0 && pChild->eType==FTS5_STRING ){
  1038         -        /* TODO */
  1039         -      }else{
  1040         -        int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
  1041         -        if( cmp>0 ){
  1042         -          /* Advance pChild until it points to iLast or laster */
  1043         -          rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
  1044         -          if( rc!=SQLITE_OK ) return rc;
  1045         -        }
         1127  +      int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
         1128  +      if( cmp>0 ){
         1129  +        /* Advance pChild until it points to iLast or laster */
         1130  +        rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
         1131  +        if( rc!=SQLITE_OK ) return rc;
  1046   1132         }
  1047   1133   
  1048   1134         /* If the child node is now at EOF, so is the parent AND node. Otherwise,
  1049   1135         ** the child node is guaranteed to have advanced at least as far as
  1050   1136         ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
  1051   1137         ** new lastest rowid seen so far.  */
  1052   1138         assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 );
................................................................................
  1068   1154     if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){
  1069   1155       fts5ExprNodeZeroPoslist(pAnd);
  1070   1156     }
  1071   1157     pAnd->iRowid = iLast;
  1072   1158     return SQLITE_OK;
  1073   1159   }
  1074   1160   
  1075         -
  1076         -/*
  1077         -** Compare the values currently indicated by the two nodes as follows:
  1078         -**
  1079         -**    res = (*p1) - (*p2)
  1080         -**
  1081         -** Nodes that point to values that come later in the iteration order are
  1082         -** considered to be larger. Nodes at EOF are the largest of all.
  1083         -**
  1084         -** This means that if the iteration order is ASC, then numerically larger
  1085         -** rowids are considered larger. Or if it is the default DESC, numerically
  1086         -** smaller rowids are larger.
  1087         -*/
  1088         -static int fts5NodeCompare(
  1089         -  Fts5Expr *pExpr,
  1090         -  Fts5ExprNode *p1, 
  1091         -  Fts5ExprNode *p2
  1092         -){
  1093         -  if( p2->bEof ) return -1;
  1094         -  if( p1->bEof ) return +1;
  1095         -  return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
  1096         -}
  1097         -
  1098         -/*
  1099         -** Advance node iterator pNode, part of expression pExpr. If argument
  1100         -** bFromValid is zero, then pNode is advanced exactly once. Or, if argument
  1101         -** bFromValid is non-zero, then pNode is advanced until it is at or past
  1102         -** rowid value iFrom. Whether "past" means "less than" or "greater than"
  1103         -** depends on whether this is an ASC or DESC iterator.
  1104         -*/
  1105         -static int fts5ExprNodeNext(
         1161  +static int fts5ExprNodeNext_AND(
         1162  +  Fts5Expr *pExpr, 
         1163  +  Fts5ExprNode *pNode,
         1164  +  int bFromValid,
         1165  +  i64 iFrom
         1166  +){
         1167  +  int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
         1168  +  if( rc==SQLITE_OK ){
         1169  +    rc = fts5ExprNodeTest_AND(pExpr, pNode);
         1170  +  }
         1171  +  return rc;
         1172  +}
         1173  +
         1174  +static int fts5ExprNodeTest_NOT(
         1175  +  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
         1176  +  Fts5ExprNode *pNode             /* FTS5_NOT node to advance */
         1177  +){
         1178  +  int rc = SQLITE_OK;
         1179  +  Fts5ExprNode *p1 = pNode->apChild[0];
         1180  +  Fts5ExprNode *p2 = pNode->apChild[1];
         1181  +  assert( pNode->nChild==2 );
         1182  +
         1183  +  while( rc==SQLITE_OK && p1->bEof==0 ){
         1184  +    int cmp = fts5NodeCompare(pExpr, p1, p2);
         1185  +    if( cmp>0 ){
         1186  +      rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid);
         1187  +      cmp = fts5NodeCompare(pExpr, p1, p2);
         1188  +    }
         1189  +    assert( rc!=SQLITE_OK || cmp<=0 );
         1190  +    if( cmp || p2->bNomatch ) break;
         1191  +    rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
         1192  +  }
         1193  +  pNode->bEof = p1->bEof;
         1194  +  pNode->bNomatch = p1->bNomatch;
         1195  +  pNode->iRowid = p1->iRowid;
         1196  +  if( p1->bEof ){
         1197  +    fts5ExprNodeZeroPoslist(p2);
         1198  +  }
         1199  +  return rc;
         1200  +}
         1201  +
         1202  +static int fts5ExprNodeNext_NOT(
  1106   1203     Fts5Expr *pExpr, 
  1107   1204     Fts5ExprNode *pNode,
  1108   1205     int bFromValid,
  1109   1206     i64 iFrom
  1110   1207   ){
  1111         -  int rc = SQLITE_OK;
  1112         -
  1113         -  if( pNode->bEof==0 ){
  1114         -    switch( pNode->eType ){
  1115         -      case FTS5_STRING: {
  1116         -        rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom);
  1117         -        break;
  1118         -      };
  1119         -
  1120         -      case FTS5_TERM: {
  1121         -        Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
  1122         -        if( bFromValid ){
  1123         -          rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
  1124         -        }else{
  1125         -          rc = sqlite3Fts5IterNext(pIter);
  1126         -        }
  1127         -        if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){
  1128         -          assert( rc==SQLITE_OK );
  1129         -          rc = fts5ExprTokenTest(pExpr, pNode);
  1130         -        }else{
  1131         -          pNode->bEof = 1;
  1132         -        }
  1133         -        return rc;
  1134         -      };
  1135         -
  1136         -      case FTS5_AND: {
  1137         -        Fts5ExprNode *pLeft = pNode->apChild[0];
  1138         -        rc = fts5ExprNodeNext(pExpr, pLeft, bFromValid, iFrom);
  1139         -        break;
  1140         -      }
  1141         -
  1142         -      case FTS5_OR: {
  1143         -        int i;
  1144         -        i64 iLast = pNode->iRowid;
  1145         -
  1146         -        for(i=0; rc==SQLITE_OK && i<pNode->nChild; i++){
  1147         -          Fts5ExprNode *p1 = pNode->apChild[i];
  1148         -          assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 );
  1149         -          if( p1->bEof==0 ){
  1150         -            if( (p1->iRowid==iLast) 
  1151         -             || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
  1152         -            ){
  1153         -              rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
  1154         -            }
  1155         -          }
  1156         -        }
  1157         -
  1158         -        break;
  1159         -      }
  1160         -
  1161         -      default: assert( pNode->eType==FTS5_NOT ); {
  1162         -        assert( pNode->nChild==2 );
  1163         -        rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  1164         -        break;
  1165         -      }
  1166         -    }
  1167         -
  1168         -    if( rc==SQLITE_OK ){
  1169         -      rc = fts5ExprNodeNextMatch(pExpr, pNode);
  1170         -    }
         1208  +  int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
         1209  +  if( rc==SQLITE_OK ){
         1210  +    rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  1171   1211     }
  1172         -
  1173         -  /* Assert that if bFromValid was true, either:
  1174         -  **
  1175         -  **   a) an error occurred, or
  1176         -  **   b) the node is now at EOF, or
  1177         -  **   c) the node is now at or past rowid iFrom.
  1178         -  */
  1179         -  assert( bFromValid==0 
  1180         -      || rc!=SQLITE_OK                                                  /* a */
  1181         -      || pNode->bEof                                                    /* b */
  1182         -      || pNode->iRowid==iFrom || pExpr->bDesc==(pNode->iRowid<iFrom)    /* c */
  1183         -  );
  1184         -
  1185   1212     return rc;
  1186   1213   }
  1187         -
  1188   1214   
  1189   1215   /*
  1190   1216   ** If pNode currently points to a match, this function returns SQLITE_OK
  1191   1217   ** without modifying it. Otherwise, pNode is advanced until it does point
  1192   1218   ** to a match or EOF is reached.
  1193   1219   */
  1194         -static int fts5ExprNodeNextMatch(
         1220  +static int fts5ExprNodeTest(
  1195   1221     Fts5Expr *pExpr,                /* Expression of which pNode is a part */
  1196   1222     Fts5ExprNode *pNode             /* Expression node to test */
  1197   1223   ){
  1198   1224     int rc = SQLITE_OK;
  1199   1225     if( pNode->bEof==0 ){
  1200   1226       switch( pNode->eType ){
  1201   1227   
  1202   1228         case FTS5_STRING: {
  1203         -        /* Advance the iterators until they all point to the same rowid */
  1204         -        rc = fts5ExprNearNextMatch(pExpr, pNode);
         1229  +        rc = fts5ExprNodeTest_STRING(pExpr, pNode);
  1205   1230           break;
  1206   1231         }
  1207   1232   
  1208   1233         case FTS5_TERM: {
  1209         -        rc = fts5ExprTokenTest(pExpr, pNode);
         1234  +        rc = fts5ExprNodeTest_TERM(pExpr, pNode);
  1210   1235           break;
  1211   1236         }
  1212   1237   
  1213   1238         case FTS5_AND: {
  1214         -        rc = fts5ExprAndNextRowid(pExpr, pNode);
         1239  +        rc = fts5ExprNodeTest_AND(pExpr, pNode);
  1215   1240           break;
  1216   1241         }
  1217   1242   
  1218   1243         case FTS5_OR: {
  1219         -        Fts5ExprNode *pNext = pNode->apChild[0];
  1220         -        int i;
  1221         -
  1222         -        for(i=1; i<pNode->nChild; i++){
  1223         -          Fts5ExprNode *pChild = pNode->apChild[i];
  1224         -          int cmp = fts5NodeCompare(pExpr, pNext, pChild);
  1225         -          if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){
  1226         -            pNext = pChild;
  1227         -          }
  1228         -        }
  1229         -        pNode->iRowid = pNext->iRowid;
  1230         -        pNode->bEof = pNext->bEof;
  1231         -        pNode->bNomatch = pNext->bNomatch;
         1244  +        fts5ExprNodeTest_OR(pExpr, pNode);
  1232   1245           break;
  1233   1246         }
  1234   1247   
  1235   1248         default: assert( pNode->eType==FTS5_NOT ); {
  1236         -        Fts5ExprNode *p1 = pNode->apChild[0];
  1237         -        Fts5ExprNode *p2 = pNode->apChild[1];
  1238         -        assert( pNode->nChild==2 );
  1239         -
  1240         -        while( rc==SQLITE_OK && p1->bEof==0 ){
  1241         -          int cmp = fts5NodeCompare(pExpr, p1, p2);
  1242         -          if( cmp>0 ){
  1243         -            rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid);
  1244         -            cmp = fts5NodeCompare(pExpr, p1, p2);
  1245         -          }
  1246         -          assert( rc!=SQLITE_OK || cmp<=0 );
  1247         -          if( cmp || p2->bNomatch ) break;
  1248         -          rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
  1249         -        }
  1250         -        pNode->bEof = p1->bEof;
  1251         -        pNode->iRowid = p1->iRowid;
  1252         -        if( p1->bEof ){
  1253         -          fts5ExprNodeZeroPoslist(p2);
  1254         -        }
         1249  +        rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  1255   1250           break;
  1256   1251         }
  1257   1252       }
  1258   1253     }
  1259   1254     return rc;
  1260   1255   }
  1261   1256   
................................................................................
  1266   1261   **
  1267   1262   ** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise.
  1268   1263   ** It is not an error if there are no matches.
  1269   1264   */
  1270   1265   static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
  1271   1266     int rc = SQLITE_OK;
  1272   1267     pNode->bEof = 0;
         1268  +  pNode->bNomatch = 0;
  1273   1269   
  1274   1270     if( Fts5NodeIsString(pNode) ){
  1275   1271       /* Initialize all term iterators in the NEAR object. */
  1276   1272       rc = fts5ExprNearInitAll(pExpr, pNode);
         1273  +  }else if( pNode->xNext==0 ){
         1274  +    pNode->bEof = 1;
  1277   1275     }else{
  1278   1276       int i;
         1277  +    int nEof = 0;
  1279   1278       for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){
         1279  +      Fts5ExprNode *pChild = pNode->apChild[i];
  1280   1280         rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]);
         1281  +      assert( pChild->bEof==0 || pChild->bEof==1 );
         1282  +      nEof += pChild->bEof;
  1281   1283       }
  1282   1284       pNode->iRowid = pNode->apChild[0]->iRowid;
         1285  +
         1286  +    switch( pNode->eType ){
         1287  +      case FTS5_AND:
         1288  +        if( nEof>0 ) fts5ExprSetEof(pNode);
         1289  +        break;
         1290  +
         1291  +      case FTS5_OR:
         1292  +        if( pNode->nChild==nEof ) fts5ExprSetEof(pNode);
         1293  +        break;
         1294  +
         1295  +      default:
         1296  +        assert( pNode->eType==FTS5_NOT );
         1297  +        pNode->bEof = pNode->apChild[0]->bEof;
         1298  +        break;
         1299  +    }
  1283   1300     }
  1284   1301   
  1285   1302     if( rc==SQLITE_OK ){
  1286         -    rc = fts5ExprNodeNextMatch(pExpr, pNode);
         1303  +    rc = fts5ExprNodeTest(pExpr, pNode);
  1287   1304     }
  1288   1305     return rc;
  1289   1306   }
  1290   1307   
  1291   1308   
  1292   1309   /*
  1293   1310   ** Begin iterating through the set of documents in index pIdx matched by
................................................................................
  1302   1319   ** equal to iFirst.
  1303   1320   **
  1304   1321   ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
  1305   1322   ** is not considered an error if the query does not match any documents.
  1306   1323   */
  1307   1324   int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
  1308   1325     Fts5ExprNode *pRoot = p->pRoot;
  1309         -  int rc = SQLITE_OK;
  1310         -  if( pRoot ){
  1311         -    p->pIndex = pIdx;
  1312         -    p->bDesc = bDesc;
  1313         -    rc = fts5ExprNodeFirst(p, pRoot);
         1326  +  int rc;                         /* Return code */
  1314   1327   
  1315         -    /* If not at EOF but the current rowid occurs earlier than iFirst in
  1316         -    ** the iteration order, move to document iFirst or later. */
  1317         -    if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
  1318         -      rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
  1319         -    }
         1328  +  p->pIndex = pIdx;
         1329  +  p->bDesc = bDesc;
         1330  +  rc = fts5ExprNodeFirst(p, pRoot);
  1320   1331   
  1321         -    /* If the iterator is not at a real match, skip forward until it is. */
  1322         -    while( pRoot->bNomatch && rc==SQLITE_OK && pRoot->bEof==0 ){
  1323         -      rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  1324         -    }
         1332  +  /* If not at EOF but the current rowid occurs earlier than iFirst in
         1333  +  ** the iteration order, move to document iFirst or later. */
         1334  +  if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
         1335  +    rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
         1336  +  }
         1337  +
         1338  +  /* If the iterator is not at a real match, skip forward until it is. */
         1339  +  while( pRoot->bNomatch ){
         1340  +    assert( pRoot->bEof==0 && rc==SQLITE_OK );
         1341  +    rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  1325   1342     }
  1326   1343     return rc;
  1327   1344   }
  1328   1345   
  1329   1346   /*
  1330   1347   ** Move to the next document 
  1331   1348   **
  1332   1349   ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
  1333   1350   ** is not considered an error if the query does not match any documents.
  1334   1351   */
  1335   1352   int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){
  1336   1353     int rc;
  1337   1354     Fts5ExprNode *pRoot = p->pRoot;
         1355  +  assert( pRoot->bEof==0 && pRoot->bNomatch==0 );
  1338   1356     do {
  1339   1357       rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  1340         -  }while( pRoot->bNomatch && pRoot->bEof==0 && rc==SQLITE_OK );
         1358  +    assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) );
         1359  +  }while( pRoot->bNomatch );
  1341   1360     if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
  1342   1361       pRoot->bEof = 1;
  1343   1362     }
  1344   1363     return rc;
  1345   1364   }
  1346   1365   
  1347   1366   int sqlite3Fts5ExprEof(Fts5Expr *p){
  1348         -  return (p->pRoot==0 || p->pRoot->bEof);
         1367  +  return p->pRoot->bEof;
  1349   1368   }
  1350   1369   
  1351   1370   i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
  1352   1371     return p->pRoot->iRowid;
  1353   1372   }
  1354   1373   
  1355   1374   static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
................................................................................
  1366   1385       int i;
  1367   1386       for(i=0; i<pPhrase->nTerm; i++){
  1368   1387         Fts5ExprTerm *pSyn;
  1369   1388         Fts5ExprTerm *pNext;
  1370   1389         Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
  1371   1390         sqlite3_free(pTerm->zTerm);
  1372   1391         sqlite3Fts5IterClose(pTerm->pIter);
  1373         -
  1374   1392         for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
  1375   1393           pNext = pSyn->pSynonym;
  1376   1394           sqlite3Fts5IterClose(pSyn->pIter);
         1395  +        fts5BufferFree((Fts5Buffer*)&pSyn[1]);
  1377   1396           sqlite3_free(pSyn);
  1378   1397         }
  1379   1398       }
  1380   1399       if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
  1381   1400       sqlite3_free(pPhrase);
  1382   1401     }
  1383   1402   }
................................................................................
  1424   1443     }
  1425   1444   
  1426   1445     if( pRet==0 ){
  1427   1446       assert( pParse->rc!=SQLITE_OK );
  1428   1447       sqlite3Fts5ParseNearsetFree(pNear);
  1429   1448       sqlite3Fts5ParsePhraseFree(pPhrase);
  1430   1449     }else{
         1450  +    if( pRet->nPhrase>0 ){
         1451  +      Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
         1452  +      assert( pLast==pParse->apPhrase[pParse->nPhrase-2] );
         1453  +      if( pPhrase->nTerm==0 ){
         1454  +        fts5ExprPhraseFree(pPhrase);
         1455  +        pRet->nPhrase--;
         1456  +        pParse->nPhrase--;
         1457  +        pPhrase = pLast;
         1458  +      }else if( pLast->nTerm==0 ){
         1459  +        fts5ExprPhraseFree(pLast);
         1460  +        pParse->apPhrase[pParse->nPhrase-2] = pPhrase;
         1461  +        pParse->nPhrase--;
         1462  +        pRet->nPhrase--;
         1463  +      }
         1464  +    }
  1431   1465       pRet->apPhrase[pRet->nPhrase++] = pPhrase;
  1432   1466     }
  1433   1467     return pRet;
  1434   1468   }
  1435   1469   
  1436   1470   typedef struct TokenCtx TokenCtx;
  1437   1471   struct TokenCtx {
................................................................................
  1450   1484     int iUnused1,                   /* Start offset of token */
  1451   1485     int iUnused2                    /* End offset of token */
  1452   1486   ){
  1453   1487     int rc = SQLITE_OK;
  1454   1488     const int SZALLOC = 8;
  1455   1489     TokenCtx *pCtx = (TokenCtx*)pContext;
  1456   1490     Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
         1491  +
         1492  +  UNUSED_PARAM2(iUnused1, iUnused2);
  1457   1493   
  1458   1494     /* If an error has already occurred, this is a no-op */
  1459   1495     if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
         1496  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  1460   1497   
  1461         -  assert( pPhrase==0 || pPhrase->nTerm>0 );
  1462         -  if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
         1498  +  if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
  1463   1499       Fts5ExprTerm *pSyn;
  1464         -    int nByte = sizeof(Fts5ExprTerm) + nToken+1;
         1500  +    int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
  1465   1501       pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
  1466   1502       if( pSyn==0 ){
  1467   1503         rc = SQLITE_NOMEM;
  1468   1504       }else{
  1469   1505         memset(pSyn, 0, nByte);
  1470         -      pSyn->zTerm = (char*)&pSyn[1];
         1506  +      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
  1471   1507         memcpy(pSyn->zTerm, pToken, nToken);
  1472   1508         pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
  1473   1509         pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
  1474   1510       }
  1475   1511     }else{
  1476   1512       Fts5ExprTerm *pTerm;
  1477   1513       if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
................................................................................
  1556   1592       rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
  1557   1593     }
  1558   1594     sqlite3_free(z);
  1559   1595     if( rc || (rc = sCtx.rc) ){
  1560   1596       pParse->rc = rc;
  1561   1597       fts5ExprPhraseFree(sCtx.pPhrase);
  1562   1598       sCtx.pPhrase = 0;
  1563         -  }else if( sCtx.pPhrase ){
         1599  +  }else{
  1564   1600   
  1565   1601       if( pAppend==0 ){
  1566   1602         if( (pParse->nPhrase % 8)==0 ){
  1567   1603           int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
  1568   1604           Fts5ExprPhrase **apNew;
  1569   1605           apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
  1570   1606           if( apNew==0 ){
................................................................................
  1573   1609             return 0;
  1574   1610           }
  1575   1611           pParse->apPhrase = apNew;
  1576   1612         }
  1577   1613         pParse->nPhrase++;
  1578   1614       }
  1579   1615   
         1616  +    if( sCtx.pPhrase==0 ){
         1617  +      /* This happens when parsing a token or quoted phrase that contains
         1618  +      ** no token characters at all. (e.g ... MATCH '""'). */
         1619  +      sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
         1620  +    }else if( sCtx.pPhrase->nTerm ){
         1621  +      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
         1622  +    }
  1580   1623       pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  1581         -    assert( sCtx.pPhrase->nTerm>0 );
  1582         -    sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
  1583   1624     }
  1584   1625   
  1585   1626     return sCtx.pPhrase;
  1586   1627   }
  1587   1628   
  1588   1629   /*
  1589   1630   ** Create a new FTS5 expression by cloning phrase iPhrase of the
  1590   1631   ** expression passed as the second argument.
  1591   1632   */
  1592   1633   int sqlite3Fts5ExprClonePhrase(
  1593         -  Fts5Config *pConfig,
  1594   1634     Fts5Expr *pExpr, 
  1595   1635     int iPhrase, 
  1596   1636     Fts5Expr **ppNew
  1597   1637   ){
  1598   1638     int rc = SQLITE_OK;             /* Return code */
  1599   1639     Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
  1600   1640     int i;                          /* Used to iterate through phrase terms */
  1601         -
  1602   1641     Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
  1603         -
  1604   1642     TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
  1605   1643   
  1606         -
  1607   1644     pOrig = pExpr->apExprPhrase[iPhrase];
  1608         -
  1609   1645     pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
  1610   1646     if( rc==SQLITE_OK ){
  1611   1647       pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, 
  1612   1648           sizeof(Fts5ExprPhrase*));
  1613   1649     }
  1614   1650     if( rc==SQLITE_OK ){
  1615   1651       pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, 
................................................................................
  1642   1678       pNew->apExprPhrase[0] = sCtx.pPhrase;
  1643   1679       pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
  1644   1680       pNew->pRoot->pNear->nPhrase = 1;
  1645   1681       sCtx.pPhrase->pNode = pNew->pRoot;
  1646   1682   
  1647   1683       if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
  1648   1684         pNew->pRoot->eType = FTS5_TERM;
         1685  +      pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
  1649   1686       }else{
  1650   1687         pNew->pRoot->eType = FTS5_STRING;
         1688  +      pNew->pRoot->xNext = fts5ExprNodeNext_STRING;
  1651   1689       }
  1652   1690     }else{
  1653   1691       sqlite3Fts5ExprFree(pNew);
  1654   1692       fts5ExprPhraseFree(sCtx.pPhrase);
  1655   1693       pNew = 0;
  1656   1694     }
  1657   1695   
................................................................................
  1674   1712   }
  1675   1713   
  1676   1714   void sqlite3Fts5ParseSetDistance(
  1677   1715     Fts5Parse *pParse, 
  1678   1716     Fts5ExprNearset *pNear,
  1679   1717     Fts5Token *p
  1680   1718   ){
  1681         -  int nNear = 0;
  1682         -  int i;
  1683         -  if( p->n ){
  1684         -    for(i=0; i<p->n; i++){
  1685         -      char c = (char)p->p[i];
  1686         -      if( c<'0' || c>'9' ){
  1687         -        sqlite3Fts5ParseError(
  1688         -            pParse, "expected integer, got \"%.*s\"", p->n, p->p
  1689         -        );
  1690         -        return;
         1719  +  if( pNear ){
         1720  +    int nNear = 0;
         1721  +    int i;
         1722  +    if( p->n ){
         1723  +      for(i=0; i<p->n; i++){
         1724  +        char c = (char)p->p[i];
         1725  +        if( c<'0' || c>'9' ){
         1726  +          sqlite3Fts5ParseError(
         1727  +              pParse, "expected integer, got \"%.*s\"", p->n, p->p
         1728  +              );
         1729  +          return;
         1730  +        }
         1731  +        nNear = nNear * 10 + (p->p[i] - '0');
  1691   1732         }
  1692         -      nNear = nNear * 10 + (p->p[i] - '0');
         1733  +    }else{
         1734  +      nNear = FTS5_DEFAULT_NEARDIST;
  1693   1735       }
  1694         -  }else{
  1695         -    nNear = FTS5_DEFAULT_NEARDIST;
         1736  +    pNear->nNear = nNear;
  1696   1737     }
  1697         -  pNear->nNear = nNear;
  1698   1738   }
  1699   1739   
  1700   1740   /*
  1701   1741   ** The second argument passed to this function may be NULL, or it may be
  1702   1742   ** an existing Fts5Colset object. This function returns a pointer to
  1703   1743   ** a new colset object containing the contents of (p) with new value column
  1704   1744   ** number iCol appended. 
................................................................................
  1790   1830   
  1791   1831     if( pNear ){
  1792   1832       pNear->pColset = pColset;
  1793   1833     }else{
  1794   1834       sqlite3_free(pColset);
  1795   1835     }
  1796   1836   }
         1837  +
         1838  +static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
         1839  +  switch( pNode->eType ){
         1840  +    case FTS5_STRING: {
         1841  +      Fts5ExprNearset *pNear = pNode->pNear;
         1842  +      if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
         1843  +       && pNear->apPhrase[0]->aTerm[0].pSynonym==0
         1844  +      ){
         1845  +        pNode->eType = FTS5_TERM;
         1846  +        pNode->xNext = fts5ExprNodeNext_TERM;
         1847  +      }else{
         1848  +        pNode->xNext = fts5ExprNodeNext_STRING;
         1849  +      }
         1850  +      break;
         1851  +    };
         1852  +
         1853  +    case FTS5_OR: {
         1854  +      pNode->xNext = fts5ExprNodeNext_OR;
         1855  +      break;
         1856  +    };
         1857  +
         1858  +    case FTS5_AND: {
         1859  +      pNode->xNext = fts5ExprNodeNext_AND;
         1860  +      break;
         1861  +    };
         1862  +
         1863  +    default: assert( pNode->eType==FTS5_NOT ); {
         1864  +      pNode->xNext = fts5ExprNodeNext_NOT;
         1865  +      break;
         1866  +    };
         1867  +  }
         1868  +}
  1797   1869   
  1798   1870   static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
  1799   1871     if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
  1800   1872       int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
  1801   1873       memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
  1802   1874       p->nChild += pSub->nChild;
  1803   1875       sqlite3_free(pSub);
................................................................................
  1840   1912   
  1841   1913       nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1);
  1842   1914       pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
  1843   1915   
  1844   1916       if( pRet ){
  1845   1917         pRet->eType = eType;
  1846   1918         pRet->pNear = pNear;
         1919  +      fts5ExprAssignXNext(pRet);
  1847   1920         if( eType==FTS5_STRING ){
  1848   1921           int iPhrase;
  1849   1922           for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
  1850   1923             pNear->apPhrase[iPhrase]->pNode = pRet;
  1851         -        }
  1852         -        if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
  1853         -          if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){
  1854         -            pRet->eType = FTS5_TERM;
         1924  +          if( pNear->apPhrase[iPhrase]->nTerm==0 ){
         1925  +            pRet->xNext = 0;
         1926  +            pRet->eType = FTS5_EOF;
  1855   1927             }
  1856         -        }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
         1928  +        }
         1929  +
         1930  +        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
         1931  +         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
         1932  +        ){
  1857   1933             assert( pParse->rc==SQLITE_OK );
  1858   1934             pParse->rc = SQLITE_ERROR;
  1859   1935             assert( pParse->zErr==0 );
  1860   1936             pParse->zErr = sqlite3_mprintf(
  1861   1937                 "fts5: %s queries are not supported (detail!=full)", 
  1862   1938                 pNear->nPhrase==1 ? "phrase": "NEAR"
  1863   1939             );
  1864   1940             sqlite3_free(pRet);
  1865   1941             pRet = 0;
  1866   1942           }
         1943  +
  1867   1944         }else{
  1868   1945           fts5ExprAddChildren(pRet, pLeft);
  1869   1946           fts5ExprAddChildren(pRet, pRight);
  1870   1947         }
  1871   1948       }
  1872   1949     }
  1873   1950   
  1874   1951     if( pRet==0 ){
  1875   1952       assert( pParse->rc!=SQLITE_OK );
  1876   1953       sqlite3Fts5ParseNodeFree(pLeft);
  1877   1954       sqlite3Fts5ParseNodeFree(pRight);
  1878   1955       sqlite3Fts5ParseNearsetFree(pNear);
  1879   1956     }
         1957  +  return pRet;
         1958  +}
         1959  +
         1960  +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
         1961  +  Fts5Parse *pParse,              /* Parse context */
         1962  +  Fts5ExprNode *pLeft,            /* Left hand child expression */
         1963  +  Fts5ExprNode *pRight            /* Right hand child expression */
         1964  +){
         1965  +  Fts5ExprNode *pRet = 0;
         1966  +  Fts5ExprNode *pPrev;
         1967  +
         1968  +  if( pParse->rc ){
         1969  +    sqlite3Fts5ParseNodeFree(pLeft);
         1970  +    sqlite3Fts5ParseNodeFree(pRight);
         1971  +  }else{
         1972  +
         1973  +    assert( pLeft->eType==FTS5_STRING 
         1974  +        || pLeft->eType==FTS5_TERM
         1975  +        || pLeft->eType==FTS5_EOF
         1976  +        || pLeft->eType==FTS5_AND
         1977  +    );
         1978  +    assert( pRight->eType==FTS5_STRING 
         1979  +        || pRight->eType==FTS5_TERM 
         1980  +        || pRight->eType==FTS5_EOF 
         1981  +    );
         1982  +
         1983  +    if( pLeft->eType==FTS5_AND ){
         1984  +      pPrev = pLeft->apChild[pLeft->nChild-1];
         1985  +    }else{
         1986  +      pPrev = pLeft;
         1987  +    }
         1988  +    assert( pPrev->eType==FTS5_STRING 
         1989  +        || pPrev->eType==FTS5_TERM 
         1990  +        || pPrev->eType==FTS5_EOF 
         1991  +        );
         1992  +
         1993  +    if( pRight->eType==FTS5_EOF ){
         1994  +      assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
         1995  +      sqlite3Fts5ParseNodeFree(pRight);
         1996  +      pRet = pLeft;
         1997  +      pParse->nPhrase--;
         1998  +    }
         1999  +    else if( pPrev->eType==FTS5_EOF ){
         2000  +      Fts5ExprPhrase **ap;
         2001  +
         2002  +      if( pPrev==pLeft ){
         2003  +        pRet = pRight;
         2004  +      }else{
         2005  +        pLeft->apChild[pLeft->nChild-1] = pRight;
         2006  +        pRet = pLeft;
         2007  +      }
         2008  +
         2009  +      ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase];
         2010  +      assert( ap[0]==pPrev->pNear->apPhrase[0] );
         2011  +      memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase);
         2012  +      pParse->nPhrase--;
         2013  +
         2014  +      sqlite3Fts5ParseNodeFree(pPrev);
         2015  +    }
         2016  +    else{
         2017  +      pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0);
         2018  +    }
         2019  +  }
         2020  +
  1880   2021     return pRet;
  1881   2022   }
  1882   2023   
  1883   2024   static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
  1884   2025     int nByte = 0;
  1885   2026     Fts5ExprTerm *p;
  1886   2027     char *zQuoted;
................................................................................
  2010   2151     }
  2011   2152   
  2012   2153     return zRet;
  2013   2154   }
  2014   2155   
  2015   2156   static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
  2016   2157     char *zRet = 0;
         2158  +  if( pExpr->eType==0 ){
         2159  +    return sqlite3_mprintf("\"\"");
         2160  +  }else
  2017   2161     if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){
  2018   2162       Fts5ExprNearset *pNear = pExpr->pNear;
  2019   2163       int i; 
  2020   2164       int iTerm;
  2021   2165   
  2022   2166       if( pNear->pColset ){
  2023   2167         int iCol = pNear->pColset->aiCol[0];
................................................................................
  2070   2214       for(i=0; i<pExpr->nChild; i++){
  2071   2215         char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]);
  2072   2216         if( z==0 ){
  2073   2217           sqlite3_free(zRet);
  2074   2218           zRet = 0;
  2075   2219         }else{
  2076   2220           int e = pExpr->apChild[i]->eType;
  2077         -        int b = (e!=FTS5_STRING && e!=FTS5_TERM);
         2221  +        int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF);
  2078   2222           zRet = fts5PrintfAppend(zRet, "%s%s%z%s", 
  2079   2223               (i==0 ? "" : zOp),
  2080   2224               (b?"(":""), z, (b?")":"")
  2081   2225           );
  2082   2226         }
  2083   2227         if( zRet==0 ) break;
  2084   2228       }
................................................................................
  2142   2286   
  2143   2287     rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  2144   2288     if( rc==SQLITE_OK ){
  2145   2289       rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
  2146   2290     }
  2147   2291     if( rc==SQLITE_OK ){
  2148   2292       char *zText;
  2149         -    if( pExpr->pRoot==0 ){
         2293  +    if( pExpr->pRoot->xNext==0 ){
  2150   2294         zText = sqlite3_mprintf("");
  2151   2295       }else if( bTcl ){
  2152   2296         zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
  2153   2297       }else{
  2154   2298         zText = fts5ExprPrint(pConfig, pExpr->pRoot);
  2155   2299       }
  2156   2300       if( zText==0 ){
................................................................................
  2242   2386       { "fts5_isalnum",  fts5ExprIsAlnum },
  2243   2387       { "fts5_fold",     fts5ExprFold },
  2244   2388     };
  2245   2389     int i;
  2246   2390     int rc = SQLITE_OK;
  2247   2391     void *pCtx = (void*)pGlobal;
  2248   2392   
  2249         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aFunc); i++){
         2393  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){
  2250   2394       struct Fts5ExprFunc *p = &aFunc[i];
  2251   2395       rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
  2252   2396     }
  2253   2397   
  2254   2398     /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
  2255   2399   #ifndef NDEBUG
  2256   2400     (void)sqlite3Fts5ParserTrace;
................................................................................
  2339   2483   }
  2340   2484   
  2341   2485   static int fts5ExprPopulatePoslistsCb(
  2342   2486     void *pCtx,                /* Copy of 2nd argument to xTokenize() */
  2343   2487     int tflags,                /* Mask of FTS5_TOKEN_* flags */
  2344   2488     const char *pToken,        /* Pointer to buffer containing token */
  2345   2489     int nToken,                /* Size of token in bytes */
  2346         -  int iStart,                /* Byte offset of token within input text */
  2347         -  int iEnd                   /* Byte offset of end of token within input text */
         2490  +  int iUnused1,              /* Byte offset of token within input text */
         2491  +  int iUnused2               /* Byte offset of end of token within input text */
  2348   2492   ){
  2349   2493     Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
  2350   2494     Fts5Expr *pExpr = p->pExpr;
  2351   2495     int i;
  2352   2496   
         2497  +  UNUSED_PARAM2(iUnused1, iUnused2);
         2498  +
         2499  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  2353   2500     if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
  2354   2501     for(i=0; i<pExpr->nPhrase; i++){
  2355   2502       Fts5ExprTerm *pTerm;
  2356   2503       if( p->aPopulator[i].bOk==0 ) continue;
  2357   2504       for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
  2358         -      int nTerm = strlen(pTerm->zTerm);
         2505  +      int nTerm = (int)strlen(pTerm->zTerm);
  2359   2506         if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
  2360   2507          && memcmp(pTerm->zTerm, pToken, nTerm)==0
  2361   2508         ){
  2362   2509           int rc = sqlite3Fts5PoslistWriterAppend(
  2363   2510               &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
  2364   2511           );
  2365   2512           if( rc ) return rc;
................................................................................
  2480   2627     int *pnCollist
  2481   2628   ){
  2482   2629     Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
  2483   2630     Fts5ExprNode *pNode = pPhrase->pNode;
  2484   2631     int rc = SQLITE_OK;
  2485   2632   
  2486   2633     assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
         2634  +  assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         2635  +
  2487   2636     if( pNode->bEof==0 
  2488   2637      && pNode->iRowid==pExpr->pRoot->iRowid 
  2489   2638      && pPhrase->poslist.n>0
  2490   2639     ){
  2491   2640       Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
  2492   2641       if( pTerm->pSynonym ){
  2493         -      int bDel = 0;
  2494         -      u8 *a;
         2642  +      Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1];
  2495   2643         rc = fts5ExprSynonymList(
  2496         -          pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist
         2644  +          pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
  2497   2645         );
  2498         -      if( bDel ){
  2499         -        sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a);
  2500         -        *ppCollist = pPhrase->poslist.p;
  2501         -        sqlite3_free(a);
  2502         -      }else{
  2503         -        *ppCollist = a;
  2504         -      }
  2505   2646       }else{
  2506         -      sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
         2647  +      *ppCollist = pPhrase->aTerm[0].pIter->pData;
         2648  +      *pnCollist = pPhrase->aTerm[0].pIter->nData;
  2507   2649       }
  2508   2650     }else{
  2509   2651       *ppCollist = 0;
  2510   2652       *pnCollist = 0;
  2511   2653     }
  2512   2654   
  2513   2655     return rc;
  2514   2656   }
  2515   2657   

Changes to ext/fts5/fts5_hash.c.

    58     58   struct Fts5HashEntry {
    59     59     Fts5HashEntry *pHashNext;       /* Next hash entry with same hash-key */
    60     60     Fts5HashEntry *pScanNext;       /* Next entry in sorted order */
    61     61     
    62     62     int nAlloc;                     /* Total size of allocation */
    63     63     int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
    64     64     int nData;                      /* Total bytes of data (incl. structure) */
           65  +  int nKey;                       /* Length of zKey[] in bytes */
    65     66     u8 bDel;                        /* Set delete-flag @ iSzPoslist */
    66     67     u8 bContent;                    /* Set content-flag (detail=none mode) */
    67         -
    68         -  int iCol;                       /* Column of last value written */
           68  +  i16 iCol;                       /* Column of last value written */
    69     69     int iPos;                       /* Position of last value written */
    70     70     i64 iRowid;                     /* Rowid of last value written */
    71     71     char zKey[8];                   /* Nul-terminated entry key */
    72     72   };
    73     73   
    74     74   /*
    75     75   ** Size of Fts5HashEntry without the zKey[] array.
................................................................................
   241    241     
   242    242     bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
   243    243   
   244    244     /* Attempt to locate an existing hash entry */
   245    245     iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   246    246     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   247    247       if( p->zKey[0]==bByte 
          248  +     && p->nKey==nToken
   248    249        && memcmp(&p->zKey[1], pToken, nToken)==0 
   249         -     && p->zKey[nToken+1]==0 
   250    250       ){
   251    251         break;
   252    252       }
   253    253     }
   254    254   
   255    255     /* If an existing hash entry cannot be found, create a new one. */
   256    256     if( p==0 ){
................................................................................
   269    269       p = (Fts5HashEntry*)sqlite3_malloc(nByte);
   270    270       if( !p ) return SQLITE_NOMEM;
   271    271       memset(p, 0, FTS5_HASHENTRYSIZE);
   272    272       p->nAlloc = nByte;
   273    273       p->zKey[0] = bByte;
   274    274       memcpy(&p->zKey[1], pToken, nToken);
   275    275       assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
          276  +    p->nKey = nToken;
   276    277       p->zKey[nToken+1] = '\0';
   277    278       p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
   278    279       p->pHashNext = pHash->aSlot[iHash];
   279    280       pHash->aSlot[iHash] = p;
   280    281       pHash->nEntry++;
   281    282   
   282    283       /* Add the first rowid field to the hash-entry */

Changes to ext/fts5/fts5_index.c.

   257    257   #define FTS5_DATA_ZERO_PADDING 8
   258    258   #define FTS5_DATA_PADDING 20
   259    259   
   260    260   typedef struct Fts5Data Fts5Data;
   261    261   typedef struct Fts5DlidxIter Fts5DlidxIter;
   262    262   typedef struct Fts5DlidxLvl Fts5DlidxLvl;
   263    263   typedef struct Fts5DlidxWriter Fts5DlidxWriter;
          264  +typedef struct Fts5Iter Fts5Iter;
   264    265   typedef struct Fts5PageWriter Fts5PageWriter;
   265    266   typedef struct Fts5SegIter Fts5SegIter;
   266    267   typedef struct Fts5DoclistIter Fts5DoclistIter;
   267    268   typedef struct Fts5SegWriter Fts5SegWriter;
   268    269   typedef struct Fts5Structure Fts5Structure;
   269    270   typedef struct Fts5StructureLevel Fts5StructureLevel;
   270    271   typedef struct Fts5StructureSegment Fts5StructureSegment;
................................................................................
   299    300     sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
   300    301     sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   301    302     sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   302    303     sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
   303    304     sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
   304    305     sqlite3_stmt *pIdxSelect;
   305    306     int nRead;                      /* Total number of blocks read */
          307  +
          308  +  sqlite3_stmt *pDataVersion;
          309  +  i64 iStructVersion;             /* data_version when pStruct read */
          310  +  Fts5Structure *pStruct;         /* Current db structure (or NULL) */
   306    311   };
   307    312   
   308    313   struct Fts5DoclistIter {
   309    314     u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
   310    315   
   311    316     /* Output variables. aPoslist==0 at EOF */
   312    317     i64 iRowid;
................................................................................
   499    504   ** aFirst[1] contains the index in aSeg[] of the iterator that points to
   500    505   ** the smallest key overall. aFirst[0] is unused. 
   501    506   **
   502    507   ** poslist:
   503    508   **   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
   504    509   **   There is no way to tell if this is populated or not.
   505    510   */
   506         -struct Fts5IndexIter {
          511  +struct Fts5Iter {
          512  +  Fts5IndexIter base;             /* Base class containing output vars */
          513  +
   507    514     Fts5Index *pIndex;              /* Index that owns this iterator */
   508    515     Fts5Structure *pStruct;         /* Database structure for this iterator */
   509    516     Fts5Buffer poslist;             /* Buffer containing current poslist */
          517  +  Fts5Colset *pColset;            /* Restrict matches to these columns */
          518  +
          519  +  /* Invoked to set output variables. */
          520  +  void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
   510    521   
   511    522     int nSeg;                       /* Size of aSeg[] array */
   512    523     int bRev;                       /* True to iterate in reverse order */
   513    524     u8 bSkipEmpty;                  /* True to skip deleted entries */
   514         -  u8 bEof;                        /* True at EOF */
   515         -  u8 bFiltered;                   /* True if column-filter already applied */
   516    525   
   517    526     i64 iSwitchRowid;               /* Firstest rowid of other than aFirst[1] */
   518    527     Fts5CResult *aFirst;            /* Current merge state (see above) */
   519    528     Fts5SegIter aSeg[1];            /* Array of segment iterators */
   520    529   };
   521    530   
   522    531   
................................................................................
   598    607   */
   599    608   static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){
   600    609     int nCmp = MIN(pLeft->n, pRight->n);
   601    610     int res = memcmp(pLeft->p, pRight->p, nCmp);
   602    611     return (res==0 ? (pLeft->n - pRight->n) : res);
   603    612   }
   604    613   
   605         -#ifdef SQLITE_DEBUG
   606         -static int fts5BlobCompare(
   607         -  const u8 *pLeft, int nLeft, 
   608         -  const u8 *pRight, int nRight
   609         -){
   610         -  int nCmp = MIN(nLeft, nRight);
   611         -  int res = memcmp(pLeft, pRight, nCmp);
   612         -  return (res==0 ? (nLeft - nRight) : res);
   613         -}
   614         -#endif
   615         -
   616    614   static int fts5LeafFirstTermOff(Fts5Data *pLeaf){
   617    615     int ret;
   618    616     fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret);
   619    617     return ret;
   620    618   }
   621    619   
   622    620   /*
................................................................................
   699    697       p->rc = rc;
   700    698       p->nRead++;
   701    699     }
   702    700   
   703    701     assert( (pRet==0)==(p->rc!=SQLITE_OK) );
   704    702     return pRet;
   705    703   }
          704  +
   706    705   
   707    706   /*
   708    707   ** Release a reference to data record returned by an earlier call to
   709    708   ** fts5DataRead().
   710    709   */
   711    710   static void fts5DataRelease(Fts5Data *pData){
   712    711     sqlite3_free(pData);
................................................................................
   867    866       pRet->nRef = 1;
   868    867       pRet->nLevel = nLevel;
   869    868       pRet->nSegment = nSegment;
   870    869       i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter);
   871    870   
   872    871       for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){
   873    872         Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl];
   874         -      int nTotal;
          873  +      int nTotal = 0;
   875    874         int iSeg;
   876    875   
   877         -      i += fts5GetVarint32(&pData[i], pLvl->nMerge);
   878         -      i += fts5GetVarint32(&pData[i], nTotal);
   879         -      assert( nTotal>=pLvl->nMerge );
   880         -      pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, 
   881         -          nTotal * sizeof(Fts5StructureSegment)
   882         -      );
          876  +      if( i>=nData ){
          877  +        rc = FTS5_CORRUPT;
          878  +      }else{
          879  +        i += fts5GetVarint32(&pData[i], pLvl->nMerge);
          880  +        i += fts5GetVarint32(&pData[i], nTotal);
          881  +        assert( nTotal>=pLvl->nMerge );
          882  +        pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, 
          883  +            nTotal * sizeof(Fts5StructureSegment)
          884  +        );
          885  +      }
   883    886   
   884    887         if( rc==SQLITE_OK ){
   885    888           pLvl->nSeg = nTotal;
   886    889           for(iSeg=0; iSeg<nTotal; iSeg++){
          890  +          if( i>=nData ){
          891  +            rc = FTS5_CORRUPT;
          892  +            break;
          893  +          }
   887    894             i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
   888    895             i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
   889    896             i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
   890    897           }
   891         -      }else{
   892         -        fts5StructureRelease(pRet);
   893         -        pRet = 0;
   894    898         }
          899  +    }
          900  +    if( rc!=SQLITE_OK ){
          901  +      fts5StructureRelease(pRet);
          902  +      pRet = 0;
   895    903       }
   896    904     }
   897    905   
   898    906     *ppOut = pRet;
   899    907     return rc;
   900    908   }
   901    909   
................................................................................
   950    958         }
   951    959         pLvl->aSeg = aNew;
   952    960       }else{
   953    961         *pRc = SQLITE_NOMEM;
   954    962       }
   955    963     }
   956    964   }
          965  +
          966  +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
          967  +  Fts5Structure *pRet = 0;
          968  +  Fts5Config *pConfig = p->pConfig;
          969  +  int iCookie;                    /* Configuration cookie */
          970  +  Fts5Data *pData;
          971  +
          972  +  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
          973  +  if( p->rc==SQLITE_OK ){
          974  +    /* TODO: Do we need this if the leaf-index is appended? Probably... */
          975  +    memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
          976  +    p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
          977  +    if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
          978  +      p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
          979  +    }
          980  +    fts5DataRelease(pData);
          981  +    if( p->rc!=SQLITE_OK ){
          982  +      fts5StructureRelease(pRet);
          983  +      pRet = 0;
          984  +    }
          985  +  }
          986  +
          987  +  return pRet;
          988  +}
          989  +
          990  +static i64 fts5IndexDataVersion(Fts5Index *p){
          991  +  i64 iVersion = 0;
          992  +
          993  +  if( p->rc==SQLITE_OK ){
          994  +    if( p->pDataVersion==0 ){
          995  +      p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, 
          996  +          sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
          997  +          );
          998  +      if( p->rc ) return 0;
          999  +    }
         1000  +
         1001  +    if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
         1002  +      iVersion = sqlite3_column_int64(p->pDataVersion, 0);
         1003  +    }
         1004  +    p->rc = sqlite3_reset(p->pDataVersion);
         1005  +  }
         1006  +
         1007  +  return iVersion;
         1008  +}
   957   1009   
   958   1010   /*
   959   1011   ** Read, deserialize and return the structure record.
   960   1012   **
   961   1013   ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
   962   1014   ** are over-allocated as described for function fts5StructureDecode() 
   963   1015   ** above.
   964   1016   **
   965   1017   ** If an error occurs, NULL is returned and an error code left in the
   966   1018   ** Fts5Index handle. If an error has already occurred when this function
   967   1019   ** is called, it is a no-op.
   968   1020   */
   969   1021   static Fts5Structure *fts5StructureRead(Fts5Index *p){
   970         -  Fts5Config *pConfig = p->pConfig;
   971         -  Fts5Structure *pRet = 0;        /* Object to return */
   972         -  int iCookie;                    /* Configuration cookie */
   973         -  Fts5Data *pData;
   974         -
   975         -  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
   976         -  if( p->rc ) return 0;
   977         -  /* TODO: Do we need this if the leaf-index is appended? Probably... */
   978         -  memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
   979         -  p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
   980         -  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
   981         -    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
   982         -  }
   983         -
   984         -  fts5DataRelease(pData);
   985         -  if( p->rc!=SQLITE_OK ){
   986         -    fts5StructureRelease(pRet);
   987         -    pRet = 0;
   988         -  }
   989         -  return pRet;
         1022  +
         1023  +  if( p->pStruct==0 ){
         1024  +    p->iStructVersion = fts5IndexDataVersion(p);
         1025  +    if( p->rc==SQLITE_OK ){
         1026  +      p->pStruct = fts5StructureReadUncached(p);
         1027  +    }
         1028  +  }
         1029  +
         1030  +#if 0
         1031  +  else{
         1032  +    Fts5Structure *pTest = fts5StructureReadUncached(p);
         1033  +    if( pTest ){
         1034  +      int i, j;
         1035  +      assert_nc( p->pStruct->nSegment==pTest->nSegment );
         1036  +      assert_nc( p->pStruct->nLevel==pTest->nLevel );
         1037  +      for(i=0; i<pTest->nLevel; i++){
         1038  +        assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
         1039  +        assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
         1040  +        for(j=0; j<pTest->aLevel[i].nSeg; j++){
         1041  +          Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
         1042  +          Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j];
         1043  +          assert_nc( p1->iSegid==p2->iSegid );
         1044  +          assert_nc( p1->pgnoFirst==p2->pgnoFirst );
         1045  +          assert_nc( p1->pgnoLast==p2->pgnoLast );
         1046  +        }
         1047  +      }
         1048  +      fts5StructureRelease(pTest);
         1049  +    }
         1050  +  }
         1051  +#endif
         1052  +
         1053  +  if( p->rc!=SQLITE_OK ) return 0;
         1054  +  assert( p->iStructVersion!=0 );
         1055  +  assert( p->pStruct!=0 );
         1056  +  fts5StructureRef(p->pStruct);
         1057  +  return p->pStruct;
         1058  +}
         1059  +
         1060  +static void fts5StructureInvalidate(Fts5Index *p){
         1061  +  if( p->pStruct ){
         1062  +    fts5StructureRelease(p->pStruct);
         1063  +    p->pStruct = 0;
         1064  +  }
   990   1065   }
   991   1066   
   992   1067   /*
   993   1068   ** Return the total number of segments in index structure pStruct. This
   994   1069   ** function is only ever used as part of assert() conditions.
   995   1070   */
   996   1071   #ifdef SQLITE_DEBUG
................................................................................
  1555   1630   */
  1556   1631   static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
  1557   1632     u8 *a = pIter->pLeaf->p;        /* Buffer to read data from */
  1558   1633     int iOff = pIter->iLeafOffset;  /* Offset to read at */
  1559   1634     int nNew;                       /* Bytes of new data */
  1560   1635   
  1561   1636     iOff += fts5GetVarint32(&a[iOff], nNew);
         1637  +  if( iOff+nNew>pIter->pLeaf->nn ){
         1638  +    p->rc = FTS5_CORRUPT;
         1639  +    return;
         1640  +  }
  1562   1641     pIter->term.n = nKeep;
  1563   1642     fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
  1564   1643     iOff += nNew;
  1565   1644     pIter->iTermLeafOffset = iOff;
  1566   1645     pIter->iTermLeafPgno = pIter->iLeafPgno;
  1567   1646     pIter->iLeafOffset = iOff;
  1568   1647   
................................................................................
  1748   1827   }
  1749   1828   
  1750   1829   /*
  1751   1830   ** Return true if the iterator passed as the second argument currently
  1752   1831   ** points to a delete marker. A delete marker is an entry with a 0 byte
  1753   1832   ** position-list.
  1754   1833   */
  1755         -static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){
         1834  +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){
  1756   1835     Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
  1757   1836     return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
  1758   1837   }
  1759   1838   
  1760   1839   /*
  1761   1840   ** Advance iterator pIter to the next entry.
  1762   1841   **
  1763   1842   ** This version of fts5SegIterNext() is only used by reverse iterators.
  1764   1843   */
  1765   1844   static void fts5SegIterNext_Reverse(
  1766   1845     Fts5Index *p,                   /* FTS5 backend object */
  1767   1846     Fts5SegIter *pIter,             /* Iterator to advance */
  1768         -  int *pbNewTerm                  /* OUT: Set for new term */
         1847  +  int *pbUnused                   /* Unused */
  1769   1848   ){
  1770   1849     assert( pIter->flags & FTS5_SEGITER_REVERSE );
  1771   1850     assert( pIter->pNextLeaf==0 );
         1851  +  UNUSED_PARAM(pbUnused);
         1852  +
  1772   1853     if( pIter->iRowidOffset>0 ){
  1773   1854       u8 *a = pIter->pLeaf->p;
  1774   1855       int iOff;
  1775   1856       i64 iDelta;
  1776   1857   
  1777   1858       pIter->iRowidOffset--;
  1778   1859       pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset];
................................................................................
  2020   2101       int iPoslist;
  2021   2102       if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
  2022   2103         iPoslist = pIter->iTermLeafOffset;
  2023   2104       }else{
  2024   2105         iPoslist = 4;
  2025   2106       }
  2026   2107       fts5IndexSkipVarint(pLeaf->p, iPoslist);
  2027         -    assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==(
  2028         -        pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel)
  2029         -    ));
  2030   2108       pIter->iLeafOffset = iPoslist;
  2031   2109   
  2032   2110       /* If this condition is true then the largest rowid for the current
  2033   2111       ** term may not be stored on the current page. So search forward to
  2034   2112       ** see where said rowid really is.  */
  2035   2113       if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
  2036   2114         int pgno;
................................................................................
  2144   2222     int bEndOfPage = 0;
  2145   2223   
  2146   2224     assert( p->rc==SQLITE_OK );
  2147   2225   
  2148   2226     iPgidx = szLeaf;
  2149   2227     iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
  2150   2228     iOff = iTermOff;
         2229  +  if( iOff>n ){
         2230  +    p->rc = FTS5_CORRUPT;
         2231  +    return;
         2232  +  }
  2151   2233   
  2152   2234     while( 1 ){
  2153   2235   
  2154   2236       /* Figure out how many new bytes are in this term */
  2155   2237       fts5FastGetVarint32(a, iOff, nNew);
  2156   2238       if( nKeep<nMatch ){
  2157   2239         goto search_failed;
................................................................................
  2234   2316       pIter->iEndofDoclist = iTermOff + nExtra;
  2235   2317     }
  2236   2318     pIter->iPgidxOff = iPgidx;
  2237   2319   
  2238   2320     fts5SegIterLoadRowid(p, pIter);
  2239   2321     fts5SegIterLoadNPos(p, pIter);
  2240   2322   }
         2323  +
         2324  +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){
         2325  +  if( p->pIdxSelect==0 ){
         2326  +    Fts5Config *pConfig = p->pConfig;
         2327  +    fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
         2328  +          "SELECT pgno FROM '%q'.'%q_idx' WHERE "
         2329  +          "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
         2330  +          pConfig->zDb, pConfig->zName
         2331  +    ));
         2332  +  }
         2333  +  return p->pIdxSelect;
         2334  +}
  2241   2335   
  2242   2336   /*
  2243   2337   ** Initialize the object pIter to point to term pTerm/nTerm within segment
  2244   2338   ** pSeg. If there is no such term in the index, the iterator is set to EOF.
  2245   2339   **
  2246   2340   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
  2247   2341   ** an error has already occurred when this function is called, it is a no-op.
  2248   2342   */
  2249   2343   static void fts5SegIterSeekInit(
  2250   2344     Fts5Index *p,                   /* FTS5 backend */
  2251         -  Fts5Buffer *pBuf,               /* Buffer to use for loading pages */
  2252   2345     const u8 *pTerm, int nTerm,     /* Term to seek to */
  2253   2346     int flags,                      /* Mask of FTS5INDEX_XXX flags */
  2254   2347     Fts5StructureSegment *pSeg,     /* Description of segment */
  2255   2348     Fts5SegIter *pIter              /* Object to populate */
  2256   2349   ){
  2257   2350     int iPg = 1;
  2258   2351     int bGe = (flags & FTS5INDEX_QUERY_SCAN);
  2259   2352     int bDlidx = 0;                 /* True if there is a doclist-index */
  2260         -
  2261         -  static int nCall = 0;
  2262         -  nCall++;
         2353  +  sqlite3_stmt *pIdxSelect = 0;
  2263   2354   
  2264   2355     assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
  2265   2356     assert( pTerm && nTerm );
  2266   2357     memset(pIter, 0, sizeof(*pIter));
  2267   2358     pIter->pSeg = pSeg;
  2268   2359   
  2269   2360     /* This block sets stack variable iPg to the leaf page number that may
  2270   2361     ** contain term (pTerm/nTerm), if it is present in the segment. */
  2271         -  if( p->pIdxSelect==0 ){
  2272         -    Fts5Config *pConfig = p->pConfig;
  2273         -    fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
  2274         -          "SELECT pgno FROM '%q'.'%q_idx' WHERE "
  2275         -          "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
  2276         -          pConfig->zDb, pConfig->zName
  2277         -    ));
  2278         -  }
         2362  +  pIdxSelect = fts5IdxSelectStmt(p);
  2279   2363     if( p->rc ) return;
  2280         -  sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid);
  2281         -  sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
  2282         -  if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){
  2283         -    i64 val = sqlite3_column_int(p->pIdxSelect, 0);
         2364  +  sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid);
         2365  +  sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
         2366  +  if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){
         2367  +    i64 val = sqlite3_column_int(pIdxSelect, 0);
  2284   2368       iPg = (int)(val>>1);
  2285   2369       bDlidx = (val & 0x0001);
  2286   2370     }
  2287         -  p->rc = sqlite3_reset(p->pIdxSelect);
         2371  +  p->rc = sqlite3_reset(pIdxSelect);
  2288   2372   
  2289   2373     if( iPg<pSeg->pgnoFirst ){
  2290   2374       iPg = pSeg->pgnoFirst;
  2291   2375       bDlidx = 0;
  2292   2376     }
  2293   2377   
  2294   2378     pIter->iLeafPgno = iPg - 1;
................................................................................
  2402   2486   /*
  2403   2487   ** This function is used as part of the big assert() procedure implemented by
  2404   2488   ** fts5AssertMultiIterSetup(). It ensures that the result currently stored
  2405   2489   ** in *pRes is the correct result of comparing the current positions of the
  2406   2490   ** two iterators.
  2407   2491   */
  2408   2492   static void fts5AssertComparisonResult(
  2409         -  Fts5IndexIter *pIter, 
         2493  +  Fts5Iter *pIter, 
  2410   2494     Fts5SegIter *p1,
  2411   2495     Fts5SegIter *p2,
  2412   2496     Fts5CResult *pRes
  2413   2497   ){
  2414   2498     int i1 = p1 - pIter->aSeg;
  2415   2499     int i2 = p2 - pIter->aSeg;
  2416   2500   
................................................................................
  2443   2527   
  2444   2528   /*
  2445   2529   ** This function is a no-op unless SQLITE_DEBUG is defined when this module
  2446   2530   ** is compiled. In that case, this function is essentially an assert() 
  2447   2531   ** statement used to verify that the contents of the pIter->aFirst[] array
  2448   2532   ** are correct.
  2449   2533   */
  2450         -static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
         2534  +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){
  2451   2535     if( p->rc==SQLITE_OK ){
  2452   2536       Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  2453   2537       int i;
  2454   2538   
  2455         -    assert( (pFirst->pLeaf==0)==pIter->bEof );
         2539  +    assert( (pFirst->pLeaf==0)==pIter->base.bEof );
  2456   2540   
  2457   2541       /* Check that pIter->iSwitchRowid is set correctly. */
  2458   2542       for(i=0; i<pIter->nSeg; i++){
  2459   2543         Fts5SegIter *p1 = &pIter->aSeg[i];
  2460   2544         assert( p1==pFirst 
  2461   2545              || p1->pLeaf==0 
  2462   2546              || fts5BufferCompare(&pFirst->term, &p1->term) 
................................................................................
  2488   2572   ** Do the comparison necessary to populate pIter->aFirst[iOut].
  2489   2573   **
  2490   2574   ** If the returned value is non-zero, then it is the index of an entry
  2491   2575   ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing
  2492   2576   ** to a key that is a duplicate of another, higher priority, 
  2493   2577   ** segment-iterator in the pSeg->aSeg[] array.
  2494   2578   */
  2495         -static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){
         2579  +static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
  2496   2580     int i1;                         /* Index of left-hand Fts5SegIter */
  2497   2581     int i2;                         /* Index of right-hand Fts5SegIter */
  2498   2582     int iRes;
  2499   2583     Fts5SegIter *p1;                /* Left-hand Fts5SegIter */
  2500   2584     Fts5SegIter *p2;                /* Right-hand Fts5SegIter */
  2501   2585     Fts5CResult *pRes = &pIter->aFirst[iOut];
  2502   2586   
................................................................................
  2634   2718     }while( p->rc==SQLITE_OK );
  2635   2719   }
  2636   2720   
  2637   2721   
  2638   2722   /*
  2639   2723   ** Free the iterator object passed as the second argument.
  2640   2724   */
  2641         -static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){
         2725  +static void fts5MultiIterFree(Fts5Iter *pIter){
  2642   2726     if( pIter ){
  2643   2727       int i;
  2644   2728       for(i=0; i<pIter->nSeg; i++){
  2645   2729         fts5SegIterClear(&pIter->aSeg[i]);
  2646   2730       }
  2647   2731       fts5StructureRelease(pIter->pStruct);
  2648   2732       fts5BufferFree(&pIter->poslist);
  2649   2733       sqlite3_free(pIter);
  2650   2734     }
  2651   2735   }
  2652   2736   
  2653   2737   static void fts5MultiIterAdvanced(
  2654   2738     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2655         -  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
         2739  +  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
  2656   2740     int iChanged,                   /* Index of sub-iterator just advanced */
  2657   2741     int iMinset                     /* Minimum entry in aFirst[] to set */
  2658   2742   ){
  2659   2743     int i;
  2660   2744     for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){
  2661   2745       int iEq;
  2662   2746       if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
................................................................................
  2675   2759   ** If it does so successfully, 0 is returned. Otherwise 1.
  2676   2760   **
  2677   2761   ** If non-zero is returned, the caller should call fts5MultiIterAdvanced()
  2678   2762   ** on the iterator instead. That function does the same as this one, except
  2679   2763   ** that it deals with more complicated cases as well.
  2680   2764   */ 
  2681   2765   static int fts5MultiIterAdvanceRowid(
  2682         -  Fts5Index *p,                   /* FTS5 backend to iterate within */
  2683         -  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
  2684         -  int iChanged                    /* Index of sub-iterator just advanced */
         2766  +  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
         2767  +  int iChanged,                   /* Index of sub-iterator just advanced */
         2768  +  Fts5SegIter **ppFirst
  2685   2769   ){
  2686   2770     Fts5SegIter *pNew = &pIter->aSeg[iChanged];
  2687   2771   
  2688   2772     if( pNew->iRowid==pIter->iSwitchRowid
  2689   2773      || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev
  2690   2774     ){
  2691   2775       int i;
................................................................................
  2710   2794         pRes->iFirst = (u16)(pNew - pIter->aSeg);
  2711   2795         if( i==1 ) break;
  2712   2796   
  2713   2797         pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
  2714   2798       }
  2715   2799     }
  2716   2800   
         2801  +  *ppFirst = pNew;
  2717   2802     return 0;
  2718   2803   }
  2719   2804   
  2720   2805   /*
  2721   2806   ** Set the pIter->bEof variable based on the state of the sub-iterators.
  2722   2807   */
  2723         -static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
         2808  +static void fts5MultiIterSetEof(Fts5Iter *pIter){
  2724   2809     Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  2725         -  pIter->bEof = pSeg->pLeaf==0;
         2810  +  pIter->base.bEof = pSeg->pLeaf==0;
  2726   2811     pIter->iSwitchRowid = pSeg->iRowid;
  2727   2812   }
  2728   2813   
  2729   2814   /*
  2730   2815   ** Move the iterator to the next entry. 
  2731   2816   **
  2732   2817   ** If an error occurs, an error code is left in Fts5Index.rc. It is not 
  2733   2818   ** considered an error if the iterator reaches EOF, or if it is already at 
  2734   2819   ** EOF when this function is called.
  2735   2820   */
  2736   2821   static void fts5MultiIterNext(
  2737   2822     Fts5Index *p, 
  2738         -  Fts5IndexIter *pIter,
         2823  +  Fts5Iter *pIter,
  2739   2824     int bFrom,                      /* True if argument iFrom is valid */
  2740   2825     i64 iFrom                       /* Advance at least as far as this */
  2741   2826   ){
  2742         -  if( p->rc==SQLITE_OK ){
  2743         -    int bUseFrom = bFrom;
  2744         -    do {
  2745         -      int iFirst = pIter->aFirst[1].iFirst;
  2746         -      int bNewTerm = 0;
  2747         -      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2748         -      assert( p->rc==SQLITE_OK );
  2749         -      if( bUseFrom && pSeg->pDlidx ){
  2750         -        fts5SegIterNextFrom(p, pSeg, iFrom);
  2751         -      }else{
  2752         -        pSeg->xNext(p, pSeg, &bNewTerm);
  2753         -      }
  2754         -
  2755         -      if( pSeg->pLeaf==0 || bNewTerm 
  2756         -       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
  2757         -      ){
  2758         -        fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2759         -        fts5MultiIterSetEof(pIter);
  2760         -      }
  2761         -      fts5AssertMultiIterSetup(p, pIter);
  2762         -
  2763         -      bUseFrom = 0;
  2764         -    }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) );
         2827  +  int bUseFrom = bFrom;
         2828  +  while( p->rc==SQLITE_OK ){
         2829  +    int iFirst = pIter->aFirst[1].iFirst;
         2830  +    int bNewTerm = 0;
         2831  +    Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
         2832  +    assert( p->rc==SQLITE_OK );
         2833  +    if( bUseFrom && pSeg->pDlidx ){
         2834  +      fts5SegIterNextFrom(p, pSeg, iFrom);
         2835  +    }else{
         2836  +      pSeg->xNext(p, pSeg, &bNewTerm);
         2837  +    }
         2838  +
         2839  +    if( pSeg->pLeaf==0 || bNewTerm 
         2840  +     || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
         2841  +    ){
         2842  +      fts5MultiIterAdvanced(p, pIter, iFirst, 1);
         2843  +      fts5MultiIterSetEof(pIter);
         2844  +      pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
         2845  +      if( pSeg->pLeaf==0 ) return;
         2846  +    }
         2847  +
         2848  +    fts5AssertMultiIterSetup(p, pIter);
         2849  +    assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf );
         2850  +    if( pIter->bSkipEmpty==0 || pSeg->nPos ){
         2851  +      pIter->xSetOutputs(pIter, pSeg);
         2852  +      return;
         2853  +    }
         2854  +    bUseFrom = 0;
  2765   2855     }
  2766   2856   }
  2767   2857   
  2768   2858   static void fts5MultiIterNext2(
  2769   2859     Fts5Index *p, 
  2770         -  Fts5IndexIter *pIter,
         2860  +  Fts5Iter *pIter,
  2771   2861     int *pbNewTerm                  /* OUT: True if *might* be new term */
  2772   2862   ){
  2773   2863     assert( pIter->bSkipEmpty );
  2774   2864     if( p->rc==SQLITE_OK ){
  2775   2865       do {
  2776   2866         int iFirst = pIter->aFirst[1].iFirst;
  2777   2867         Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2778   2868         int bNewTerm = 0;
  2779   2869   
  2780   2870         assert( p->rc==SQLITE_OK );
  2781   2871         pSeg->xNext(p, pSeg, &bNewTerm);
  2782   2872         if( pSeg->pLeaf==0 || bNewTerm 
  2783         -       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
         2873  +       || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
  2784   2874         ){
  2785   2875           fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2786   2876           fts5MultiIterSetEof(pIter);
  2787   2877           *pbNewTerm = 1;
  2788   2878         }else{
  2789   2879           *pbNewTerm = 0;
  2790   2880         }
  2791   2881         fts5AssertMultiIterSetup(p, pIter);
  2792   2882   
  2793   2883       }while( fts5MultiIterIsEmpty(p, pIter) );
  2794   2884     }
  2795   2885   }
  2796   2886   
         2887  +static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){
         2888  +  UNUSED_PARAM2(pUnused1, pUnused2);
         2889  +}
  2797   2890   
  2798         -static Fts5IndexIter *fts5MultiIterAlloc(
         2891  +static Fts5Iter *fts5MultiIterAlloc(
  2799   2892     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2800   2893     int nSeg
  2801   2894   ){
  2802         -  Fts5IndexIter *pNew;
         2895  +  Fts5Iter *pNew;
  2803   2896     int nSlot;                      /* Power of two >= nSeg */
  2804   2897   
  2805   2898     for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
  2806   2899     pNew = fts5IdxMalloc(p, 
  2807         -      sizeof(Fts5IndexIter) +             /* pNew */
         2900  +      sizeof(Fts5Iter) +                  /* pNew */
  2808   2901         sizeof(Fts5SegIter) * (nSlot-1) +   /* pNew->aSeg[] */
  2809   2902         sizeof(Fts5CResult) * nSlot         /* pNew->aFirst[] */
  2810   2903     );
  2811   2904     if( pNew ){
  2812   2905       pNew->nSeg = nSlot;
  2813   2906       pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
  2814   2907       pNew->pIndex = p;
         2908  +    pNew->xSetOutputs = fts5IterSetOutputs_Noop;
  2815   2909     }
  2816   2910     return pNew;
  2817   2911   }
  2818   2912   
  2819         -/*
  2820         -** Allocate a new Fts5IndexIter object.
  2821         -**
  2822         -** The new object will be used to iterate through data in structure pStruct.
  2823         -** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
  2824         -** is zero or greater, data from the first nSegment segments on level iLevel
  2825         -** is merged.
  2826         -**
  2827         -** The iterator initially points to the first term/rowid entry in the 
  2828         -** iterated data.
  2829         -*/
  2830         -static void fts5MultiIterNew(
  2831         -  Fts5Index *p,                   /* FTS5 backend to iterate within */
  2832         -  Fts5Structure *pStruct,         /* Structure of specific index */
  2833         -  int bSkipEmpty,                 /* True to ignore delete-keys */
  2834         -  int flags,                      /* FTS5INDEX_QUERY_XXX flags */
  2835         -  const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
  2836         -  int iLevel,                     /* Level to iterate (-1 for all) */
  2837         -  int nSegment,                   /* Number of segments to merge (iLevel>=0) */
  2838         -  Fts5IndexIter **ppOut           /* New object */
  2839         -){
  2840         -  int nSeg = 0;                   /* Number of segment-iters in use */
  2841         -  int iIter = 0;                  /* */
  2842         -  int iSeg;                       /* Used to iterate through segments */
  2843         -  Fts5Buffer buf = {0,0,0};       /* Buffer used by fts5SegIterSeekInit() */
  2844         -  Fts5StructureLevel *pLvl;
  2845         -  Fts5IndexIter *pNew;
  2846         -
  2847         -  assert( (pTerm==0 && nTerm==0) || iLevel<0 );
  2848         -
  2849         -  /* Allocate space for the new multi-seg-iterator. */
  2850         -  if( p->rc==SQLITE_OK ){
  2851         -    if( iLevel<0 ){
  2852         -      assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  2853         -      nSeg = pStruct->nSegment;
  2854         -      nSeg += (p->pHash ? 1 : 0);
  2855         -    }else{
  2856         -      nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
  2857         -    }
  2858         -  }
  2859         -  *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
  2860         -  if( pNew==0 ) return;
  2861         -  pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
  2862         -  pNew->bSkipEmpty = (u8)bSkipEmpty;
  2863         -  pNew->pStruct = pStruct;
  2864         -  fts5StructureRef(pStruct);
  2865         -
  2866         -  /* Initialize each of the component segment iterators. */
  2867         -  if( iLevel<0 ){
  2868         -    Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
  2869         -    if( p->pHash ){
  2870         -      /* Add a segment iterator for the current contents of the hash table. */
  2871         -      Fts5SegIter *pIter = &pNew->aSeg[iIter++];
  2872         -      fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
  2873         -    }
  2874         -    for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
  2875         -      for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
  2876         -        Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
  2877         -        Fts5SegIter *pIter = &pNew->aSeg[iIter++];
  2878         -        if( pTerm==0 ){
  2879         -          fts5SegIterInit(p, pSeg, pIter);
         2913  +static void fts5PoslistCallback(
         2914  +  Fts5Index *pUnused, 
         2915  +  void *pContext, 
         2916  +  const u8 *pChunk, int nChunk
         2917  +){
         2918  +  UNUSED_PARAM(pUnused);
         2919  +  assert_nc( nChunk>=0 );
         2920  +  if( nChunk>0 ){
         2921  +    fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk);
         2922  +  }
         2923  +}
         2924  +
         2925  +typedef struct PoslistCallbackCtx PoslistCallbackCtx;
         2926  +struct PoslistCallbackCtx {
         2927  +  Fts5Buffer *pBuf;               /* Append to this buffer */
         2928  +  Fts5Colset *pColset;            /* Restrict matches to this column */
         2929  +  int eState;                     /* See above */
         2930  +};
         2931  +
         2932  +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx;
         2933  +struct PoslistOffsetsCtx {
         2934  +  Fts5Buffer *pBuf;               /* Append to this buffer */
         2935  +  Fts5Colset *pColset;            /* Restrict matches to this column */
         2936  +  int iRead;
         2937  +  int iWrite;
         2938  +};
         2939  +
         2940  +/*
         2941  +** TODO: Make this more efficient!
         2942  +*/
         2943  +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
         2944  +  int i;
         2945  +  for(i=0; i<pColset->nCol; i++){
         2946  +    if( pColset->aiCol[i]==iCol ) return 1;
         2947  +  }
         2948  +  return 0;
         2949  +}
         2950  +
         2951  +static void fts5PoslistOffsetsCallback(
         2952  +  Fts5Index *pUnused, 
         2953  +  void *pContext, 
         2954  +  const u8 *pChunk, int nChunk
         2955  +){
         2956  +  PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext;
         2957  +  UNUSED_PARAM(pUnused);
         2958  +  assert_nc( nChunk>=0 );
         2959  +  if( nChunk>0 ){
         2960  +    int i = 0;
         2961  +    while( i<nChunk ){
         2962  +      int iVal;
         2963  +      i += fts5GetVarint32(&pChunk[i], iVal);
         2964  +      iVal += pCtx->iRead - 2;
         2965  +      pCtx->iRead = iVal;
         2966  +      if( fts5IndexColsetTest(pCtx->pColset, iVal) ){
         2967  +        fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite);
         2968  +        pCtx->iWrite = iVal;
         2969  +      }
         2970  +    }
         2971  +  }
         2972  +}
         2973  +
         2974  +static void fts5PoslistFilterCallback(
         2975  +  Fts5Index *pUnused,
         2976  +  void *pContext, 
         2977  +  const u8 *pChunk, int nChunk
         2978  +){
         2979  +  PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
         2980  +  UNUSED_PARAM(pUnused);
         2981  +  assert_nc( nChunk>=0 );
         2982  +  if( nChunk>0 ){
         2983  +    /* Search through to find the first varint with value 1. This is the
         2984  +    ** start of the next columns hits. */
         2985  +    int i = 0;
         2986  +    int iStart = 0;
         2987  +
         2988  +    if( pCtx->eState==2 ){
         2989  +      int iCol;
         2990  +      fts5FastGetVarint32(pChunk, i, iCol);
         2991  +      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
         2992  +        pCtx->eState = 1;
         2993  +        fts5BufferSafeAppendVarint(pCtx->pBuf, 1);
         2994  +      }else{
         2995  +        pCtx->eState = 0;
         2996  +      }
         2997  +    }
         2998  +
         2999  +    do {
         3000  +      while( i<nChunk && pChunk[i]!=0x01 ){
         3001  +        while( pChunk[i] & 0x80 ) i++;
         3002  +        i++;
         3003  +      }
         3004  +      if( pCtx->eState ){
         3005  +        fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
         3006  +      }
         3007  +      if( i<nChunk ){
         3008  +        int iCol;
         3009  +        iStart = i;
         3010  +        i++;
         3011  +        if( i>=nChunk ){
         3012  +          pCtx->eState = 2;
  2880   3013           }else{
  2881         -          fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter);
  2882         -        }
  2883         -      }
  2884         -    }
  2885         -  }else{
  2886         -    pLvl = &pStruct->aLevel[iLevel];
  2887         -    for(iSeg=nSeg-1; iSeg>=0; iSeg--){
  2888         -      fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
  2889         -    }
  2890         -  }
  2891         -  assert( iIter==nSeg );
  2892         -
  2893         -  /* If the above was successful, each component iterators now points 
  2894         -  ** to the first entry in its segment. In this case initialize the 
  2895         -  ** aFirst[] array. Or, if an error has occurred, free the iterator
  2896         -  ** object and set the output variable to NULL.  */
  2897         -  if( p->rc==SQLITE_OK ){
  2898         -    for(iIter=pNew->nSeg-1; iIter>0; iIter--){
  2899         -      int iEq;
  2900         -      if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){
  2901         -        Fts5SegIter *pSeg = &pNew->aSeg[iEq];
  2902         -        if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0);
  2903         -        fts5MultiIterAdvanced(p, pNew, iEq, iIter);
  2904         -      }
  2905         -    }
  2906         -    fts5MultiIterSetEof(pNew);
  2907         -    fts5AssertMultiIterSetup(p, pNew);
  2908         -
  2909         -    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
  2910         -      fts5MultiIterNext(p, pNew, 0, 0);
  2911         -    }
  2912         -  }else{
  2913         -    fts5MultiIterFree(p, pNew);
  2914         -    *ppOut = 0;
  2915         -  }
  2916         -  fts5BufferFree(&buf);
  2917         -}
  2918         -
  2919         -/*
  2920         -** Create an Fts5IndexIter that iterates through the doclist provided
  2921         -** as the second argument.
  2922         -*/
  2923         -static void fts5MultiIterNew2(
  2924         -  Fts5Index *p,                   /* FTS5 backend to iterate within */
  2925         -  Fts5Data *pData,                /* Doclist to iterate through */
  2926         -  int bDesc,                      /* True for descending rowid order */
  2927         -  Fts5IndexIter **ppOut           /* New object */
  2928         -){
  2929         -  Fts5IndexIter *pNew;
  2930         -  pNew = fts5MultiIterAlloc(p, 2);
  2931         -  if( pNew ){
  2932         -    Fts5SegIter *pIter = &pNew->aSeg[1];
  2933         -
  2934         -    pNew->bFiltered = 1;
  2935         -    pIter->flags = FTS5_SEGITER_ONETERM;
  2936         -    if( pData->szLeaf>0 ){
  2937         -      pIter->pLeaf = pData;
  2938         -      pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
  2939         -      pIter->iEndofDoclist = pData->nn;
  2940         -      pNew->aFirst[1].iFirst = 1;
  2941         -      if( bDesc ){
  2942         -        pNew->bRev = 1;
  2943         -        pIter->flags |= FTS5_SEGITER_REVERSE;
  2944         -        fts5SegIterReverseInitPage(p, pIter);
  2945         -      }else{
  2946         -        fts5SegIterLoadNPos(p, pIter);
  2947         -      }
  2948         -      pData = 0;
  2949         -    }else{
  2950         -      pNew->bEof = 1;
  2951         -    }
  2952         -    fts5SegIterSetNext(p, pIter);
  2953         -
  2954         -    *ppOut = pNew;
  2955         -  }
  2956         -
  2957         -  fts5DataRelease(pData);
  2958         -}
  2959         -
  2960         -/*
  2961         -** Return true if the iterator is at EOF or if an error has occurred. 
  2962         -** False otherwise.
  2963         -*/
  2964         -static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){
  2965         -  assert( p->rc 
  2966         -      || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof 
  2967         -  );
  2968         -  return (p->rc || pIter->bEof);
  2969         -}
  2970         -
  2971         -/*
  2972         -** Return the rowid of the entry that the iterator currently points
  2973         -** to. If the iterator points to EOF when this function is called the
  2974         -** results are undefined.
  2975         -*/
  2976         -static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){
  2977         -  assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
  2978         -  return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
  2979         -}
  2980         -
  2981         -/*
  2982         -** Move the iterator to the next entry at or following iMatch.
  2983         -*/
  2984         -static void fts5MultiIterNextFrom(
  2985         -  Fts5Index *p, 
  2986         -  Fts5IndexIter *pIter, 
  2987         -  i64 iMatch
  2988         -){
  2989         -  while( 1 ){
  2990         -    i64 iRowid;
  2991         -    fts5MultiIterNext(p, pIter, 1, iMatch);
  2992         -    if( fts5MultiIterEof(p, pIter) ) break;
  2993         -    iRowid = fts5MultiIterRowid(pIter);
  2994         -    if( pIter->bRev==0 && iRowid>=iMatch ) break;
  2995         -    if( pIter->bRev!=0 && iRowid<=iMatch ) break;
  2996         -  }
  2997         -}
  2998         -
  2999         -/*
  3000         -** Return a pointer to a buffer containing the term associated with the 
  3001         -** entry that the iterator currently points to.
  3002         -*/
  3003         -static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){
  3004         -  Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  3005         -  *pn = p->term.n;
  3006         -  return p->term.p;
         3014  +          fts5FastGetVarint32(pChunk, i, iCol);
         3015  +          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
         3016  +          if( pCtx->eState ){
         3017  +            fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
         3018  +            iStart = i;
         3019  +          }
         3020  +        }
         3021  +      }
         3022  +    }while( i<nChunk );
         3023  +  }
  3007   3024   }
  3008   3025   
  3009   3026   static void fts5ChunkIterate(
  3010   3027     Fts5Index *p,                   /* Index object */
  3011   3028     Fts5SegIter *pSeg,              /* Poslist of this iterator */
  3012   3029     void *pCtx,                     /* Context pointer for xChunk callback */
  3013   3030     void (*xChunk)(Fts5Index*, void*, const u8*, int)
................................................................................
  3043   3060           pSeg->pNextLeaf = pData;
  3044   3061           pData = 0;
  3045   3062         }
  3046   3063       }
  3047   3064     }
  3048   3065   }
  3049   3066   
  3050         -
         3067  +/*
         3068  +** Iterator pIter currently points to a valid entry (not EOF). This
         3069  +** function appends the position list data for the current entry to
         3070  +** buffer pBuf. It does not make a copy of the position-list size
         3071  +** field.
         3072  +*/
         3073  +static void fts5SegiterPoslist(
         3074  +  Fts5Index *p,
         3075  +  Fts5SegIter *pSeg,
         3076  +  Fts5Colset *pColset,
         3077  +  Fts5Buffer *pBuf
         3078  +){
         3079  +  if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
         3080  +    if( pColset==0 ){
         3081  +      fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
         3082  +    }else{
         3083  +      if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){
         3084  +        PoslistCallbackCtx sCtx;
         3085  +        sCtx.pBuf = pBuf;
         3086  +        sCtx.pColset = pColset;
         3087  +        sCtx.eState = fts5IndexColsetTest(pColset, 0);
         3088  +        assert( sCtx.eState==0 || sCtx.eState==1 );
         3089  +        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
         3090  +      }else{
         3091  +        PoslistOffsetsCtx sCtx;
         3092  +        memset(&sCtx, 0, sizeof(sCtx));
         3093  +        sCtx.pBuf = pBuf;
         3094  +        sCtx.pColset = pColset;
         3095  +        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
         3096  +      }
         3097  +    }
         3098  +  }
         3099  +}
         3100  +
         3101  +/*
         3102  +** IN/OUT parameter (*pa) points to a position list n bytes in size. If
         3103  +** the position list contains entries for column iCol, then (*pa) is set
         3104  +** to point to the sub-position-list for that column and the number of
         3105  +** bytes in it returned. Or, if the argument position list does not
         3106  +** contain any entries for column iCol, return 0.
         3107  +*/
         3108  +static int fts5IndexExtractCol(
         3109  +  const u8 **pa,                  /* IN/OUT: Pointer to poslist */
         3110  +  int n,                          /* IN: Size of poslist in bytes */
         3111  +  int iCol                        /* Column to extract from poslist */
         3112  +){
         3113  +  int iCurrent = 0;               /* Anything before the first 0x01 is col 0 */
         3114  +  const u8 *p = *pa;
         3115  +  const u8 *pEnd = &p[n];         /* One byte past end of position list */
         3116  +
         3117  +  while( iCol>iCurrent ){
         3118  +    /* Advance pointer p until it points to pEnd or an 0x01 byte that is
         3119  +    ** not part of a varint. Note that it is not possible for a negative
         3120  +    ** or extremely large varint to occur within an uncorrupted position 
         3121  +    ** list. So the last byte of each varint may be assumed to have a clear
         3122  +    ** 0x80 bit.  */
         3123  +    while( *p!=0x01 ){
         3124  +      while( *p++ & 0x80 );
         3125  +      if( p>=pEnd ) return 0;
         3126  +    }
         3127  +    *pa = p++;
         3128  +    iCurrent = *p++;
         3129  +    if( iCurrent & 0x80 ){
         3130  +      p--;
         3131  +      p += fts5GetVarint32(p, iCurrent);
         3132  +    }
         3133  +  }
         3134  +  if( iCol!=iCurrent ) return 0;
         3135  +
         3136  +  /* Advance pointer p until it points to pEnd or an 0x01 byte that is
         3137  +  ** not part of a varint */
         3138  +  while( p<pEnd && *p!=0x01 ){
         3139  +    while( *p++ & 0x80 );
         3140  +  }
         3141  +
         3142  +  return p - (*pa);
         3143  +}
         3144  +
         3145  +static int fts5IndexExtractColset (
         3146  +  Fts5Colset *pColset,            /* Colset to filter on */
         3147  +  const u8 *pPos, int nPos,       /* Position list */
         3148  +  Fts5Buffer *pBuf                /* Output buffer */
         3149  +){
         3150  +  int rc = SQLITE_OK;
         3151  +  int i;
         3152  +
         3153  +  fts5BufferZero(pBuf);
         3154  +  for(i=0; i<pColset->nCol; i++){
         3155  +    const u8 *pSub = pPos;
         3156  +    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
         3157  +    if( nSub ){
         3158  +      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
         3159  +    }
         3160  +  }
         3161  +  return rc;
         3162  +}
         3163  +
         3164  +/*
         3165  +** xSetOutputs callback used by detail=none tables.
         3166  +*/
         3167  +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3168  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
         3169  +  pIter->base.iRowid = pSeg->iRowid;
         3170  +  pIter->base.nData = pSeg->nPos;
         3171  +}
         3172  +
         3173  +/*
         3174  +** xSetOutputs callback used by detail=full and detail=col tables when no
         3175  +** column filters are specified.
         3176  +*/
         3177  +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3178  +  pIter->base.iRowid = pSeg->iRowid;
         3179  +  pIter->base.nData = pSeg->nPos;
         3180  +
         3181  +  assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE );
         3182  +  assert( pIter->pColset==0 );
         3183  +
         3184  +  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
         3185  +    /* All data is stored on the current page. Populate the output 
         3186  +    ** variables to point into the body of the page object. */
         3187  +    pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
         3188  +  }else{
         3189  +    /* The data is distributed over two or more pages. Copy it into the
         3190  +    ** Fts5Iter.poslist buffer and then set the output pointer to point
         3191  +    ** to this buffer.  */
         3192  +    fts5BufferZero(&pIter->poslist);
         3193  +    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
         3194  +    pIter->base.pData = pIter->poslist.p;
         3195  +  }
         3196  +}
         3197  +
         3198  +/*
         3199  +** xSetOutputs callback used by detail=col when there is a column filter
         3200  +** and there are 100 or more columns. Also called as a fallback from
         3201  +** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
         3202  +*/
         3203  +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3204  +  fts5BufferZero(&pIter->poslist);
         3205  +  fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);
         3206  +  pIter->base.iRowid = pSeg->iRowid;
         3207  +  pIter->base.pData = pIter->poslist.p;
         3208  +  pIter->base.nData = pIter->poslist.n;
         3209  +}
         3210  +
         3211  +/*
         3212  +** xSetOutputs callback used when: 
         3213  +**
         3214  +**   * detail=col,
         3215  +**   * there is a column filter, and
         3216  +**   * the table contains 100 or fewer columns. 
         3217  +**
         3218  +** The last point is to ensure all column numbers are stored as 
         3219  +** single-byte varints.
         3220  +*/
         3221  +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3222  +
         3223  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         3224  +  assert( pIter->pColset );
         3225  +
         3226  +  if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){
         3227  +    fts5IterSetOutputs_Col(pIter, pSeg);
         3228  +  }else{
         3229  +    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
         3230  +    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
         3231  +    int iPrev = 0;
         3232  +    int *aiCol = pIter->pColset->aiCol;
         3233  +    int *aiColEnd = &aiCol[pIter->pColset->nCol];
         3234  +
         3235  +    u8 *aOut = pIter->poslist.p;
         3236  +    int iPrevOut = 0;
         3237  +
         3238  +    pIter->base.iRowid = pSeg->iRowid;
         3239  +
         3240  +    while( a<pEnd ){
         3241  +      iPrev += (int)a++[0] - 2;
         3242  +      while( *aiCol<iPrev ){
         3243  +        aiCol++;
         3244  +        if( aiCol==aiColEnd ) goto setoutputs_col_out;
         3245  +      }
         3246  +      if( *aiCol==iPrev ){
         3247  +        *aOut++ = (iPrev - iPrevOut) + 2;
         3248  +        iPrevOut = iPrev;
         3249  +      }
         3250  +    }
         3251  +
         3252  +setoutputs_col_out:
         3253  +    pIter->base.pData = pIter->poslist.p;
         3254  +    pIter->base.nData = aOut - pIter->poslist.p;
         3255  +  }
         3256  +}
         3257  +
         3258  +/*
         3259  +** xSetOutputs callback used by detail=full when there is a column filter.
         3260  +*/
         3261  +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3262  +  Fts5Colset *pColset = pIter->pColset;
         3263  +  pIter->base.iRowid = pSeg->iRowid;
         3264  +
         3265  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL );
         3266  +  assert( pColset );
         3267  +
         3268  +  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
         3269  +    /* All data is stored on the current page. Populate the output 
         3270  +    ** variables to point into the body of the page object. */
         3271  +    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
         3272  +    if( pColset->nCol==1 ){
         3273  +      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
         3274  +      pIter->base.pData = a;
         3275  +    }else{
         3276  +      fts5BufferZero(&pIter->poslist);
         3277  +      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
         3278  +      pIter->base.pData = pIter->poslist.p;
         3279  +      pIter->base.nData = pIter->poslist.n;
         3280  +    }
         3281  +  }else{
         3282  +    /* The data is distributed over two or more pages. Copy it into the
         3283  +    ** Fts5Iter.poslist buffer and then set the output pointer to point
         3284  +    ** to this buffer.  */
         3285  +    fts5BufferZero(&pIter->poslist);
         3286  +    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
         3287  +    pIter->base.pData = pIter->poslist.p;
         3288  +    pIter->base.nData = pIter->poslist.n;
         3289  +  }
         3290  +}
         3291  +
         3292  +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
         3293  +  if( *pRc==SQLITE_OK ){
         3294  +    Fts5Config *pConfig = pIter->pIndex->pConfig;
         3295  +    if( pConfig->eDetail==FTS5_DETAIL_NONE ){
         3296  +      pIter->xSetOutputs = fts5IterSetOutputs_None;
         3297  +    }
         3298  +
         3299  +    else if( pIter->pColset==0 ){
         3300  +      pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
         3301  +    }
         3302  +
         3303  +    else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
         3304  +      pIter->xSetOutputs = fts5IterSetOutputs_Full;
         3305  +    }
         3306  +
         3307  +    else{
         3308  +      assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         3309  +      if( pConfig->nCol<=100 ){
         3310  +        pIter->xSetOutputs = fts5IterSetOutputs_Col100;
         3311  +        sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
         3312  +      }else{
         3313  +        pIter->xSetOutputs = fts5IterSetOutputs_Col;
         3314  +      }
         3315  +    }
         3316  +  }
         3317  +}
         3318  +
         3319  +
         3320  +/*
         3321  +** Allocate a new Fts5Iter object.
         3322  +**
         3323  +** The new object will be used to iterate through data in structure pStruct.
         3324  +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
         3325  +** is zero or greater, data from the first nSegment segments on level iLevel
         3326  +** is merged.
         3327  +**
         3328  +** The iterator initially points to the first term/rowid entry in the 
         3329  +** iterated data.
         3330  +*/
         3331  +static void fts5MultiIterNew(
         3332  +  Fts5Index *p,                   /* FTS5 backend to iterate within */
         3333  +  Fts5Structure *pStruct,         /* Structure of specific index */
         3334  +  int flags,                      /* FTS5INDEX_QUERY_XXX flags */
         3335  +  Fts5Colset *pColset,            /* Colset to filter on (or NULL) */
         3336  +  const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
         3337  +  int iLevel,                     /* Level to iterate (-1 for all) */
         3338  +  int nSegment,                   /* Number of segments to merge (iLevel>=0) */
         3339  +  Fts5Iter **ppOut                /* New object */
         3340  +){
         3341  +  int nSeg = 0;                   /* Number of segment-iters in use */
         3342  +  int iIter = 0;                  /* */
         3343  +  int iSeg;                       /* Used to iterate through segments */
         3344  +  Fts5StructureLevel *pLvl;
         3345  +  Fts5Iter *pNew;
         3346  +
         3347  +  assert( (pTerm==0 && nTerm==0) || iLevel<0 );
         3348  +
         3349  +  /* Allocate space for the new multi-seg-iterator. */
         3350  +  if( p->rc==SQLITE_OK ){
         3351  +    if( iLevel<0 ){
         3352  +      assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
         3353  +      nSeg = pStruct->nSegment;
         3354  +      nSeg += (p->pHash ? 1 : 0);
         3355  +    }else{
         3356  +      nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
         3357  +    }
         3358  +  }
         3359  +  *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
         3360  +  if( pNew==0 ) return;
         3361  +  pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
         3362  +  pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY));
         3363  +  pNew->pStruct = pStruct;
         3364  +  pNew->pColset = pColset;
         3365  +  fts5StructureRef(pStruct);
         3366  +  if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){
         3367  +    fts5IterSetOutputCb(&p->rc, pNew);
         3368  +  }
         3369  +
         3370  +  /* Initialize each of the component segment iterators. */
         3371  +  if( p->rc==SQLITE_OK ){
         3372  +    if( iLevel<0 ){
         3373  +      Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
         3374  +      if( p->pHash ){
         3375  +        /* Add a segment iterator for the current contents of the hash table. */
         3376  +        Fts5SegIter *pIter = &pNew->aSeg[iIter++];
         3377  +        fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
         3378  +      }
         3379  +      for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
         3380  +        for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
         3381  +          Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
         3382  +          Fts5SegIter *pIter = &pNew->aSeg[iIter++];
         3383  +          if( pTerm==0 ){
         3384  +            fts5SegIterInit(p, pSeg, pIter);
         3385  +          }else{
         3386  +            fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
         3387  +          }
         3388  +        }
         3389  +      }
         3390  +    }else{
         3391  +      pLvl = &pStruct->aLevel[iLevel];
         3392  +      for(iSeg=nSeg-1; iSeg>=0; iSeg--){
         3393  +        fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
         3394  +      }
         3395  +    }
         3396  +    assert( iIter==nSeg );
         3397  +  }
         3398  +
         3399  +  /* If the above was successful, each component iterators now points 
         3400  +  ** to the first entry in its segment. In this case initialize the 
         3401  +  ** aFirst[] array. Or, if an error has occurred, free the iterator
         3402  +  ** object and set the output variable to NULL.  */
         3403  +  if( p->rc==SQLITE_OK ){
         3404  +    for(iIter=pNew->nSeg-1; iIter>0; iIter--){
         3405  +      int iEq;
         3406  +      if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){
         3407  +        Fts5SegIter *pSeg = &pNew->aSeg[iEq];
         3408  +        if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0);
         3409  +        fts5MultiIterAdvanced(p, pNew, iEq, iIter);
         3410  +      }
         3411  +    }
         3412  +    fts5MultiIterSetEof(pNew);
         3413  +    fts5AssertMultiIterSetup(p, pNew);
         3414  +
         3415  +    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
         3416  +      fts5MultiIterNext(p, pNew, 0, 0);
         3417  +    }else if( pNew->base.bEof==0 ){
         3418  +      Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst];
         3419  +      pNew->xSetOutputs(pNew, pSeg);
         3420  +    }
         3421  +
         3422  +  }else{
         3423  +    fts5MultiIterFree(pNew);
         3424  +    *ppOut = 0;
         3425  +  }
         3426  +}
         3427  +
         3428  +/*
         3429  +** Create an Fts5Iter that iterates through the doclist provided
         3430  +** as the second argument.
         3431  +*/
         3432  +static void fts5MultiIterNew2(
         3433  +  Fts5Index *p,                   /* FTS5 backend to iterate within */
         3434  +  Fts5Data *pData,                /* Doclist to iterate through */
         3435  +  int bDesc,                      /* True for descending rowid order */
         3436  +  Fts5Iter **ppOut                /* New object */
         3437  +){
         3438  +  Fts5Iter *pNew;
         3439  +  pNew = fts5MultiIterAlloc(p, 2);
         3440  +  if( pNew ){
         3441  +    Fts5SegIter *pIter = &pNew->aSeg[1];
         3442  +
         3443  +    pIter->flags = FTS5_SEGITER_ONETERM;
         3444  +    if( pData->szLeaf>0 ){
         3445  +      pIter->pLeaf = pData;
         3446  +      pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
         3447  +      pIter->iEndofDoclist = pData->nn;
         3448  +      pNew->aFirst[1].iFirst = 1;
         3449  +      if( bDesc ){
         3450  +        pNew->bRev = 1;
         3451  +        pIter->flags |= FTS5_SEGITER_REVERSE;
         3452  +        fts5SegIterReverseInitPage(p, pIter);
         3453  +      }else{
         3454  +        fts5SegIterLoadNPos(p, pIter);
         3455  +      }
         3456  +      pData = 0;
         3457  +    }else{
         3458  +      pNew->base.bEof = 1;
         3459  +    }
         3460  +    fts5SegIterSetNext(p, pIter);
         3461  +
         3462  +    *ppOut = pNew;
         3463  +  }
         3464  +
         3465  +  fts5DataRelease(pData);
         3466  +}
         3467  +
         3468  +/*
         3469  +** Return true if the iterator is at EOF or if an error has occurred. 
         3470  +** False otherwise.
         3471  +*/
         3472  +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){
         3473  +  assert( p->rc 
         3474  +      || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof 
         3475  +  );
         3476  +  return (p->rc || pIter->base.bEof);
         3477  +}
         3478  +
         3479  +/*
         3480  +** Return the rowid of the entry that the iterator currently points
         3481  +** to. If the iterator points to EOF when this function is called the
         3482  +** results are undefined.
         3483  +*/
         3484  +static i64 fts5MultiIterRowid(Fts5Iter *pIter){
         3485  +  assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
         3486  +  return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
         3487  +}
         3488  +
         3489  +/*
         3490  +** Move the iterator to the next entry at or following iMatch.
         3491  +*/
         3492  +static void fts5MultiIterNextFrom(
         3493  +  Fts5Index *p, 
         3494  +  Fts5Iter *pIter, 
         3495  +  i64 iMatch
         3496  +){
         3497  +  while( 1 ){
         3498  +    i64 iRowid;
         3499  +    fts5MultiIterNext(p, pIter, 1, iMatch);
         3500  +    if( fts5MultiIterEof(p, pIter) ) break;
         3501  +    iRowid = fts5MultiIterRowid(pIter);
         3502  +    if( pIter->bRev==0 && iRowid>=iMatch ) break;
         3503  +    if( pIter->bRev!=0 && iRowid<=iMatch ) break;
         3504  +  }
         3505  +}
         3506  +
         3507  +/*
         3508  +** Return a pointer to a buffer containing the term associated with the 
         3509  +** entry that the iterator currently points to.
         3510  +*/
         3511  +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){
         3512  +  Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
         3513  +  *pn = p->term.n;
         3514  +  return p->term.p;
         3515  +}
  3051   3516   
  3052   3517   /*
  3053   3518   ** Allocate a new segment-id for the structure pStruct. The new segment
  3054   3519   ** id must be between 1 and 65335 inclusive, and must not be used by 
  3055   3520   ** any currently existing segment. If a free segment id cannot be found,
  3056   3521   ** SQLITE_FULL is returned.
  3057   3522   **
................................................................................
  3061   3526   static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
  3062   3527     int iSegid = 0;
  3063   3528   
  3064   3529     if( p->rc==SQLITE_OK ){
  3065   3530       if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
  3066   3531         p->rc = SQLITE_FULL;
  3067   3532       }else{
  3068         -      while( iSegid==0 ){
  3069         -        int iLvl, iSeg;
  3070         -        sqlite3_randomness(sizeof(u32), (void*)&iSegid);
  3071         -        iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
  3072         -        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
  3073         -          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  3074         -            if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
  3075         -              iSegid = 0;
  3076         -            }
         3533  +      /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following
         3534  +      ** array is 63 elements, or 252 bytes, in size.  */
         3535  +      u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32];
         3536  +      int iLvl, iSeg;
         3537  +      int i;
         3538  +      u32 mask;
         3539  +      memset(aUsed, 0, sizeof(aUsed));
         3540  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3541  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3542  +          int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
         3543  +          if( iId<=FTS5_MAX_SEGMENT ){
         3544  +            aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
  3077   3545             }
  3078   3546           }
  3079   3547         }
         3548  +
         3549  +      for(i=0; aUsed[i]==0xFFFFFFFF; i++);
         3550  +      mask = aUsed[i];
         3551  +      for(iSegid=0; mask & (1 << iSegid); iSegid++);
         3552  +      iSegid += 1 + i*32;
         3553  +
         3554  +#ifdef SQLITE_DEBUG
         3555  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3556  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3557  +          assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid );
         3558  +        }
         3559  +      }
         3560  +      assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT );
         3561  +
         3562  +      {
         3563  +        sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
         3564  +        if( p->rc==SQLITE_OK ){
         3565  +          u8 aBlob[2] = {0xff, 0xff};
         3566  +          sqlite3_bind_int(pIdxSelect, 1, iSegid);
         3567  +          sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
         3568  +          assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
         3569  +          p->rc = sqlite3_reset(pIdxSelect);
         3570  +        }
         3571  +      }
         3572  +#endif
  3080   3573       }
  3081   3574     }
  3082   3575   
  3083   3576     return iSegid;
  3084   3577   }
  3085   3578   
  3086   3579   /*
................................................................................
  3091   3584     if( p->pHash ){
  3092   3585       sqlite3Fts5HashClear(p->pHash);
  3093   3586       p->nPendingData = 0;
  3094   3587     }
  3095   3588   }
  3096   3589   
  3097   3590   /*
  3098         -** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
  3099         -** with buffer (nOld/pOld).
         3591  +** Return the size of the prefix, in bytes, that buffer 
         3592  +** (pNew/<length-unknown>) shares with buffer (pOld/nOld).
         3593  +**
         3594  +** Buffer (pNew/<length-unknown>) is guaranteed to be greater 
         3595  +** than buffer (pOld/nOld).
  3100   3596   */
  3101         -static int fts5PrefixCompress(
  3102         -  int nOld, const u8 *pOld,
  3103         -  int nNew, const u8 *pNew
  3104         -){
         3597  +static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){
  3105   3598     int i;
  3106         -  assert( fts5BlobCompare(pOld, nOld, pNew, nNew)<0 );
  3107   3599     for(i=0; i<nOld; i++){
  3108   3600       if( pOld[i]!=pNew[i] ) break;
  3109   3601     }
  3110   3602     return i;
  3111   3603   }
  3112   3604   
  3113   3605   static void fts5WriteDlidxClear(
................................................................................
  3318   3810     }
  3319   3811   }
  3320   3812   
  3321   3813   static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
  3322   3814     static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
  3323   3815     Fts5PageWriter *pPage = &pWriter->writer;
  3324   3816     i64 iRowid;
         3817  +
         3818  +static int nCall = 0;
         3819  +nCall++;
  3325   3820   
  3326   3821     assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
  3327   3822   
  3328   3823     /* Set the szLeaf header field. */
  3329   3824     assert( 0==fts5GetU16(&pPage->buf.p[2]) );
  3330   3825     fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
  3331   3826   
................................................................................
  3409   3904         ** Usually, the previous term is available in pPage->term. The exception
  3410   3905         ** is if this is the first term written in an incremental-merge step.
  3411   3906         ** In this case the previous term is not available, so just write a
  3412   3907         ** copy of (pTerm/nTerm) into the parent node. This is slightly
  3413   3908         ** inefficient, but still correct.  */
  3414   3909         int n = nTerm;
  3415   3910         if( pPage->term.n ){
  3416         -        n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
         3911  +        n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
  3417   3912         }
  3418   3913         fts5WriteBtreeTerm(p, pWriter, n, pTerm);
  3419   3914         pPage = &pWriter->writer;
  3420   3915       }
  3421   3916     }else{
  3422         -    nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
         3917  +    nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
  3423   3918       fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix);
  3424   3919     }
  3425   3920   
  3426   3921     /* Append the number of bytes of new data, then the term data itself
  3427   3922     ** to the page. */
  3428   3923     fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix);
  3429   3924     fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]);
................................................................................
  3518   4013     Fts5PageWriter *pLeaf = &pWriter->writer;
  3519   4014     if( p->rc==SQLITE_OK ){
  3520   4015       assert( pLeaf->pgno>=1 );
  3521   4016       if( pLeaf->buf.n>4 ){
  3522   4017         fts5WriteFlushLeaf(p, pWriter);
  3523   4018       }
  3524   4019       *pnLeaf = pLeaf->pgno-1;
  3525         -    fts5WriteFlushBtree(p, pWriter);
         4020  +    if( pLeaf->pgno>1 ){
         4021  +      fts5WriteFlushBtree(p, pWriter);
         4022  +    }
  3526   4023     }
  3527   4024     fts5BufferFree(&pLeaf->term);
  3528   4025     fts5BufferFree(&pLeaf->buf);
  3529   4026     fts5BufferFree(&pLeaf->pgidx);
  3530   4027     fts5BufferFree(&pWriter->btterm);
  3531   4028   
  3532   4029     for(i=0; i<pWriter->nDlidx; i++){
................................................................................
  3578   4075   }
  3579   4076   
  3580   4077   /*
  3581   4078   ** Iterator pIter was used to iterate through the input segments of on an
  3582   4079   ** incremental merge operation. This function is called if the incremental
  3583   4080   ** merge step has finished but the input has not been completely exhausted.
  3584   4081   */
  3585         -static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
         4082  +static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
  3586   4083     int i;
  3587   4084     Fts5Buffer buf;
  3588   4085     memset(&buf, 0, sizeof(Fts5Buffer));
  3589   4086     for(i=0; i<pIter->nSeg; i++){
  3590   4087       Fts5SegIter *pSeg = &pIter->aSeg[i];
  3591   4088       if( pSeg->pSeg==0 ){
  3592   4089         /* no-op */
................................................................................
  3656   4153     Fts5Structure **ppStruct,       /* IN/OUT: Stucture of index */
  3657   4154     int iLvl,                       /* Level to read input from */
  3658   4155     int *pnRem                      /* Write up to this many output leaves */
  3659   4156   ){
  3660   4157     Fts5Structure *pStruct = *ppStruct;
  3661   4158     Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
  3662   4159     Fts5StructureLevel *pLvlOut;
  3663         -  Fts5IndexIter *pIter = 0;       /* Iterator to read input data */
         4160  +  Fts5Iter *pIter = 0;       /* Iterator to read input data */
  3664   4161     int nRem = pnRem ? *pnRem : 0;  /* Output leaf pages left to write */
  3665   4162     int nInput;                     /* Number of input segments */
  3666   4163     Fts5SegWriter writer;           /* Writer object */
  3667   4164     Fts5StructureSegment *pSeg;     /* Output segment */
  3668   4165     Fts5Buffer term;
  3669   4166     int bOldest;                    /* True if the output segment is the oldest */
  3670   4167     int eDetail = p->pConfig->eDetail;
         4168  +  const int flags = FTS5INDEX_QUERY_NOOUTPUT;
  3671   4169   
  3672   4170     assert( iLvl<pStruct->nLevel );
  3673   4171     assert( pLvl->nMerge<=pLvl->nSeg );
  3674   4172   
  3675   4173     memset(&writer, 0, sizeof(Fts5SegWriter));
  3676   4174     memset(&term, 0, sizeof(Fts5Buffer));
  3677   4175     if( pLvl->nMerge ){
................................................................................
  3708   4206   
  3709   4207       /* Read input from all segments in the input level */
  3710   4208       nInput = pLvl->nSeg;
  3711   4209     }
  3712   4210     bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
  3713   4211   
  3714   4212     assert( iLvl>=0 );
  3715         -  for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter);
         4213  +  for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter);
  3716   4214         fts5MultiIterEof(p, pIter)==0;
  3717   4215         fts5MultiIterNext(p, pIter, 0, 0)
  3718   4216     ){
  3719   4217       Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  3720   4218       int nPos;                     /* position-list size field value */
  3721   4219       int nTerm;
  3722   4220       const u8 *pTerm;
................................................................................
  3780   4278       }
  3781   4279     }else{
  3782   4280       assert( pSeg->pgnoLast>0 );
  3783   4281       fts5TrimSegments(p, pIter);
  3784   4282       pLvl->nMerge = nInput;
  3785   4283     }
  3786   4284   
  3787         -  fts5MultiIterFree(p, pIter);
         4285  +  fts5MultiIterFree(pIter);
  3788   4286     fts5BufferFree(&term);
  3789   4287     if( pnRem ) *pnRem -= writer.nLeafWritten;
  3790   4288   }
  3791   4289   
  3792   4290   /*
  3793   4291   ** Do up to nPg pages of automerge work on the index.
         4292  +**
         4293  +** Return true if any changes were actually made, or false otherwise.
  3794   4294   */
  3795         -static void fts5IndexMerge(
         4295  +static int fts5IndexMerge(
  3796   4296     Fts5Index *p,                   /* FTS5 backend object */
  3797   4297     Fts5Structure **ppStruct,       /* IN/OUT: Current structure of index */
  3798         -  int nPg                         /* Pages of work to do */
         4298  +  int nPg,                        /* Pages of work to do */
         4299  +  int nMin                        /* Minimum number of segments to merge */
  3799   4300   ){
  3800   4301     int nRem = nPg;
         4302  +  int bRet = 0;
  3801   4303     Fts5Structure *pStruct = *ppStruct;
  3802   4304     while( nRem>0 && p->rc==SQLITE_OK ){
  3803   4305       int iLvl;                   /* To iterate through levels */
  3804   4306       int iBestLvl = 0;           /* Level offering the most input segments */
  3805   4307       int nBest = 0;              /* Number of input segments on best level */
  3806   4308   
  3807   4309       /* Set iBestLvl to the level to read input segments from. */
................................................................................
  3824   4326       /* If nBest is still 0, then the index must be empty. */
  3825   4327   #ifdef SQLITE_DEBUG
  3826   4328       for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
  3827   4329         assert( pStruct->aLevel[iLvl].nSeg==0 );
  3828   4330       }
  3829   4331   #endif
  3830   4332   
  3831         -    if( nBest<p->pConfig->nAutomerge 
  3832         -        && pStruct->aLevel[iBestLvl].nMerge==0 
  3833         -      ){
         4333  +    if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
  3834   4334         break;
  3835   4335       }
         4336  +    bRet = 1;
  3836   4337       fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
  3837   4338       if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
  3838   4339         fts5StructurePromote(p, iBestLvl+1, pStruct);
  3839   4340       }
  3840   4341     }
  3841   4342     *ppStruct = pStruct;
         4343  +  return bRet;
  3842   4344   }
  3843   4345   
  3844   4346   /*
  3845   4347   ** A total of nLeaf leaf pages of data has just been flushed to a level-0
  3846   4348   ** segment. This function updates the write-counter accordingly and, if
  3847   4349   ** necessary, performs incremental merge work.
  3848   4350   **
................................................................................
  3862   4364   
  3863   4365       /* Update the write-counter. While doing so, set nWork. */
  3864   4366       nWrite = pStruct->nWriteCounter;
  3865   4367       nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit));
  3866   4368       pStruct->nWriteCounter += nLeaf;
  3867   4369       nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);
  3868   4370   
  3869         -    fts5IndexMerge(p, ppStruct, nRem);
         4371  +    fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge);
  3870   4372     }
  3871   4373   }
  3872   4374   
  3873   4375   static void fts5IndexCrisismerge(
  3874   4376     Fts5Index *p,                   /* FTS5 backend object */
  3875   4377     Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
  3876   4378   ){
................................................................................
  3932   4434     int iSegid;
  3933   4435     int pgnoLast = 0;                 /* Last leaf page number in segment */
  3934   4436   
  3935   4437     /* Obtain a reference to the index structure and allocate a new segment-id
  3936   4438     ** for the new level-0 segment.  */
  3937   4439     pStruct = fts5StructureRead(p);
  3938   4440     iSegid = fts5AllocateSegid(p, pStruct);
         4441  +  fts5StructureInvalidate(p);
  3939   4442   
  3940   4443     if( iSegid ){
  3941   4444       const int pgsz = p->pConfig->pgsz;
  3942   4445       int eDetail = p->pConfig->eDetail;
  3943   4446       Fts5StructureSegment *pSeg;   /* New segment within pStruct */
  3944   4447       Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
  3945   4448       Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
................................................................................
  4082   4585     if( p->nPendingData ){
  4083   4586       assert( p->pHash );
  4084   4587       p->nPendingData = 0;
  4085   4588       fts5FlushOneHash(p);
  4086   4589     }
  4087   4590   }
  4088   4591   
  4089         -
  4090         -int sqlite3Fts5IndexOptimize(Fts5Index *p){
  4091         -  Fts5Structure *pStruct;
         4592  +static Fts5Structure *fts5IndexOptimizeStruct(
         4593  +  Fts5Index *p, 
         4594  +  Fts5Structure *pStruct
         4595  +){
  4092   4596     Fts5Structure *pNew = 0;
  4093         -  int nSeg = 0;
  4094         -
  4095         -  assert( p->rc==SQLITE_OK );
  4096         -  fts5IndexFlush(p);
  4097         -  pStruct = fts5StructureRead(p);
  4098         -
  4099         -  if( pStruct ){
  4100         -    assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  4101         -    nSeg = pStruct->nSegment;
  4102         -    if( nSeg>1 ){
  4103         -      int nByte = sizeof(Fts5Structure);
  4104         -      nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
  4105         -      pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
  4106         -    }
  4107         -  }
         4597  +  int nByte = sizeof(Fts5Structure);
         4598  +  int nSeg = pStruct->nSegment;
         4599  +  int i;
         4600  +
         4601  +  /* Figure out if this structure requires optimization. A structure does
         4602  +  ** not require optimization if either:
         4603  +  **
         4604  +  **  + it consists of fewer than two segments, or 
         4605  +  **  + all segments are on the same level, or
         4606  +  **  + all segments except one are currently inputs to a merge operation.
         4607  +  **
         4608  +  ** In the first case, return NULL. In the second, increment the ref-count
         4609  +  ** on *pStruct and return a copy of the pointer to it.
         4610  +  */
         4611  +  if( nSeg<2 ) return 0;
         4612  +  for(i=0; i<pStruct->nLevel; i++){
         4613  +    int nThis = pStruct->aLevel[i].nSeg;
         4614  +    if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){
         4615  +      fts5StructureRef(pStruct);
         4616  +      return pStruct;
         4617  +    }
         4618  +    assert( pStruct->aLevel[i].nMerge<=nThis );
         4619  +  }
         4620  +
         4621  +  nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
         4622  +  pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
         4623  +
  4108   4624     if( pNew ){
  4109   4625       Fts5StructureLevel *pLvl;
  4110         -    int nByte = nSeg * sizeof(Fts5StructureSegment);
         4626  +    nByte = nSeg * sizeof(Fts5StructureSegment);
  4111   4627       pNew->nLevel = pStruct->nLevel+1;
  4112   4628       pNew->nRef = 1;
  4113   4629       pNew->nWriteCounter = pStruct->nWriteCounter;
  4114   4630       pLvl = &pNew->aLevel[pStruct->nLevel];
  4115   4631       pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
  4116   4632       if( pLvl->aSeg ){
  4117   4633         int iLvl, iSeg;
  4118   4634         int iSegOut = 0;
  4119         -      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         4635  +      /* Iterate through all segments, from oldest to newest. Add them to
         4636  +      ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest
         4637  +      ** segment in the data structure.  */
         4638  +      for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
  4120   4639           for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  4121   4640             pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
  4122   4641             iSegOut++;
  4123   4642           }
  4124   4643         }
  4125   4644         pNew->nSegment = pLvl->nSeg = nSeg;
  4126   4645       }else{
  4127   4646         sqlite3_free(pNew);
  4128   4647         pNew = 0;
  4129   4648       }
  4130   4649     }
  4131   4650   
         4651  +  return pNew;
         4652  +}
         4653  +
         4654  +int sqlite3Fts5IndexOptimize(Fts5Index *p){
         4655  +  Fts5Structure *pStruct;
         4656  +  Fts5Structure *pNew = 0;
         4657  +
         4658  +  assert( p->rc==SQLITE_OK );
         4659  +  fts5IndexFlush(p);
         4660  +  pStruct = fts5StructureRead(p);
         4661  +  fts5StructureInvalidate(p);
         4662  +
         4663  +  if( pStruct ){
         4664  +    pNew = fts5IndexOptimizeStruct(p, pStruct);
         4665  +  }
         4666  +  fts5StructureRelease(pStruct);
         4667  +
         4668  +  assert( pNew==0 || pNew->nSegment>0 );
  4132   4669     if( pNew ){
  4133         -    int iLvl = pNew->nLevel-1;
         4670  +    int iLvl;
         4671  +    for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){}
  4134   4672       while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
  4135   4673         int nRem = FTS5_OPT_WORK_UNIT;
  4136   4674         fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
  4137   4675       }
  4138   4676   
  4139   4677       fts5StructureWrite(p, pNew);
  4140   4678       fts5StructureRelease(pNew);
  4141   4679     }
  4142   4680   
  4143         -  fts5StructureRelease(pStruct);
  4144   4681     return fts5IndexReturn(p); 
  4145   4682   }
  4146   4683   
         4684  +/*
         4685  +** This is called to implement the special "VALUES('merge', $nMerge)"
         4686  +** INSERT command.
         4687  +*/
  4147   4688   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  4148         -  Fts5Structure *pStruct;
  4149         -
  4150         -  pStruct = fts5StructureRead(p);
  4151         -  if( pStruct && pStruct->nLevel ){
  4152         -    fts5IndexMerge(p, &pStruct, nMerge);
  4153         -    fts5StructureWrite(p, pStruct);
         4689  +  Fts5Structure *pStruct = fts5StructureRead(p);
         4690  +  if( pStruct ){
         4691  +    int nMin = p->pConfig->nUsermerge;
         4692  +    fts5StructureInvalidate(p);
         4693  +    if( nMerge<0 ){
         4694  +      Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
         4695  +      fts5StructureRelease(pStruct);
         4696  +      pStruct = pNew;
         4697  +      nMin = 2;
         4698  +      nMerge = nMerge*-1;
         4699  +    }
         4700  +    if( pStruct && pStruct->nLevel ){
         4701  +      if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){
         4702  +        fts5StructureWrite(p, pStruct);
         4703  +      }
         4704  +    }
         4705  +    fts5StructureRelease(pStruct);
  4154   4706     }
  4155         -  fts5StructureRelease(pStruct);
  4156         -
  4157   4707     return fts5IndexReturn(p);
  4158   4708   }
  4159   4709   
  4160         -static void fts5PoslistCallback(
  4161         -  Fts5Index *p, 
  4162         -  void *pContext, 
  4163         -  const u8 *pChunk, int nChunk
  4164         -){
  4165         -  assert_nc( nChunk>=0 );
  4166         -  if( nChunk>0 ){
  4167         -    fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk);
  4168         -  }
  4169         -}
  4170         -
  4171         -typedef struct PoslistCallbackCtx PoslistCallbackCtx;
  4172         -struct PoslistCallbackCtx {
  4173         -  Fts5Buffer *pBuf;               /* Append to this buffer */
  4174         -  Fts5Colset *pColset;            /* Restrict matches to this column */
  4175         -  int eState;                     /* See above */
  4176         -};
  4177         -
  4178         -typedef struct PoslistOffsetsCtx PoslistOffsetsCtx;
  4179         -struct PoslistOffsetsCtx {
  4180         -  Fts5Buffer *pBuf;               /* Append to this buffer */
  4181         -  Fts5Colset *pColset;            /* Restrict matches to this column */
  4182         -  int iRead;
  4183         -  int iWrite;
  4184         -};
  4185         -
  4186         -/*
  4187         -** TODO: Make this more efficient!
  4188         -*/
  4189         -static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
  4190         -  int i;
  4191         -  for(i=0; i<pColset->nCol; i++){
  4192         -    if( pColset->aiCol[i]==iCol ) return 1;
  4193         -  }
  4194         -  return 0;
  4195         -}
  4196         -
  4197         -static void fts5PoslistOffsetsCallback(
  4198         -  Fts5Index *p, 
  4199         -  void *pContext, 
  4200         -  const u8 *pChunk, int nChunk
  4201         -){
  4202         -  PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext;
  4203         -  assert_nc( nChunk>=0 );
  4204         -  if( nChunk>0 ){
  4205         -    int i = 0;
  4206         -    while( i<nChunk ){
  4207         -      int iVal;
  4208         -      i += fts5GetVarint32(&pChunk[i], iVal);
  4209         -      iVal += pCtx->iRead - 2;
  4210         -      pCtx->iRead = iVal;
  4211         -      if( fts5IndexColsetTest(pCtx->pColset, iVal) ){
  4212         -        fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite);
  4213         -        pCtx->iWrite = iVal;
  4214         -      }
  4215         -    }
  4216         -  }
  4217         -}
  4218         -
  4219         -static void fts5PoslistFilterCallback(
  4220         -  Fts5Index *p, 
  4221         -  void *pContext, 
  4222         -  const u8 *pChunk, int nChunk
  4223         -){
  4224         -  PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
  4225         -  assert_nc( nChunk>=0 );
  4226         -  if( nChunk>0 ){
  4227         -    /* Search through to find the first varint with value 1. This is the
  4228         -    ** start of the next columns hits. */
  4229         -    int i = 0;
  4230         -    int iStart = 0;
  4231         -
  4232         -    if( pCtx->eState==2 ){
  4233         -      int iCol;
  4234         -      fts5FastGetVarint32(pChunk, i, iCol);
  4235         -      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
  4236         -        pCtx->eState = 1;
  4237         -        fts5BufferSafeAppendVarint(pCtx->pBuf, 1);
  4238         -      }else{
  4239         -        pCtx->eState = 0;
  4240         -      }
  4241         -    }
  4242         -
  4243         -    do {
  4244         -      while( i<nChunk && pChunk[i]!=0x01 ){
  4245         -        while( pChunk[i] & 0x80 ) i++;
  4246         -        i++;
  4247         -      }
  4248         -      if( pCtx->eState ){
  4249         -        fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
  4250         -      }
  4251         -      if( i<nChunk ){
  4252         -        int iCol;
  4253         -        iStart = i;
  4254         -        i++;
  4255         -        if( i>=nChunk ){
  4256         -          pCtx->eState = 2;
  4257         -        }else{
  4258         -          fts5FastGetVarint32(pChunk, i, iCol);
  4259         -          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
  4260         -          if( pCtx->eState ){
  4261         -            fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
  4262         -            iStart = i;
  4263         -          }
  4264         -        }
  4265         -      }
  4266         -    }while( i<nChunk );
  4267         -  }
  4268         -}
  4269         -
  4270         -/*
  4271         -** Iterator pIter currently points to a valid entry (not EOF). This
  4272         -** function appends the position list data for the current entry to
  4273         -** buffer pBuf. It does not make a copy of the position-list size
  4274         -** field.
  4275         -*/
  4276         -static void fts5SegiterPoslist(
         4710  +static void fts5AppendRowid(
         4711  +  Fts5Index *p,
         4712  +  i64 iDelta,
         4713  +  Fts5Iter *pUnused,
         4714  +  Fts5Buffer *pBuf
         4715  +){
         4716  +  UNUSED_PARAM(pUnused);
         4717  +  fts5BufferAppendVarint(&p->rc, pBuf, iDelta);
         4718  +}
         4719  +
         4720  +static void fts5AppendPoslist(
  4277   4721     Fts5Index *p,
  4278         -  Fts5SegIter *pSeg,
  4279         -  Fts5Colset *pColset,
         4722  +  i64 iDelta,
         4723  +  Fts5Iter *pMulti,
  4280   4724     Fts5Buffer *pBuf
  4281   4725   ){
  4282         -  if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
  4283         -    if( pColset==0 ){
  4284         -      fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
  4285         -    }else{
  4286         -      if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){
  4287         -        PoslistCallbackCtx sCtx;
  4288         -        sCtx.pBuf = pBuf;
  4289         -        sCtx.pColset = pColset;
  4290         -        sCtx.eState = fts5IndexColsetTest(pColset, 0);
  4291         -        assert( sCtx.eState==0 || sCtx.eState==1 );
  4292         -        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
  4293         -      }else{
  4294         -        PoslistOffsetsCtx sCtx;
  4295         -        memset(&sCtx, 0, sizeof(sCtx));
  4296         -        sCtx.pBuf = pBuf;
  4297         -        sCtx.pColset = pColset;
  4298         -        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
  4299         -      }
  4300         -    }
  4301         -  }
  4302         -}
  4303         -
  4304         -/*
  4305         -** IN/OUT parameter (*pa) points to a position list n bytes in size. If
  4306         -** the position list contains entries for column iCol, then (*pa) is set
  4307         -** to point to the sub-position-list for that column and the number of
  4308         -** bytes in it returned. Or, if the argument position list does not
  4309         -** contain any entries for column iCol, return 0.
  4310         -*/
  4311         -static int fts5IndexExtractCol(
  4312         -  const u8 **pa,                  /* IN/OUT: Pointer to poslist */
  4313         -  int n,                          /* IN: Size of poslist in bytes */
  4314         -  int iCol                        /* Column to extract from poslist */
  4315         -){
  4316         -  int iCurrent = 0;               /* Anything before the first 0x01 is col 0 */
  4317         -  const u8 *p = *pa;
  4318         -  const u8 *pEnd = &p[n];         /* One byte past end of position list */
  4319         -  u8 prev = 0;
  4320         -
  4321         -  while( iCol>iCurrent ){
  4322         -    /* Advance pointer p until it points to pEnd or an 0x01 byte that is
  4323         -    ** not part of a varint */
  4324         -    while( (prev & 0x80) || *p!=0x01 ){
  4325         -      prev = *p++;
  4326         -      if( p==pEnd ) return 0;
  4327         -    }
  4328         -    *pa = p++;
  4329         -    p += fts5GetVarint32(p, iCurrent);
  4330         -  }
  4331         -  if( iCol!=iCurrent ) return 0;
  4332         -
  4333         -  /* Advance pointer p until it points to pEnd or an 0x01 byte that is
  4334         -  ** not part of a varint */
  4335         -  assert( (prev & 0x80)==0 );
  4336         -  while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
  4337         -    prev = *p++;
  4338         -  }
  4339         -  return p - (*pa);
  4340         -}
  4341         -
  4342         -static int fts5AppendRowid(
  4343         -  Fts5Index *p,
  4344         -  i64 iDelta,
  4345         -  Fts5IndexIter *pMulti,
  4346         -  Fts5Colset *pColset,
  4347         -  Fts5Buffer *pBuf
  4348         -){
  4349         -  fts5BufferAppendVarint(&p->rc, pBuf, iDelta);
  4350         -  return 0;
  4351         -}
  4352         -
  4353         -/*
  4354         -** Iterator pMulti currently points to a valid entry (not EOF). This
  4355         -** function appends the following to buffer pBuf:
  4356         -**
  4357         -**   * The varint iDelta, and
  4358         -**   * the position list that currently points to, including the size field.
  4359         -**
  4360         -** If argument pColset is NULL, then the position list is filtered according
  4361         -** to pColset before being appended to the buffer. If this means there are
  4362         -** no entries in the position list, nothing is appended to the buffer (not
  4363         -** even iDelta).
  4364         -**
  4365         -** If an error occurs, an error code is left in p->rc. 
  4366         -*/
  4367         -static int fts5AppendPoslist(
  4368         -  Fts5Index *p,
  4369         -  i64 iDelta,
  4370         -  Fts5IndexIter *pMulti,
  4371         -  Fts5Colset *pColset,
  4372         -  Fts5Buffer *pBuf
  4373         -){
  4374         -  if( p->rc==SQLITE_OK ){
  4375         -    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  4376         -    assert( fts5MultiIterEof(p, pMulti)==0 );
  4377         -    assert( pSeg->nPos>0 );
  4378         -    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
  4379         -      if( p->pConfig->eDetail==FTS5_DETAIL_FULL
  4380         -       && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  4381         -       && (pColset==0 || pColset->nCol==1)
  4382         -      ){
  4383         -        const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4384         -        int nPos;
  4385         -        if( pColset ){
  4386         -          nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
  4387         -          if( nPos==0 ) return 1;
  4388         -        }else{
  4389         -          nPos = pSeg->nPos;
  4390         -        }
  4391         -        assert( nPos>0 );
  4392         -        fts5BufferSafeAppendVarint(pBuf, iDelta);
  4393         -        fts5BufferSafeAppendVarint(pBuf, nPos*2);
  4394         -        fts5BufferSafeAppendBlob(pBuf, pPos, nPos);
  4395         -      }else{
  4396         -        int iSv1;
  4397         -        int iSv2;
  4398         -        int iData;
  4399         -
  4400         -        /* Append iDelta */
  4401         -        iSv1 = pBuf->n;
  4402         -        fts5BufferSafeAppendVarint(pBuf, iDelta);
  4403         -
  4404         -        /* WRITEPOSLISTSIZE */
  4405         -        iSv2 = pBuf->n;
  4406         -        fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
  4407         -        iData = pBuf->n;
  4408         -
  4409         -        fts5SegiterPoslist(p, pSeg, pColset, pBuf);
  4410         -
  4411         -        if( pColset ){
  4412         -          int nActual = pBuf->n - iData;
  4413         -          if( nActual!=pSeg->nPos ){
  4414         -            if( nActual==0 ){
  4415         -              pBuf->n = iSv1;
  4416         -              return 1;
  4417         -            }else{
  4418         -              int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
  4419         -              while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
  4420         -              sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
  4421         -            }
  4422         -          }
  4423         -        }
  4424         -      }
  4425         -    }
  4426         -  }
  4427         -
  4428         -  return 0;
         4726  +  int nData = pMulti->base.nData;
         4727  +  assert( nData>0 );
         4728  +  if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){
         4729  +    fts5BufferSafeAppendVarint(pBuf, iDelta);
         4730  +    fts5BufferSafeAppendVarint(pBuf, nData*2);
         4731  +    fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData);
         4732  +  }
  4429   4733   }
  4430   4734   
  4431   4735   
  4432   4736   static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  4433   4737     u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
  4434   4738   
  4435   4739     assert( pIter->aPoslist );
................................................................................
  4565   4869     Fts5Buffer *p1,                 /* First list to merge */
  4566   4870     Fts5Buffer *p2                  /* Second list to merge */
  4567   4871   ){
  4568   4872     if( p2->n ){
  4569   4873       i64 iLastRowid = 0;
  4570   4874       Fts5DoclistIter i1;
  4571   4875       Fts5DoclistIter i2;
  4572         -    Fts5Buffer out;
  4573         -    Fts5Buffer tmp;
  4574         -    memset(&out, 0, sizeof(out));
  4575         -    memset(&tmp, 0, sizeof(tmp));
         4876  +    Fts5Buffer out = {0, 0, 0};
         4877  +    Fts5Buffer tmp = {0, 0, 0};
  4576   4878   
  4577         -    sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
         4879  +    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
  4578   4880       fts5DoclistIterInit(p1, &i1);
  4579   4881       fts5DoclistIterInit(p2, &i2);
  4580         -    while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
  4581         -      if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
         4882  +
         4883  +    while( 1 ){
         4884  +      if( i1.iRowid<i2.iRowid ){
  4582   4885           /* Copy entry from i1 */
  4583   4886           fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  4584   4887           fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
  4585   4888           fts5DoclistIterNext(&i1);
         4889  +        if( i1.aPoslist==0 ) break;
  4586   4890         }
  4587         -      else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
         4891  +      else if( i2.iRowid!=i1.iRowid ){
  4588   4892           /* Copy entry from i2 */
  4589   4893           fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4590   4894           fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
  4591   4895           fts5DoclistIterNext(&i2);
         4896  +        if( i2.aPoslist==0 ) break;
  4592   4897         }
  4593   4898         else{
         4899  +        /* Merge the two position lists. */ 
  4594   4900           i64 iPos1 = 0;
  4595   4901           i64 iPos2 = 0;
  4596   4902           int iOff1 = 0;
  4597   4903           int iOff2 = 0;
  4598   4904           u8 *a1 = &i1.aPoslist[i1.nSize];
  4599   4905           u8 *a2 = &i2.aPoslist[i2.nSize];
  4600   4906   
         4907  +        i64 iPrev = 0;
  4601   4908           Fts5PoslistWriter writer;
  4602   4909           memset(&writer, 0, sizeof(writer));
  4603   4910   
  4604         -        /* Merge the two position lists. */ 
  4605   4911           fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4606   4912           fts5BufferZero(&tmp);
         4913  +        sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist);
         4914  +        if( p->rc ) break;
  4607   4915   
  4608   4916           sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
  4609   4917           sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4918  +        assert( iPos1>=0 && iPos2>=0 );
  4610   4919   
  4611         -        while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){
  4612         -          i64 iNew;
  4613         -          if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
  4614         -            iNew = iPos1;
  4615         -            sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
  4616         -          }else{
  4617         -            iNew = iPos2;
  4618         -            sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4619         -            if( iPos1==iPos2 ){
  4620         -              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
         4920  +        if( iPos1<iPos2 ){
         4921  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
         4922  +          sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4923  +        }else{
         4924  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
         4925  +          sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4926  +        }
         4927  +
         4928  +        if( iPos1>=0 && iPos2>=0 ){
         4929  +          while( 1 ){
         4930  +            if( iPos1<iPos2 ){
         4931  +              if( iPos1!=iPrev ){
         4932  +                sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
         4933  +              }
         4934  +              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4935  +              if( iPos1<0 ) break;
         4936  +            }else{
         4937  +              assert( iPos2!=iPrev );
         4938  +              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
         4939  +              sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4940  +              if( iPos2<0 ) break;
  4621   4941               }
  4622   4942             }
  4623         -          if( iNew!=writer.iPrev || tmp.n==0 ){
  4624         -            p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
         4943  +        }
         4944  +
         4945  +        if( iPos1>=0 ){
         4946  +          if( iPos1!=iPrev ){
         4947  +            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4625   4948             }
         4949  +          fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1);
         4950  +        }else{
         4951  +          assert( iPos2>=0 && iPos2!=iPrev );
         4952  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
         4953  +          fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2);
  4626   4954           }
  4627   4955   
  4628   4956           /* WRITEPOSLISTSIZE */
  4629   4957           fts5BufferSafeAppendVarint(&out, tmp.n * 2);
  4630   4958           fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
  4631   4959           fts5DoclistIterNext(&i1);
  4632   4960           fts5DoclistIterNext(&i2);
         4961  +        if( i1.aPoslist==0 || i2.aPoslist==0 ) break;
  4633   4962         }
  4634   4963       }
         4964  +
         4965  +    if( i1.aPoslist ){
         4966  +      fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
         4967  +      fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
         4968  +    }
         4969  +    else if( i2.aPoslist ){
         4970  +      fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4971  +      fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
         4972  +    }
  4635   4973   
  4636   4974       fts5BufferSet(&p->rc, p1, out.n, out.p);
  4637   4975       fts5BufferFree(&tmp);
  4638   4976       fts5BufferFree(&out);
  4639   4977     }
  4640   4978   }
  4641   4979   
  4642   4980   static void fts5SetupPrefixIter(
  4643   4981     Fts5Index *p,                   /* Index to read from */
  4644   4982     int bDesc,                      /* True for "ORDER BY rowid DESC" */
  4645   4983     const u8 *pToken,               /* Buffer containing prefix to match */
  4646   4984     int nToken,                     /* Size of buffer pToken in bytes */
  4647   4985     Fts5Colset *pColset,            /* Restrict matches to these columns */
  4648         -  Fts5IndexIter **ppIter          /* OUT: New iterator */
         4986  +  Fts5Iter **ppIter          /* OUT: New iterator */
  4649   4987   ){
  4650   4988     Fts5Structure *pStruct;
  4651   4989     Fts5Buffer *aBuf;
  4652   4990     const int nBuf = 32;
  4653   4991   
  4654   4992     void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*);
  4655         -  int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*);
         4993  +  void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*);
  4656   4994     if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
  4657   4995       xMerge = fts5MergeRowidLists;
  4658   4996       xAppend = fts5AppendRowid;
  4659   4997     }else{
  4660   4998       xMerge = fts5MergePrefixLists;
  4661   4999       xAppend = fts5AppendPoslist;
  4662   5000     }
  4663   5001   
  4664   5002     aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  4665   5003     pStruct = fts5StructureRead(p);
  4666   5004   
  4667   5005     if( aBuf && pStruct ){
  4668         -    const int flags = FTS5INDEX_QUERY_SCAN;
         5006  +    const int flags = FTS5INDEX_QUERY_SCAN 
         5007  +                    | FTS5INDEX_QUERY_SKIPEMPTY 
         5008  +                    | FTS5INDEX_QUERY_NOOUTPUT;
  4669   5009       int i;
  4670   5010       i64 iLastRowid = 0;
  4671         -    Fts5IndexIter *p1 = 0;     /* Iterator used to gather data from index */
         5011  +    Fts5Iter *p1 = 0;     /* Iterator used to gather data from index */
  4672   5012       Fts5Data *pData;
  4673   5013       Fts5Buffer doclist;
  4674   5014       int bNewTerm = 1;
  4675   5015   
  4676   5016       memset(&doclist, 0, sizeof(doclist));
  4677         -    for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
         5017  +    fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1);
         5018  +    fts5IterSetOutputCb(&p->rc, p1);
         5019  +    for( /* no-op */ ;
  4678   5020           fts5MultiIterEof(p, p1)==0;
  4679   5021           fts5MultiIterNext2(p, p1, &bNewTerm)
  4680   5022       ){
  4681         -      i64 iRowid = fts5MultiIterRowid(p1);
  4682         -      int nTerm;
  4683         -      const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
         5023  +      Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
         5024  +      int nTerm = pSeg->term.n;
         5025  +      const u8 *pTerm = pSeg->term.p;
         5026  +      p1->xSetOutputs(p1, pSeg);
         5027  +
  4684   5028         assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
  4685   5029         if( bNewTerm ){
  4686   5030           if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
  4687   5031         }
  4688   5032   
  4689         -      if( doclist.n>0 && iRowid<=iLastRowid ){
         5033  +      if( p1->base.nData==0 ) continue;
         5034  +
         5035  +      if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){
  4690   5036           for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
  4691   5037             assert( i<nBuf );
  4692   5038             if( aBuf[i].n==0 ){
  4693   5039               fts5BufferSwap(&doclist, &aBuf[i]);
  4694   5040               fts5BufferZero(&doclist);
  4695   5041             }else{
  4696   5042               xMerge(p, &doclist, &aBuf[i]);
  4697   5043               fts5BufferZero(&aBuf[i]);
  4698   5044             }
  4699   5045           }
  4700   5046           iLastRowid = 0;
  4701   5047         }
  4702   5048   
  4703         -      if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
  4704         -        iLastRowid = iRowid;
  4705         -      }
         5049  +      xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist);
         5050  +      iLastRowid = p1->base.iRowid;
  4706   5051       }
  4707   5052   
  4708   5053       for(i=0; i<nBuf; i++){
  4709   5054         if( p->rc==SQLITE_OK ){
  4710   5055           xMerge(p, &doclist, &aBuf[i]);
  4711   5056         }
  4712   5057         fts5BufferFree(&aBuf[i]);
  4713   5058       }
  4714         -    fts5MultiIterFree(p, p1);
         5059  +    fts5MultiIterFree(p1);
  4715   5060   
  4716   5061       pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
  4717   5062       if( pData ){
  4718   5063         pData->p = (u8*)&pData[1];
  4719   5064         pData->nn = pData->szLeaf = doclist.n;
  4720   5065         memcpy(pData->p, doclist.p, doclist.n);
  4721   5066         fts5MultiIterNew2(p, pData, bDesc, ppIter);
................................................................................
  4768   5113   ** to the database. Additionally, assume that the contents of the %_data
  4769   5114   ** table may have changed on disk. So any in-memory caches of %_data 
  4770   5115   ** records must be invalidated.
  4771   5116   */
  4772   5117   int sqlite3Fts5IndexRollback(Fts5Index *p){
  4773   5118     fts5CloseReader(p);
  4774   5119     fts5IndexDiscardData(p);
  4775         -  assert( p->rc==SQLITE_OK );
         5120  +  fts5StructureInvalidate(p);
         5121  +  /* assert( p->rc==SQLITE_OK ); */
  4776   5122     return SQLITE_OK;
  4777   5123   }
  4778   5124   
  4779   5125   /*
  4780   5126   ** The %_data table is completely empty when this function is called. This
  4781   5127   ** function populates it with the initial structure objects for each index,
  4782   5128   ** and the initial version of the "averages" record (a zero-byte blob).
  4783   5129   */
  4784   5130   int sqlite3Fts5IndexReinit(Fts5Index *p){
  4785   5131     Fts5Structure s;
         5132  +  fts5StructureInvalidate(p);
  4786   5133     memset(&s, 0, sizeof(Fts5Structure));
  4787   5134     fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
  4788   5135     fts5StructureWrite(p, &s);
  4789   5136     return fts5IndexReturn(p);
  4790   5137   }
  4791   5138   
  4792   5139   /*
................................................................................
  4837   5184   /*
  4838   5185   ** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
  4839   5186   */
  4840   5187   int sqlite3Fts5IndexClose(Fts5Index *p){
  4841   5188     int rc = SQLITE_OK;
  4842   5189     if( p ){
  4843   5190       assert( p->pReader==0 );
         5191  +    fts5StructureInvalidate(p);
  4844   5192       sqlite3_finalize(p->pWriter);
  4845   5193       sqlite3_finalize(p->pDeleter);
  4846   5194       sqlite3_finalize(p->pIdxWriter);
  4847   5195       sqlite3_finalize(p->pIdxDeleter);
  4848   5196       sqlite3_finalize(p->pIdxSelect);
         5197  +    sqlite3_finalize(p->pDataVersion);
  4849   5198       sqlite3Fts5HashFree(p->pHash);
  4850   5199       sqlite3_free(p->zDataTbl);
  4851   5200       sqlite3_free(p);
  4852   5201     }
  4853   5202     return rc;
  4854   5203   }
  4855   5204   
................................................................................
  4940   5289     Fts5Index *p,                   /* FTS index to query */
  4941   5290     const char *pToken, int nToken, /* Token (or prefix) to query for */
  4942   5291     int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
  4943   5292     Fts5Colset *pColset,            /* Match these columns only */
  4944   5293     Fts5IndexIter **ppIter          /* OUT: New iterator object */
  4945   5294   ){
  4946   5295     Fts5Config *pConfig = p->pConfig;
  4947         -  Fts5IndexIter *pRet = 0;
  4948         -  int iIdx = 0;
         5296  +  Fts5Iter *pRet = 0;
  4949   5297     Fts5Buffer buf = {0, 0, 0};
  4950   5298   
  4951   5299     /* If the QUERY_SCAN flag is set, all other flags must be clear. */
  4952   5300     assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN );
  4953   5301   
  4954   5302     if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
         5303  +    int iIdx = 0;                 /* Index to search */
  4955   5304       memcpy(&buf.p[1], pToken, nToken);
  4956   5305   
  4957         -#ifdef SQLITE_DEBUG
  4958         -    /* If the QUERY_TEST_NOIDX flag was specified, then this must be a
         5306  +    /* Figure out which index to search and set iIdx accordingly. If this
         5307  +    ** is a prefix query for which there is no prefix index, set iIdx to
         5308  +    ** greater than pConfig->nPrefix to indicate that the query will be
         5309  +    ** satisfied by scanning multiple terms in the main index.
         5310  +    **
         5311  +    ** If the QUERY_TEST_NOIDX flag was specified, then this must be a
  4959   5312       ** prefix-query. Instead of using a prefix-index (if one exists), 
  4960   5313       ** evaluate the prefix query using the main FTS index. This is used
  4961   5314       ** for internal sanity checking by the integrity-check in debug 
  4962   5315       ** mode only.  */
         5316  +#ifdef SQLITE_DEBUG
  4963   5317       if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
  4964   5318         assert( flags & FTS5INDEX_QUERY_PREFIX );
  4965   5319         iIdx = 1+pConfig->nPrefix;
  4966   5320       }else
  4967   5321   #endif
  4968   5322       if( flags & FTS5INDEX_QUERY_PREFIX ){
  4969   5323         int nChar = fts5IndexCharlen(pToken, nToken);
  4970   5324         for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
  4971   5325           if( pConfig->aPrefix[iIdx-1]==nChar ) break;
  4972   5326         }
  4973   5327       }
  4974   5328   
  4975   5329       if( iIdx<=pConfig->nPrefix ){
         5330  +      /* Straight index lookup */
  4976   5331         Fts5Structure *pStruct = fts5StructureRead(p);
  4977   5332         buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx);
  4978   5333         if( pStruct ){
  4979         -        fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet);
         5334  +        fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, 
         5335  +            pColset, buf.p, nToken+1, -1, 0, &pRet
         5336  +        );
  4980   5337           fts5StructureRelease(pStruct);
  4981   5338         }
  4982   5339       }else{
         5340  +      /* Scan multiple terms in the main index */
  4983   5341         int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
  4984   5342         buf.p[0] = FTS5_MAIN_PREFIX;
  4985   5343         fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
         5344  +      assert( p->rc!=SQLITE_OK || pRet->pColset==0 );
         5345  +      fts5IterSetOutputCb(&p->rc, pRet);
         5346  +      if( p->rc==SQLITE_OK ){
         5347  +        Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
         5348  +        if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
         5349  +      }
  4986   5350       }
  4987   5351   
  4988   5352       if( p->rc ){
  4989         -      sqlite3Fts5IterClose(pRet);
         5353  +      sqlite3Fts5IterClose(&pRet->base);
  4990   5354         pRet = 0;
  4991   5355         fts5CloseReader(p);
  4992   5356       }
  4993         -    *ppIter = pRet;
         5357  +
         5358  +    *ppIter = &pRet->base;
  4994   5359       sqlite3Fts5BufferFree(&buf);
  4995   5360     }
  4996   5361     return fts5IndexReturn(p);
  4997   5362   }
  4998   5363   
  4999   5364   /*
  5000   5365   ** Return true if the iterator passed as the only argument is at EOF.
  5001   5366   */
  5002         -int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
  5003         -  assert( pIter->pIndex->rc==SQLITE_OK );
  5004         -  return pIter->bEof;
  5005         -}
  5006         -
  5007   5367   /*
  5008   5368   ** Move to the next matching rowid. 
  5009   5369   */
  5010         -int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
         5370  +int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){
         5371  +  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5011   5372     assert( pIter->pIndex->rc==SQLITE_OK );
  5012   5373     fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
  5013   5374     return fts5IndexReturn(pIter->pIndex);
  5014   5375   }
  5015   5376   
  5016   5377   /*
  5017   5378   ** Move to the next matching term/rowid. Used by the fts5vocab module.
  5018   5379   */
  5019         -int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
         5380  +int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){
         5381  +  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5020   5382     Fts5Index *p = pIter->pIndex;
  5021   5383   
  5022   5384     assert( pIter->pIndex->rc==SQLITE_OK );
  5023   5385   
  5024   5386     fts5MultiIterNext(p, pIter, 0, 0);
  5025   5387     if( p->rc==SQLITE_OK ){
  5026   5388       Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  5027   5389       if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){
  5028   5390         fts5DataRelease(pSeg->pLeaf);
  5029   5391         pSeg->pLeaf = 0;
  5030         -      pIter->bEof = 1;
         5392  +      pIter->base.bEof = 1;
  5031   5393       }
  5032   5394     }
  5033   5395   
  5034   5396     return fts5IndexReturn(pIter->pIndex);
  5035   5397   }
  5036   5398   
  5037   5399   /*
  5038   5400   ** Move to the next matching rowid that occurs at or after iMatch. The
  5039   5401   ** definition of "at or after" depends on whether this iterator iterates
  5040   5402   ** in ascending or descending rowid order.
  5041   5403   */
  5042         -int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
         5404  +int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){
         5405  +  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5043   5406     fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
  5044   5407     return fts5IndexReturn(pIter->pIndex);
  5045   5408   }
  5046   5409   
  5047         -/*
  5048         -** Return the current rowid.
  5049         -*/
  5050         -i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
  5051         -  return fts5MultiIterRowid(pIter);
  5052         -}
  5053         -
  5054   5410   /*
  5055   5411   ** Return the current term.
  5056   5412   */
  5057         -const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){
         5413  +const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
  5058   5414     int n;
  5059         -  const char *z = (const char*)fts5MultiIterTerm(pIter, &n);
         5415  +  const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n);
  5060   5416     *pn = n-1;
  5061   5417     return &z[1];
  5062   5418   }
  5063   5419   
  5064         -
  5065         -static int fts5IndexExtractColset (
  5066         -  Fts5Colset *pColset,            /* Colset to filter on */
  5067         -  const u8 *pPos, int nPos,       /* Position list */
  5068         -  Fts5Buffer *pBuf                /* Output buffer */
  5069         -){
  5070         -  int rc = SQLITE_OK;
  5071         -  int i;
  5072         -
  5073         -  fts5BufferZero(pBuf);
  5074         -  for(i=0; i<pColset->nCol; i++){
  5075         -    const u8 *pSub = pPos;
  5076         -    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
  5077         -    if( nSub ){
  5078         -      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
  5079         -    }
  5080         -  }
  5081         -  return rc;
  5082         -}
  5083         -
  5084         -
  5085         -/*
  5086         -** Return a pointer to a buffer containing a copy of the position list for
  5087         -** the current entry. Output variable *pn is set to the size of the buffer 
  5088         -** in bytes before returning.
  5089         -**
  5090         -** The returned position list does not include the "number of bytes" varint
  5091         -** field that starts the position list on disk.
  5092         -*/
  5093         -int sqlite3Fts5IterPoslist(
  5094         -  Fts5IndexIter *pIter, 
  5095         -  Fts5Colset *pColset,            /* Column filter (or NULL) */
  5096         -  const u8 **pp,                  /* OUT: Pointer to position-list data */
  5097         -  int *pn,                        /* OUT: Size of position-list in bytes */
  5098         -  i64 *piRowid                    /* OUT: Current rowid */
  5099         -){
  5100         -  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  5101         -  int eDetail = pIter->pIndex->pConfig->eDetail;
  5102         -
  5103         -  assert( pIter->pIndex->rc==SQLITE_OK );
  5104         -  *piRowid = pSeg->iRowid;
  5105         -  if( eDetail==FTS5_DETAIL_NONE ){
  5106         -    *pn = pSeg->nPos;
  5107         -  }else
  5108         -  if( eDetail==FTS5_DETAIL_FULL 
  5109         -   && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  5110         -  ){
  5111         -    u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  5112         -    if( pColset==0 || pIter->bFiltered ){
  5113         -      *pn = pSeg->nPos;
  5114         -      *pp = pPos;
  5115         -    }else if( pColset->nCol==1 ){
  5116         -      *pp = pPos;
  5117         -      *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]);
  5118         -    }else{
  5119         -      fts5BufferZero(&pIter->poslist);
  5120         -      fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist);
  5121         -      *pp = pIter->poslist.p;
  5122         -      *pn = pIter->poslist.n;
  5123         -    }
  5124         -  }else{
  5125         -    fts5BufferZero(&pIter->poslist);
  5126         -    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
  5127         -    if( eDetail==FTS5_DETAIL_FULL ){
  5128         -      *pp = pIter->poslist.p;
  5129         -    }
  5130         -    *pn = pIter->poslist.n;
  5131         -  }
  5132         -  return fts5IndexReturn(pIter->pIndex);
  5133         -}
  5134         -
  5135         -int sqlite3Fts5IterCollist(
  5136         -  Fts5IndexIter *pIter, 
  5137         -  const u8 **pp,                  /* OUT: Pointer to position-list data */
  5138         -  int *pn                         /* OUT: Size of position-list in bytes */
  5139         -){
  5140         -  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  5141         -  *pp = pIter->poslist.p;
  5142         -  *pn = pIter->poslist.n;
  5143         -  return SQLITE_OK;
  5144         -}
  5145         -
  5146         -/*
  5147         -** This function is similar to sqlite3Fts5IterPoslist(), except that it
  5148         -** copies the position list into the buffer supplied as the second 
  5149         -** argument.
  5150         -*/
  5151         -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
  5152         -  Fts5Index *p = pIter->pIndex;
  5153         -  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  5154         -  assert( p->rc==SQLITE_OK );
  5155         -  fts5BufferZero(pBuf);
  5156         -  fts5SegiterPoslist(p, pSeg, 0, pBuf);
  5157         -  return fts5IndexReturn(p);
  5158         -}
  5159         -
  5160   5420   /*
  5161   5421   ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
  5162   5422   */
  5163         -void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
  5164         -  if( pIter ){
         5423  +void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){
         5424  +  if( pIndexIter ){
         5425  +    Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5165   5426       Fts5Index *pIndex = pIter->pIndex;
  5166         -    fts5MultiIterFree(pIter->pIndex, pIter);
         5427  +    fts5MultiIterFree(pIter);
  5167   5428       fts5CloseReader(pIndex);
  5168   5429     }
  5169   5430   }
  5170   5431   
  5171   5432   /*
  5172   5433   ** Read and decode the "averages" record from the database. 
  5173   5434   **
................................................................................
  5324   5585     const char *z,                  /* Index key to query for */
  5325   5586     int n,                          /* Size of index key in bytes */
  5326   5587     int flags,                      /* Flags for Fts5IndexQuery */
  5327   5588     u64 *pCksum                     /* IN/OUT: Checksum value */
  5328   5589   ){
  5329   5590     int eDetail = p->pConfig->eDetail;
  5330   5591     u64 cksum = *pCksum;
  5331         -  Fts5IndexIter *pIdxIter = 0;
  5332         -  Fts5Buffer buf = {0, 0, 0};
  5333         -  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
         5592  +  Fts5IndexIter *pIter = 0;
         5593  +  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter);
  5334   5594   
  5335         -  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  5336         -    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
         5595  +  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){
         5596  +    i64 rowid = pIter->iRowid;
  5337   5597   
  5338   5598       if( eDetail==FTS5_DETAIL_NONE ){
  5339   5599         cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
  5340   5600       }else{
  5341         -      rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
  5342         -      if( rc==SQLITE_OK ){
  5343         -        Fts5PoslistReader sReader;
  5344         -        for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
  5345         -            sReader.bEof==0;
  5346         -            sqlite3Fts5PoslistReaderNext(&sReader)
  5347         -        ){
  5348         -          int iCol = FTS5_POS2COLUMN(sReader.iPos);
  5349         -          int iOff = FTS5_POS2OFFSET(sReader.iPos);
  5350         -          cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  5351         -        }
         5601  +      Fts5PoslistReader sReader;
         5602  +      for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader);
         5603  +          sReader.bEof==0;
         5604  +          sqlite3Fts5PoslistReaderNext(&sReader)
         5605  +      ){
         5606  +        int iCol = FTS5_POS2COLUMN(sReader.iPos);
         5607  +        int iOff = FTS5_POS2OFFSET(sReader.iPos);
         5608  +        cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  5352   5609         }
  5353   5610       }
  5354   5611       if( rc==SQLITE_OK ){
  5355         -      rc = sqlite3Fts5IterNext(pIdxIter);
         5612  +      rc = sqlite3Fts5IterNext(pIter);
  5356   5613       }
  5357   5614     }
  5358         -  sqlite3Fts5IterClose(pIdxIter);
  5359         -  fts5BufferFree(&buf);
         5615  +  sqlite3Fts5IterClose(pIter);
  5360   5616   
  5361   5617     *pCksum = cksum;
  5362   5618     return rc;
  5363   5619   }
  5364   5620   
  5365   5621   
  5366   5622   /*
................................................................................
  5657   5913   ** error, or some other SQLite error code if another error (e.g. OOM)
  5658   5914   ** occurs.
  5659   5915   */
  5660   5916   int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  5661   5917     int eDetail = p->pConfig->eDetail;
  5662   5918     u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  5663   5919     Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
  5664         -  Fts5IndexIter *pIter;           /* Used to iterate through entire index */
         5920  +  Fts5Iter *pIter;                /* Used to iterate through entire index */
  5665   5921     Fts5Structure *pStruct;         /* Index structure */
  5666   5922   
  5667   5923   #ifdef SQLITE_DEBUG
  5668   5924     /* Used by extra internal tests only run if NDEBUG is not defined */
  5669   5925     u64 cksum3 = 0;                 /* Checksum based on contents of indexes */
  5670   5926     Fts5Buffer term = {0,0,0};      /* Buffer used to hold most recent term */
  5671   5927   #endif
         5928  +  const int flags = FTS5INDEX_QUERY_NOOUTPUT;
  5672   5929     
  5673   5930     /* Load the FTS index structure */
  5674   5931     pStruct = fts5StructureRead(p);
  5675   5932   
  5676   5933     /* Check that the internal nodes of each segment match the leaves */
  5677   5934     if( pStruct ){
  5678   5935       int iLvl, iSeg;
................................................................................
  5693   5950     ** variable cksum2) based on entries extracted from the full-text index
  5694   5951     ** while doing a linear scan of each individual index in turn. 
  5695   5952     **
  5696   5953     ** As each term visited by the linear scans, a separate query for the
  5697   5954     ** same term is performed. cksum3 is calculated based on the entries
  5698   5955     ** extracted by these queries.
  5699   5956     */
  5700         -  for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter);
         5957  +  for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter);
  5701   5958         fts5MultiIterEof(p, pIter)==0;
  5702   5959         fts5MultiIterNext(p, pIter, 0, 0)
  5703   5960     ){
  5704   5961       int n;                      /* Size of term in bytes */
  5705   5962       i64 iPos = 0;               /* Position read from poslist */
  5706   5963       int iOff = 0;               /* Offset within poslist */
  5707   5964       i64 iRowid = fts5MultiIterRowid(pIter);
................................................................................
  5722   5979           int iTokOff = FTS5_POS2OFFSET(iPos);
  5723   5980           cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
  5724   5981         }
  5725   5982       }
  5726   5983     }
  5727   5984     fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
  5728   5985   
  5729         -  fts5MultiIterFree(p, pIter);
         5986  +  fts5MultiIterFree(pIter);
  5730   5987     if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
  5731   5988   
  5732   5989     fts5StructureRelease(pStruct);
  5733   5990   #ifdef SQLITE_DEBUG
  5734   5991     fts5BufferFree(&term);
  5735   5992   #endif
  5736   5993     fts5BufferFree(&poslist);
................................................................................
  5959   6216     u8 *a = 0;
  5960   6217     Fts5Buffer s;                   /* Build up text to return here */
  5961   6218     int rc = SQLITE_OK;             /* Return code */
  5962   6219     int nSpace = 0;
  5963   6220     int eDetailNone = (sqlite3_user_data(pCtx)!=0);
  5964   6221   
  5965   6222     assert( nArg==2 );
         6223  +  UNUSED_PARAM(nArg);
  5966   6224     memset(&s, 0, sizeof(Fts5Buffer));
  5967   6225     iRowid = sqlite3_value_int64(apVal[0]);
  5968   6226   
  5969   6227     /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[]
  5970   6228     ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents
  5971   6229     ** buffer overreads even if the record is corrupt.  */
  5972   6230     n = sqlite3_value_bytes(apVal[1]);
................................................................................
  6188   6446     if( rc==SQLITE_OK ){
  6189   6447       rc = sqlite3_create_function(
  6190   6448           db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
  6191   6449       );
  6192   6450     }
  6193   6451     return rc;
  6194   6452   }
         6453  +
         6454  +
         6455  +int sqlite3Fts5IndexReset(Fts5Index *p){
         6456  +  assert( p->pStruct==0 || p->iStructVersion!=0 );
         6457  +  if( fts5IndexDataVersion(p)!=p->iStructVersion ){
         6458  +    fts5StructureInvalidate(p);
         6459  +  }
         6460  +  return fts5IndexReturn(p);
         6461  +}

Changes to ext/fts5/fts5_main.c.

   216    216   #define FTS5_BI_ORDER_RANK   0x0020
   217    217   #define FTS5_BI_ORDER_ROWID  0x0040
   218    218   #define FTS5_BI_ORDER_DESC   0x0080
   219    219   
   220    220   /*
   221    221   ** Values for Fts5Cursor.csrflags
   222    222   */
   223         -#define FTS5CSR_REQUIRE_CONTENT   0x01
   224         -#define FTS5CSR_REQUIRE_DOCSIZE   0x02
   225         -#define FTS5CSR_REQUIRE_INST      0x04
   226         -#define FTS5CSR_EOF               0x08
          223  +#define FTS5CSR_EOF               0x01
          224  +#define FTS5CSR_REQUIRE_CONTENT   0x02
          225  +#define FTS5CSR_REQUIRE_DOCSIZE   0x04
          226  +#define FTS5CSR_REQUIRE_INST      0x08
   227    227   #define FTS5CSR_FREE_ZRANK        0x10
   228    228   #define FTS5CSR_REQUIRE_RESEEK    0x20
   229    229   #define FTS5CSR_REQUIRE_POSLIST   0x40
   230    230   
   231    231   #define BitFlagAllTest(x,y) (((x) & (y))==(y))
   232    232   #define BitFlagTest(x,y)    (((x) & (y))!=0)
   233    233   
................................................................................
   534    534     aColMap[1] = pConfig->nCol;
   535    535     aColMap[2] = pConfig->nCol+1;
   536    536   
   537    537     /* Set idxFlags flags for all WHERE clause terms that will be used. */
   538    538     for(i=0; i<pInfo->nConstraint; i++){
   539    539       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   540    540       int j;
   541         -    for(j=0; j<(int)ArraySize(aConstraint); j++){
          541  +    for(j=0; j<ArraySize(aConstraint); j++){
   542    542         struct Constraint *pC = &aConstraint[j];
   543    543         if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
   544    544           if( p->usable ){
   545    545             pC->iConsIndex = i;
   546    546             idxFlags |= pC->fts5op;
   547    547           }else if( j==0 ){
   548    548             /* As there exists an unusable MATCH constraint this is an 
................................................................................
   581    581       pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0;
   582    582     }else{
   583    583       pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0;
   584    584     }
   585    585   
   586    586     /* Assign argvIndex values to each constraint in use. */
   587    587     iNext = 1;
   588         -  for(i=0; i<(int)ArraySize(aConstraint); i++){
          588  +  for(i=0; i<ArraySize(aConstraint); i++){
   589    589       struct Constraint *pC = &aConstraint[i];
   590    590       if( pC->iConsIndex>=0 ){
   591    591         pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++;
   592    592         pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
   593    593       }
   594    594     }
   595    595   
   596    596     pInfo->idxNum = idxFlags;
   597    597     return SQLITE_OK;
   598    598   }
          599  +
          600  +static int fts5NewTransaction(Fts5Table *pTab){
          601  +  Fts5Cursor *pCsr;
          602  +  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
          603  +    if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
          604  +  }
          605  +  return sqlite3Fts5StorageReset(pTab->pStorage);
          606  +}
   599    607   
   600    608   /*
   601    609   ** Implementation of xOpen method.
   602    610   */
   603    611   static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
   604    612     Fts5Table *pTab = (Fts5Table*)pVTab;
   605    613     Fts5Config *pConfig = pTab->pConfig;
   606         -  Fts5Cursor *pCsr;               /* New cursor object */
          614  +  Fts5Cursor *pCsr = 0;           /* New cursor object */
   607    615     int nByte;                      /* Bytes of space to allocate */
   608         -  int rc = SQLITE_OK;             /* Return code */
          616  +  int rc;                         /* Return code */
   609    617   
   610         -  nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
   611         -  pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
   612         -  if( pCsr ){
   613         -    Fts5Global *pGlobal = pTab->pGlobal;
   614         -    memset(pCsr, 0, nByte);
   615         -    pCsr->aColumnSize = (int*)&pCsr[1];
   616         -    pCsr->pNext = pGlobal->pCsr;
   617         -    pGlobal->pCsr = pCsr;
   618         -    pCsr->iCsrId = ++pGlobal->iNextId;
   619         -  }else{
   620         -    rc = SQLITE_NOMEM;
          618  +  rc = fts5NewTransaction(pTab);
          619  +  if( rc==SQLITE_OK ){
          620  +    nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
          621  +    pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
          622  +    if( pCsr ){
          623  +      Fts5Global *pGlobal = pTab->pGlobal;
          624  +      memset(pCsr, 0, nByte);
          625  +      pCsr->aColumnSize = (int*)&pCsr[1];
          626  +      pCsr->pNext = pGlobal->pCsr;
          627  +      pGlobal->pCsr = pCsr;
          628  +      pCsr->iCsrId = ++pGlobal->iNextId;
          629  +    }else{
          630  +      rc = SQLITE_NOMEM;
          631  +    }
   621    632     }
   622    633     *ppCsr = (sqlite3_vtab_cursor*)pCsr;
   623    634     return rc;
   624    635   }
   625    636   
   626    637   static int fts5StmtType(Fts5Cursor *pCsr){
   627    638     if( pCsr->ePlan==FTS5_PLAN_SCAN ){
................................................................................
   774    785     assert( *pbSkip==0 );
   775    786     if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){
   776    787       Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
   777    788       int bDesc = pCsr->bDesc;
   778    789       i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
   779    790   
   780    791       rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc);
   781         -    if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
          792  +    if( rc==SQLITE_OK &&  iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
   782    793         *pbSkip = 1;
   783    794       }
   784    795   
   785    796       CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK);
   786    797       fts5CsrNewrow(pCsr);
   787    798       if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
   788    799         CsrFlagSet(pCsr, FTS5CSR_EOF);
          800  +      *pbSkip = 1;
   789    801       }
   790    802     }
   791    803     return rc;
   792    804   }
   793    805   
   794    806   
   795    807   /*
................................................................................
   798    810   **
   799    811   ** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
   800    812   ** even if we reach end-of-file.  The fts5EofMethod() will be called
   801    813   ** subsequently to determine whether or not an EOF was hit.
   802    814   */
   803    815   static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   804    816     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   805         -  int rc = SQLITE_OK;
          817  +  int rc;
   806    818   
   807    819     assert( (pCsr->ePlan<3)==
   808    820             (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
   809    821     );
          822  +  assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );
   810    823   
   811    824     if( pCsr->ePlan<3 ){
   812    825       int bSkip = 0;
   813    826       if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
   814    827       rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
   815         -    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
   816         -      CsrFlagSet(pCsr, FTS5CSR_EOF);
   817         -    }
          828  +    CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr));
   818    829       fts5CsrNewrow(pCsr);
   819    830     }else{
   820    831       switch( pCsr->ePlan ){
   821    832         case FTS5_PLAN_SPECIAL: {
   822    833           CsrFlagSet(pCsr, FTS5CSR_EOF);
          834  +        rc = SQLITE_OK;
   823    835           break;
   824    836         }
   825    837     
   826    838         case FTS5_PLAN_SORTED_MATCH: {
   827    839           rc = fts5SorterNext(pCsr);
   828    840           break;
   829    841         }
................................................................................
  1091   1103   **   1. Full-text search using a MATCH operator.
  1092   1104   **   2. A by-rowid lookup.
  1093   1105   **   3. A full-table scan.
  1094   1106   */
  1095   1107   static int fts5FilterMethod(
  1096   1108     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  1097   1109     int idxNum,                     /* Strategy index */
  1098         -  const char *idxStr,             /* Unused */
         1110  +  const char *zUnused,            /* Unused */
  1099   1111     int nVal,                       /* Number of elements in apVal */
  1100   1112     sqlite3_value **apVal           /* Arguments for the indexing scheme */
  1101   1113   ){
  1102   1114     Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  1103   1115     Fts5Config *pConfig = pTab->pConfig;
  1104   1116     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  1105   1117     int rc = SQLITE_OK;             /* Error code */
................................................................................
  1108   1120     int bOrderByRank;               /* True if ORDER BY rank */
  1109   1121     sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  1110   1122     sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  1111   1123     sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  1112   1124     sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  1113   1125     sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  1114   1126     char **pzErrmsg = pConfig->pzErrmsg;
         1127  +
         1128  +  UNUSED_PARAM(zUnused);
         1129  +  UNUSED_PARAM(nVal);
  1115   1130   
  1116   1131     if( pCsr->ePlan ){
  1117   1132       fts5FreeCursorComponents(pCsr);
  1118   1133       memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
  1119   1134     }
  1120   1135   
  1121   1136     assert( pCsr->pStmt==0 );
................................................................................
  1393   1408       }
  1394   1409     }
  1395   1410     return rc;
  1396   1411   }
  1397   1412   
  1398   1413   static int fts5SpecialDelete(
  1399   1414     Fts5Table *pTab, 
  1400         -  sqlite3_value **apVal, 
  1401         -  sqlite3_int64 *piRowid
         1415  +  sqlite3_value **apVal
  1402   1416   ){
  1403   1417     int rc = SQLITE_OK;
  1404   1418     int eType1 = sqlite3_value_type(apVal[1]);
  1405   1419     if( eType1==SQLITE_INTEGER ){
  1406   1420       sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
  1407   1421       rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]);
  1408   1422     }
................................................................................
  1470   1484      && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL 
  1471   1485     ){
  1472   1486       /* A "special" INSERT op. These are handled separately. */
  1473   1487       const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]);
  1474   1488       if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
  1475   1489         && 0==sqlite3_stricmp("delete", z) 
  1476   1490       ){
  1477         -      rc = fts5SpecialDelete(pTab, apVal, pRowid);
         1491  +      rc = fts5SpecialDelete(pTab, apVal);
  1478   1492       }else{
  1479   1493         rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
  1480   1494       }
  1481   1495     }else{
  1482   1496       /* A regular INSERT, UPDATE or DELETE statement. The trick here is that
  1483   1497       ** any conflict on the rowid value must be detected before any 
  1484   1498       ** modifications are made to the database file. There are 4 cases:
................................................................................
  1504   1518         pTab->base.zErrMsg = sqlite3_mprintf(
  1505   1519             "cannot %s contentless fts5 table: %s", 
  1506   1520             (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
  1507   1521         );
  1508   1522         rc = SQLITE_ERROR;
  1509   1523       }
  1510   1524   
  1511         -    /* Case 1: DELETE */
         1525  +    /* DELETE */
  1512   1526       else if( nArg==1 ){
  1513   1527         i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
  1514   1528         rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
  1515   1529       }
  1516   1530   
  1517         -    /* Case 2: INSERT */
         1531  +    /* INSERT */
  1518   1532       else if( eType0!=SQLITE_INTEGER ){     
  1519   1533         /* If this is a REPLACE, first remove the current entry (if any) */
  1520   1534         if( eConflict==SQLITE_REPLACE 
  1521   1535          && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
  1522   1536         ){
  1523   1537           i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
  1524   1538           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
  1525   1539         }
  1526   1540         fts5StorageInsert(&rc, pTab, apVal, pRowid);
  1527   1541       }
  1528   1542   
  1529         -    /* Case 2: UPDATE */
         1543  +    /* UPDATE */
  1530   1544       else{
  1531   1545         i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
  1532   1546         i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
  1533   1547         if( iOld!=iNew ){
  1534   1548           if( eConflict==SQLITE_REPLACE ){
  1535   1549             rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
  1536   1550             if( rc==SQLITE_OK ){
................................................................................
  1572   1586   }
  1573   1587   
  1574   1588   /*
  1575   1589   ** Implementation of xBegin() method. 
  1576   1590   */
  1577   1591   static int fts5BeginMethod(sqlite3_vtab *pVtab){
  1578   1592     fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
         1593  +  fts5NewTransaction((Fts5Table*)pVtab);
  1579   1594     return SQLITE_OK;
  1580   1595   }
  1581   1596   
  1582   1597   /*
  1583   1598   ** Implementation of xCommit() method. This is a no-op. The contents of
  1584   1599   ** the pending-terms hash-table have already been flushed into the database
  1585   1600   ** by fts5SyncMethod().
  1586   1601   */
  1587   1602   static int fts5CommitMethod(sqlite3_vtab *pVtab){
         1603  +  UNUSED_PARAM(pVtab);  /* Call below is a no-op for NDEBUG builds */
  1588   1604     fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0);
  1589   1605     return SQLITE_OK;
  1590   1606   }
  1591   1607   
  1592   1608   /*
  1593   1609   ** Implementation of xRollback(). Discard the contents of the pending-terms
  1594   1610   ** hash-table. Any changes made to the database are reverted by SQLite.
................................................................................
  1834   1850   static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  1835   1851     return fts5CursorRowid((Fts5Cursor*)pCtx);
  1836   1852   }
  1837   1853   
  1838   1854   static int fts5ColumnSizeCb(
  1839   1855     void *pContext,                 /* Pointer to int */
  1840   1856     int tflags,
  1841         -  const char *pToken,             /* Buffer containing token */
  1842         -  int nToken,                     /* Size of token in bytes */
  1843         -  int iStart,                     /* Start offset of token */
  1844         -  int iEnd                        /* End offset of token */
         1857  +  const char *pUnused,            /* Buffer containing token */
         1858  +  int nUnused,                    /* Size of token in bytes */
         1859  +  int iUnused1,                   /* Start offset of token */
         1860  +  int iUnused2                    /* End offset of token */
  1845   1861   ){
  1846   1862     int *pCnt = (int*)pContext;
         1863  +  UNUSED_PARAM2(pUnused, nUnused);
         1864  +  UNUSED_PARAM2(iUnused1, iUnused2);
  1847   1865     if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
  1848   1866       (*pCnt)++;
  1849   1867     }
  1850   1868     return SQLITE_OK;
  1851   1869   }
  1852   1870   
  1853   1871   static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
................................................................................
  1955   1973       }
  1956   1974     }
  1957   1975   
  1958   1976     return pRet;
  1959   1977   }
  1960   1978   
  1961   1979   static void fts5ApiPhraseNext(
  1962         -  Fts5Context *pCtx, 
         1980  +  Fts5Context *pUnused, 
  1963   1981     Fts5PhraseIter *pIter, 
  1964   1982     int *piCol, int *piOff
  1965   1983   ){
         1984  +  UNUSED_PARAM(pUnused);
  1966   1985     if( pIter->a>=pIter->b ){
  1967   1986       *piCol = -1;
  1968   1987       *piOff = -1;
  1969   1988     }else{
  1970   1989       int iVal;
  1971   1990       pIter->a += fts5GetVarint32(pIter->a, iVal);
  1972   1991       if( iVal==1 ){
................................................................................
  2110   2129     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  2111   2130     Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  2112   2131     int rc;
  2113   2132     Fts5Cursor *pNew = 0;
  2114   2133   
  2115   2134     rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew);
  2116   2135     if( rc==SQLITE_OK ){
  2117         -    Fts5Config *pConf = pTab->pConfig;
  2118   2136       pNew->ePlan = FTS5_PLAN_MATCH;
  2119   2137       pNew->iFirstRowid = SMALLEST_INT64;
  2120   2138       pNew->iLastRowid = LARGEST_INT64;
  2121   2139       pNew->base.pVtab = (sqlite3_vtab*)pTab;
  2122         -    rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
         2140  +    rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr);
  2123   2141     }
  2124   2142   
  2125   2143     if( rc==SQLITE_OK ){
  2126   2144       for(rc = fts5CursorFirst(pTab, pNew, 0);
  2127   2145           rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0;
  2128   2146           rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew)
  2129   2147       ){
................................................................................
  2328   2346   
  2329   2347   /*
  2330   2348   ** This routine implements the xFindFunction method for the FTS3
  2331   2349   ** virtual table.
  2332   2350   */
  2333   2351   static int fts5FindFunctionMethod(
  2334   2352     sqlite3_vtab *pVtab,            /* Virtual table handle */
  2335         -  int nArg,                       /* Number of SQL function arguments */
         2353  +  int nUnused,                    /* Number of SQL function arguments */
  2336   2354     const char *zName,              /* Name of SQL function */
  2337   2355     void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
  2338   2356     void **ppArg                    /* OUT: User data for *pxFunc */
  2339   2357   ){
  2340   2358     Fts5Table *pTab = (Fts5Table*)pVtab;
  2341   2359     Fts5Auxiliary *pAux;
  2342   2360   
         2361  +  UNUSED_PARAM(nUnused);
  2343   2362     pAux = fts5FindAuxiliary(pTab, zName);
  2344   2363     if( pAux ){
  2345   2364       *pxFunc = fts5ApiCallback;
  2346   2365       *ppArg = (void*)pAux;
  2347   2366       return 1;
  2348   2367     }
  2349   2368   
................................................................................
  2365   2384   /*
  2366   2385   ** The xSavepoint() method.
  2367   2386   **
  2368   2387   ** Flush the contents of the pending-terms table to disk.
  2369   2388   */
  2370   2389   static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2371   2390     Fts5Table *pTab = (Fts5Table*)pVtab;
         2391  +  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  2372   2392     fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
  2373   2393     fts5TripCursors(pTab);
  2374   2394     return sqlite3Fts5StorageSync(pTab->pStorage, 0);
  2375   2395   }
  2376   2396   
  2377   2397   /*
  2378   2398   ** The xRelease() method.
  2379   2399   **
  2380   2400   ** This is a no-op.
  2381   2401   */
  2382   2402   static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2383   2403     Fts5Table *pTab = (Fts5Table*)pVtab;
         2404  +  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  2384   2405     fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
  2385   2406     fts5TripCursors(pTab);
  2386   2407     return sqlite3Fts5StorageSync(pTab->pStorage, 0);
  2387   2408   }
  2388   2409   
  2389   2410   /*
  2390   2411   ** The xRollbackTo() method.
  2391   2412   **
  2392   2413   ** Discard the contents of the pending terms table.
  2393   2414   */
  2394   2415   static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  2395   2416     Fts5Table *pTab = (Fts5Table*)pVtab;
         2417  +  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
  2396   2418     fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
  2397   2419     fts5TripCursors(pTab);
  2398   2420     return sqlite3Fts5StorageRollback(pTab->pStorage);
  2399   2421   }
  2400   2422   
  2401   2423   /*
  2402   2424   ** Register a new auxiliary function with global context pGlobal.
................................................................................
  2568   2590   
  2569   2591     sqlite3_free(pGlobal);
  2570   2592   }
  2571   2593   
  2572   2594   static void fts5Fts5Func(
  2573   2595     sqlite3_context *pCtx,          /* Function call context */
  2574   2596     int nArg,                       /* Number of args */
  2575         -  sqlite3_value **apVal           /* Function arguments */
         2597  +  sqlite3_value **apUnused        /* Function arguments */
  2576   2598   ){
  2577   2599     Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
  2578   2600     char buf[8];
         2601  +  UNUSED_PARAM2(nArg, apUnused);
  2579   2602     assert( nArg==0 );
  2580   2603     assert( sizeof(buf)>=sizeof(pGlobal) );
  2581   2604     memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
  2582   2605     sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
  2583   2606   }
  2584   2607   
  2585   2608   /*
  2586   2609   ** Implementation of fts5_source_id() function.
  2587   2610   */
  2588   2611   static void fts5SourceIdFunc(
  2589   2612     sqlite3_context *pCtx,          /* Function call context */
  2590   2613     int nArg,                       /* Number of args */
  2591         -  sqlite3_value **apVal           /* Function arguments */
         2614  +  sqlite3_value **apUnused        /* Function arguments */
  2592   2615   ){
  2593   2616     assert( nArg==0 );
         2617  +  UNUSED_PARAM2(nArg, apUnused);
  2594   2618     sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
  2595   2619   }
  2596   2620   
  2597   2621   static int fts5Init(sqlite3 *db){
  2598   2622     static const sqlite3_module fts5Mod = {
  2599   2623       /* iVersion      */ 2,
  2600   2624       /* xCreate       */ fts5CreateMethod,
................................................................................
  2648   2672       }
  2649   2673       if( rc==SQLITE_OK ){
  2650   2674         rc = sqlite3_create_function(
  2651   2675             db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0
  2652   2676         );
  2653   2677       }
  2654   2678     }
         2679  +
         2680  +  /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file
         2681  +  ** fts5_test_mi.c is compiled and linked into the executable. And call
         2682  +  ** its entry point to enable the matchinfo() demo.  */
         2683  +#ifdef SQLITE_FTS5_ENABLE_TEST_MI
         2684  +  if( rc==SQLITE_OK ){
         2685  +    extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);
         2686  +    rc = sqlite3Fts5TestRegisterMatchinfo(db);
         2687  +  }
         2688  +#endif
         2689  +
  2655   2690     return rc;
  2656   2691   }
  2657   2692   
  2658   2693   /*
  2659   2694   ** The following functions are used to register the module with SQLite. If
  2660   2695   ** this module is being built as part of the SQLite core (SQLITE_CORE is
  2661   2696   ** defined), then sqlite3_open() will call sqlite3Fts5Init() directly.

Changes to ext/fts5/fts5_storage.c.

   141    141         if( rc!=SQLITE_OK && pzErrMsg ){
   142    142           *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
   143    143         }
   144    144       }
   145    145     }
   146    146   
   147    147     *ppStmt = p->aStmt[eStmt];
          148  +  sqlite3_reset(*ppStmt);
   148    149     return rc;
   149    150   }
   150    151   
   151    152   
   152    153   static int fts5ExecPrintf(
   153    154     sqlite3 *db,
   154    155     char **pzErr,
................................................................................
   334    335   */
   335    336   int sqlite3Fts5StorageClose(Fts5Storage *p){
   336    337     int rc = SQLITE_OK;
   337    338     if( p ){
   338    339       int i;
   339    340   
   340    341       /* Finalize all SQL statements */
   341         -    for(i=0; i<(int)ArraySize(p->aStmt); i++){
          342  +    for(i=0; i<ArraySize(p->aStmt); i++){
   342    343         sqlite3_finalize(p->aStmt[i]);
   343    344       }
   344    345   
   345    346       sqlite3_free(p);
   346    347     }
   347    348     return rc;
   348    349   }
................................................................................
   358    359   ** Tokenization callback used when inserting tokens into the FTS index.
   359    360   */
   360    361   static int fts5StorageInsertCallback(
   361    362     void *pContext,                 /* Pointer to Fts5InsertCtx object */
   362    363     int tflags,
   363    364     const char *pToken,             /* Buffer containing token */
   364    365     int nToken,                     /* Size of token in bytes */
   365         -  int iStart,                     /* Start offset of token */
   366         -  int iEnd                        /* End offset of token */
          366  +  int iUnused1,                   /* Start offset of token */
          367  +  int iUnused2                    /* End offset of token */
   367    368   ){
   368    369     Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
   369    370     Fts5Index *pIdx = pCtx->pStorage->pIndex;
          371  +  UNUSED_PARAM2(iUnused1, iUnused2);
          372  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
   370    373     if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
   371    374       pCtx->szCol++;
   372    375     }
   373    376     return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
   374    377   }
   375    378   
   376    379   /*
................................................................................
   633    636   int sqlite3Fts5StorageOptimize(Fts5Storage *p){
   634    637     return sqlite3Fts5IndexOptimize(p->pIndex);
   635    638   }
   636    639   
   637    640   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
   638    641     return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
   639    642   }
          643  +
          644  +int sqlite3Fts5StorageReset(Fts5Storage *p){
          645  +  return sqlite3Fts5IndexReset(p->pIndex);
          646  +}
   640    647   
   641    648   /*
   642    649   ** Allocate a new rowid. This is used for "external content" tables when
   643    650   ** a NULL value is inserted into the rowid column. The new rowid is allocated
   644    651   ** by inserting a dummy row into the %_docsize table. The dummy will be
   645    652   ** overwritten later.
   646    653   **
................................................................................
   793    800   ** Tokenization callback used by integrity check.
   794    801   */
   795    802   static int fts5StorageIntegrityCallback(
   796    803     void *pContext,                 /* Pointer to Fts5IntegrityCtx object */
   797    804     int tflags,
   798    805     const char *pToken,             /* Buffer containing token */
   799    806     int nToken,                     /* Size of token in bytes */
   800         -  int iStart,                     /* Start offset of token */
   801         -  int iEnd                        /* End offset of token */
          807  +  int iUnused1,                   /* Start offset of token */
          808  +  int iUnused2                    /* End offset of token */
   802    809   ){
   803    810     Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
   804    811     Fts5Termset *pTermset = pCtx->pTermset;
   805    812     int bPresent;
   806    813     int ii;
   807    814     int rc = SQLITE_OK;
   808    815     int iPos;
   809    816     int iCol;
   810    817   
          818  +  UNUSED_PARAM2(iUnused1, iUnused2);
          819  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
          820  +
   811    821     if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
   812    822       pCtx->szCol++;
   813    823     }
   814    824   
   815    825     switch( pCtx->pConfig->eDetail ){
   816    826       case FTS5_DETAIL_FULL:
   817    827         iPos = pCtx->szCol-1;
................................................................................
  1114   1124       rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
  1115   1125       if( rc==SQLITE_OK ){
  1116   1126         p->pConfig->iCookie = iNew;
  1117   1127       }
  1118   1128     }
  1119   1129     return rc;
  1120   1130   }
  1121         -
  1122         -

Changes to ext/fts5/fts5_test_mi.c.

    37     37   ** This file exports a single function that may be called to register the
    38     38   ** matchinfo() implementation with a database handle:
    39     39   **
    40     40   **   int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db);
    41     41   */
    42     42   
    43     43   
    44         -#ifdef SQLITE_TEST
    45     44   #ifdef SQLITE_ENABLE_FTS5
    46     45   
    47     46   #include "fts5.h"
    48         -#include <tcl.h>
    49     47   #include <assert.h>
    50     48   #include <string.h>
    51     49   
    52     50   typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
           51  +
           52  +#ifndef SQLITE_AMALGAMATION
    53     53   typedef unsigned int u32;
           54  +#endif
    54     55   
    55     56   struct Fts5MatchinfoCtx {
    56     57     int nCol;                       /* Number of cols in FTS5 table */
    57     58     int nPhrase;                    /* Number of phrases in FTS5 query */
    58     59     char *zArg;                     /* nul-term'd copy of 2nd arg */
    59     60     int nRet;                       /* Number of elements in aRet[] */
    60     61     u32 *aRet;                      /* Array of 32-bit unsigned ints to return */
................................................................................
    63     64   
    64     65   
    65     66   /*
    66     67   ** Return a pointer to the fts5_api pointer for database connection db.
    67     68   ** If an error occurs, return NULL and leave an error in the database 
    68     69   ** handle (accessible using sqlite3_errcode()/errmsg()).
    69     70   */
    70         -static fts5_api *fts5_api_from_db(sqlite3 *db){
    71         -  fts5_api *pRet = 0;
           71  +static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
    72     72     sqlite3_stmt *pStmt = 0;
           73  +  int rc;
    73     74   
    74         -  if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
    75         -   && SQLITE_ROW==sqlite3_step(pStmt) 
    76         -   && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
    77         -  ){
    78         -    memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
           75  +  *ppApi = 0;
           76  +  rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
           77  +  if( rc==SQLITE_OK ){
           78  +    if( SQLITE_ROW==sqlite3_step(pStmt) 
           79  +        && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
           80  +      ){
           81  +      memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
           82  +    }
           83  +    rc = sqlite3_finalize(pStmt);
    79     84     }
    80         -  sqlite3_finalize(pStmt);
    81         -  return pRet;
           85  +
           86  +  return rc;
    82     87   }
    83     88   
    84     89   
    85     90   /*
    86     91   ** Argument f should be a flag accepted by matchinfo() (a valid character
    87     92   ** in the string passed as the second argument). If it is not, -1 is 
    88     93   ** returned. Otherwise, if f is a valid matchinfo flag, the value returned
................................................................................
   240    245         for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
   241    246           Fts5PhraseIter iter;
   242    247           int iOff, iCol;
   243    248           for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); 
   244    249               iOff>=0; 
   245    250               pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
   246    251           ){
   247         -          if( f=='b' ){
   248         -            aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
   249         -          }else{
   250         -            aOut[nMul * (iCol + iPhrase * p->nCol)]++;
   251         -          }
          252  +          aOut[nMul * (iCol + iPhrase * p->nCol)]++;
   252    253           }
   253    254         }
   254    255   
   255    256         break;
   256    257       }
   257    258   
   258    259       case 'l': {
................................................................................
   398    399   int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
   399    400     int rc;                         /* Return code */
   400    401     fts5_api *pApi;                 /* FTS5 API functions */
   401    402   
   402    403     /* Extract the FTS5 API pointer from the database handle. The 
   403    404     ** fts5_api_from_db() function above is copied verbatim from the 
   404    405     ** FTS5 documentation. Refer there for details. */
   405         -  pApi = fts5_api_from_db(db);
          406  +  rc = fts5_api_from_db(db, &pApi);
          407  +  if( rc!=SQLITE_OK ) return rc;
   406    408   
   407    409     /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
   408    410     ** with this database handle, or an error (OOM perhaps?) has occurred.
   409    411     **
   410    412     ** Also check that the fts5_api object is version 2 or newer.  
   411    413     */ 
   412         -  if( pApi==0 || pApi->iVersion<1 ){
          414  +  if( pApi==0 || pApi->iVersion<2 ){
   413    415       return SQLITE_ERROR;
   414    416     }
   415    417   
   416    418     /* Register the implementation of matchinfo() */
   417    419     rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
   418    420   
   419    421     return rc;
   420    422   }
   421    423   
   422    424   #endif /* SQLITE_ENABLE_FTS5 */
   423         -#endif /* SQLITE_TEST */
   424    425   

Changes to ext/fts5/fts5_tokenize.c.

    58     58     sqlite3_free(p);
    59     59   }
    60     60   
    61     61   /*
    62     62   ** Create an "ascii" tokenizer.
    63     63   */
    64     64   static int fts5AsciiCreate(
    65         -  void *pCtx, 
           65  +  void *pUnused, 
    66     66     const char **azArg, int nArg,
    67     67     Fts5Tokenizer **ppOut
    68     68   ){
    69     69     int rc = SQLITE_OK;
    70     70     AsciiTokenizer *p = 0;
           71  +  UNUSED_PARAM(pUnused);
    71     72     if( nArg%2 ){
    72     73       rc = SQLITE_ERROR;
    73     74     }else{
    74     75       p = sqlite3_malloc(sizeof(AsciiTokenizer));
    75     76       if( p==0 ){
    76     77         rc = SQLITE_NOMEM;
    77     78       }else{
................................................................................
   112    113   
   113    114   /*
   114    115   ** Tokenize some text using the ascii tokenizer.
   115    116   */
   116    117   static int fts5AsciiTokenize(
   117    118     Fts5Tokenizer *pTokenizer,
   118    119     void *pCtx,
   119         -  int flags,
          120  +  int iUnused,
   120    121     const char *pText, int nText,
   121    122     int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
   122    123   ){
   123    124     AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
   124    125     int rc = SQLITE_OK;
   125    126     int ie;
   126    127     int is = 0;
   127    128   
   128    129     char aFold[64];
   129    130     int nFold = sizeof(aFold);
   130    131     char *pFold = aFold;
   131    132     unsigned char *a = p->aTokenChar;
          133  +
          134  +  UNUSED_PARAM(iUnused);
   132    135   
   133    136     while( is<nText && rc==SQLITE_OK ){
   134    137       int nByte;
   135    138   
   136    139       /* Skip any leading divider characters. */
   137    140       while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){
   138    141         is++;
................................................................................
   319    322     return;
   320    323   }
   321    324   
   322    325   /*
   323    326   ** Create a "unicode61" tokenizer.
   324    327   */
   325    328   static int fts5UnicodeCreate(
   326         -  void *pCtx, 
          329  +  void *pUnused, 
   327    330     const char **azArg, int nArg,
   328    331     Fts5Tokenizer **ppOut
   329    332   ){
   330    333     int rc = SQLITE_OK;             /* Return code */
   331    334     Unicode61Tokenizer *p = 0;      /* New tokenizer object */ 
          335  +
          336  +  UNUSED_PARAM(pUnused);
   332    337   
   333    338     if( nArg%2 ){
   334    339       rc = SQLITE_ERROR;
   335    340     }else{
   336    341       p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
   337    342       if( p ){
   338    343         int i;
................................................................................
   382    387     assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
   383    388     return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
   384    389   }
   385    390   
   386    391   static int fts5UnicodeTokenize(
   387    392     Fts5Tokenizer *pTokenizer,
   388    393     void *pCtx,
   389         -  int flags,
          394  +  int iUnused,
   390    395     const char *pText, int nText,
   391    396     int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
   392    397   ){
   393    398     Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
   394    399     int rc = SQLITE_OK;
   395    400     unsigned char *a = p->aTokenChar;
   396    401   
................................................................................
   397    402     unsigned char *zTerm = (unsigned char*)&pText[nText];
   398    403     unsigned char *zCsr = (unsigned char *)pText;
   399    404   
   400    405     /* Output buffer */
   401    406     char *aFold = p->aFold;
   402    407     int nFold = p->nFold;
   403    408     const char *pEnd = &aFold[nFold-6];
          409  +
          410  +  UNUSED_PARAM(iUnused);
   404    411   
   405    412     /* Each iteration of this loop gobbles up a contiguous run of separators,
   406    413     ** then the next token.  */
   407    414     while( rc==SQLITE_OK ){
   408    415       int iCode;                    /* non-ASCII codepoint read from input */
   409    416       char *zOut = aFold;
   410    417       int is;
................................................................................
  1216   1223       { "ascii",     {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
  1217   1224       { "porter",    {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
  1218   1225     };
  1219   1226     
  1220   1227     int rc = SQLITE_OK;             /* Return code */
  1221   1228     int i;                          /* To iterate through builtin functions */
  1222   1229   
  1223         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
         1230  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
  1224   1231       rc = pApi->xCreateTokenizer(pApi,
  1225   1232           aBuiltin[i].zName,
  1226   1233           (void*)pApi,
  1227   1234           &aBuiltin[i].x,
  1228   1235           0
  1229   1236       );
  1230   1237     }
  1231   1238   
  1232   1239     return rc;
  1233   1240   }
  1234   1241   
  1235   1242   

Changes to ext/fts5/fts5_unicode2.c.

   121    121       0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
   122    122       0x380400F0,
   123    123     };
   124    124     static const unsigned int aAscii[4] = {
   125    125       0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
   126    126     };
   127    127   
   128         -  if( c<128 ){
          128  +  if( (unsigned int)c<128 ){
   129    129       return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   130         -  }else if( c<(1<<22) ){
          130  +  }else if( (unsigned int)c<(1<<22) ){
   131    131       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   132    132       int iRes = 0;
   133    133       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   134    134       int iLo = 0;
   135    135       while( iHi>=iLo ){
   136    136         int iTest = (iHi + iLo) / 2;
   137    137         if( key >= aEntry[iTest] ){

Changes to ext/fts5/fts5_varint.c.

   329    329       return 2;
   330    330     }
   331    331     return fts5PutVarint64(p,v);
   332    332   }
   333    333   
   334    334   
   335    335   int sqlite3Fts5GetVarintLen(u32 iVal){
          336  +#if 0
   336    337     if( iVal<(1 << 7 ) ) return 1;
          338  +#endif
          339  +  assert( iVal>=(1 << 7) );
   337    340     if( iVal<(1 << 14) ) return 2;
   338    341     if( iVal<(1 << 21) ) return 3;
   339    342     if( iVal<(1 << 28) ) return 4;
   340    343     return 5;
   341    344   }
   342    345   

Changes to ext/fts5/fts5_vocab.c.

   180    180       const char *zType = bDb ? argv[5] : argv[4];
   181    181       int nDb = (int)strlen(zDb)+1; 
   182    182       int nTab = (int)strlen(zTab)+1;
   183    183       int eType = 0;
   184    184       
   185    185       rc = fts5VocabTableType(zType, pzErr, &eType);
   186    186       if( rc==SQLITE_OK ){
   187         -      assert( eType>=0 && eType<sizeof(azSchema)/sizeof(azSchema[0]) );
          187  +      assert( eType>=0 && eType<ArraySize(azSchema) );
   188    188         rc = sqlite3_declare_vtab(db, azSchema[eType]);
   189    189       }
   190    190   
   191    191       nByte = sizeof(Fts5VocabTable) + nDb + nTab;
   192    192       pRet = sqlite3Fts5MallocZero(&rc, nByte);
   193    193       if( pRet ){
   194    194         pRet->pGlobal = (Fts5Global*)pAux;
................................................................................
   233    233     return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
   234    234   }
   235    235   
   236    236   /* 
   237    237   ** Implementation of the xBestIndex method.
   238    238   */
   239    239   static int fts5VocabBestIndexMethod(
   240         -  sqlite3_vtab *pVTab, 
          240  +  sqlite3_vtab *pUnused,
   241    241     sqlite3_index_info *pInfo
   242    242   ){
   243    243     int i;
   244    244     int iTermEq = -1;
   245    245     int iTermGe = -1;
   246    246     int iTermLe = -1;
   247    247     int idxNum = 0;
   248    248     int nArg = 0;
          249  +
          250  +  UNUSED_PARAM(pUnused);
   249    251   
   250    252     for(i=0; i<pInfo->nConstraint; i++){
   251    253       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   252    254       if( p->usable==0 ) continue;
   253    255       if( p->iColumn==0 ){          /* term column */
   254    256         if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i;
   255    257         if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i;
................................................................................
   403    405         sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
   404    406         memset(pCsr->aCnt, 0, nCol * sizeof(i64));
   405    407         memset(pCsr->aDoc, 0, nCol * sizeof(i64));
   406    408         pCsr->iCol = 0;
   407    409   
   408    410         assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
   409    411         while( rc==SQLITE_OK ){
   410         -        i64 dummy;
   411    412           const u8 *pPos; int nPos;   /* Position list */
   412    413           i64 iPos = 0;               /* 64-bit position read from poslist */
   413    414           int iOff = 0;               /* Current offset within position list */
   414    415   
          416  +        pPos = pCsr->pIter->pData;
          417  +        nPos = pCsr->pIter->nData;
   415    418           switch( pCsr->pConfig->eDetail ){
   416    419             case FTS5_DETAIL_FULL:
   417         -            rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
   418         -            if( rc==SQLITE_OK ){
   419         -              if( pTab->eType==FTS5_VOCAB_ROW ){
   420         -                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   421         -                  pCsr->aCnt[0]++;
   422         -                }
   423         -                pCsr->aDoc[0]++;
   424         -              }else{
   425         -                int iCol = -1;
   426         -                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   427         -                  int ii = FTS5_POS2COLUMN(iPos);
   428         -                  pCsr->aCnt[ii]++;
   429         -                  if( iCol!=ii ){
   430         -                    if( ii>=nCol ){
   431         -                      rc = FTS5_CORRUPT;
   432         -                      break;
   433         -                    }
   434         -                    pCsr->aDoc[ii]++;
   435         -                    iCol = ii;
          420  +            pPos = pCsr->pIter->pData;
          421  +            nPos = pCsr->pIter->nData;
          422  +            if( pTab->eType==FTS5_VOCAB_ROW ){
          423  +              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
          424  +                pCsr->aCnt[0]++;
          425  +              }
          426  +              pCsr->aDoc[0]++;
          427  +            }else{
          428  +              int iCol = -1;
          429  +              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
          430  +                int ii = FTS5_POS2COLUMN(iPos);
          431  +                pCsr->aCnt[ii]++;
          432  +                if( iCol!=ii ){
          433  +                  if( ii>=nCol ){
          434  +                    rc = FTS5_CORRUPT;
          435  +                    break;
   436    436                     }
          437  +                  pCsr->aDoc[ii]++;
          438  +                  iCol = ii;
   437    439                   }
   438    440                 }
   439    441               }
   440    442               break;
   441    443   
   442    444             case FTS5_DETAIL_COLUMNS:
   443    445               if( pTab->eType==FTS5_VOCAB_ROW ){
   444    446                 pCsr->aDoc[0]++;
   445    447               }else{
   446         -              Fts5Buffer buf = {0, 0, 0};
   447         -              rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
   448         -              if( rc==SQLITE_OK ){
   449         -                while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
   450         -                  assert_nc( iPos>=0 && iPos<nCol );
   451         -                  if( iPos>=nCol ){
   452         -                    rc = FTS5_CORRUPT;
   453         -                    break;
   454         -                  }
   455         -                  pCsr->aDoc[iPos]++;
          448  +              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
          449  +                assert_nc( iPos>=0 && iPos<nCol );
          450  +                if( iPos>=nCol ){
          451  +                  rc = FTS5_CORRUPT;
          452  +                  break;
   456    453                   }
          454  +                pCsr->aDoc[iPos]++;
   457    455                 }
   458         -              sqlite3Fts5BufferFree(&buf);
   459    456               }
   460    457               break;
   461    458   
   462    459             default: 
   463    460               assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
   464    461               pCsr->aDoc[0]++;
   465    462               break;
................................................................................
   489    486   
   490    487   /*
   491    488   ** This is the xFilter implementation for the virtual table.
   492    489   */
   493    490   static int fts5VocabFilterMethod(
   494    491     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
   495    492     int idxNum,                     /* Strategy index */
   496         -  const char *idxStr,             /* Unused */
   497         -  int nVal,                       /* Number of elements in apVal */
          493  +  const char *zUnused,            /* Unused */
          494  +  int nUnused,                    /* Number of elements in apVal */
   498    495     sqlite3_value **apVal           /* Arguments for the indexing scheme */
   499    496   ){
   500    497     Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
   501    498     int rc = SQLITE_OK;
   502    499   
   503    500     int iVal = 0;
   504    501     int f = FTS5INDEX_QUERY_SCAN;
   505    502     const char *zTerm = 0;
   506    503     int nTerm = 0;
   507    504   
   508    505     sqlite3_value *pEq = 0;
   509    506     sqlite3_value *pGe = 0;
   510    507     sqlite3_value *pLe = 0;
          508  +
          509  +  UNUSED_PARAM2(zUnused, nUnused);
   511    510   
   512    511     fts5VocabResetCursor(pCsr);
   513    512     if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
   514    513     if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
   515    514     if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
   516    515   
   517    516     if( pEq ){

Changes to ext/fts5/fts5parse.y.

    24     24   
    25     25   // The generated parser function takes a 4th argument as follows:
    26     26   %extra_argument {Fts5Parse *pParse}
    27     27   
    28     28   // This code runs whenever there is a syntax error
    29     29   //
    30     30   %syntax_error {
           31  +  UNUSED_PARAM(yymajor); /* Silence a compiler warning */
    31     32     sqlite3Fts5ParseError(
    32     33       pParse, "fts5: syntax error near \"%.*s\"",TOKEN.n,TOKEN.p
    33     34     );
    34     35   }
    35     36   %stack_overflow {
    36         -  assert( 0 );
           37  +  sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow");
    37     38   }
    38     39   
    39     40   // The name of the generated procedure that implements the parser
    40     41   // is as follows:
    41     42   %name sqlite3Fts5Parser
    42     43   
    43     44   // The following text is included near the beginning of the C source
................................................................................
    99    100   }
   100    101   
   101    102   expr(A) ::= LP expr(X) RP. {A = X;}
   102    103   expr(A) ::= exprlist(X).   {A = X;}
   103    104   
   104    105   exprlist(A) ::= cnearset(X). {A = X;}
   105    106   exprlist(A) ::= exprlist(X) cnearset(Y). {
   106         -  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
          107  +  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
   107    108   }
   108    109   
   109    110   cnearset(A) ::= nearset(X). { 
   110    111     A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
   111    112   }
   112    113   cnearset(A) ::= colset(X) COLON nearset(Y). { 
   113    114     sqlite3Fts5ParseSetColset(pParse, Y, X);

Changes to ext/fts5/test/fts5_common.tcl.

    12     12   
    13     13   if {![info exists testdir]} {
    14     14     set testdir [file join [file dirname [info script]] .. .. .. test]
    15     15   }
    16     16   source $testdir/tester.tcl
    17     17   
    18     18   ifcapable !fts5 {
    19         -  finish_test
           19  +  proc return_if_no_fts5 {} {
           20  +    finish_test
           21  +    return -code return
           22  +  }
    20     23     return
           24  +} else {
           25  +  proc return_if_no_fts5 {} {}
    21     26   }
    22     27   
    23     28   catch { 
    24     29     sqlite3_fts5_may_be_corrupt 0 
    25     30     reset_db
    26     31   }
    27     32   
    28         -# If SQLITE_ENABLE_FTS5 is not defined, skip this test.
    29         -ifcapable !fts5 {
    30         -  finish_test
    31         -  return
    32         -}
    33         -
    34     33   proc fts5_test_poslist {cmd} {
    35     34     set res [list]
    36     35     for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    37     36       lappend res [string map {{ } .} [$cmd xInst $i]]
    38     37     }
    39     38     set res
    40     39   }
................................................................................
    44     43   
    45     44     for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
    46     45       $cmd xPhraseForeach $i c o {
    47     46         lappend res $i.$c.$o
    48     47       }
    49     48     }
    50     49   
    51         -  set res
           50  +  #set res
           51  +  sort_poslist $res
    52     52   }
    53     53   
    54     54   proc fts5_test_collist {cmd} {
    55     55     set res [list]
    56     56   
    57     57     for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
    58     58       $cmd xPhraseColumnForeach $i c { lappend res $i.$c }
................................................................................
   154    154   
   155    155       fts5_test_queryphrase
   156    156       fts5_test_phrasecount
   157    157     } {
   158    158       sqlite3_fts5_create_function $db $f $f
   159    159     }
   160    160   }
          161  +
          162  +proc fts5_segcount {tbl} {
          163  +  set N 0
          164  +  foreach n [fts5_level_segs $tbl] { incr N $n }
          165  +  set N
          166  +}
   161    167   
   162    168   proc fts5_level_segs {tbl} {
   163    169     set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
   164    170     set ret [list]
   165    171     foreach L [lrange [db one $sql] 1 end] {
   166    172       lappend ret [expr [llength $L] - 3]
   167    173     }

Changes to ext/fts5/test/fts5aa.test.

   428    428     set fd [db incrblob main n1_data block 10]
   429    429     fconfigure $fd -encoding binary -translation binary
   430    430     puts -nonewline $fd "\x44\x45"
   431    431     close $fd
   432    432   }
   433    433   db func funk funk
   434    434   
          435  +# This test case corrupts the structure record within the first invocation
          436  +# of function funk(). Which used to cause the bm25() function to throw an
          437  +# exception. But since bm25() can now used the cached structure record,
          438  +# it never sees the corruption introduced by funk() and so the following 
          439  +# statement no longer fails.
          440  +#
   435    441   do_catchsql_test 16.2 {
   436    442     SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
   437         -} {1 {SQL logic error or missing database}}
          443  +} {0 {{} -1e-06 {}}}
          444  +# {1 {SQL logic error or missing database}}
   438    445   
   439    446   #-------------------------------------------------------------------------
   440    447   #
   441    448   reset_db
   442    449   do_execsql_test 17.1 {
   443    450     CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
   444    451     INSERT INTO b2 VALUES('a');

Changes to ext/fts5/test/fts5ac.test.

   154    154     do_execsql_test 1.$tn2.integrity {
   155    155       INSERT INTO xx(xx) VALUES('integrity-check');
   156    156     }
   157    157   
   158    158     #-------------------------------------------------------------------------
   159    159     #
   160    160     foreach {tn expr} {
   161         -    1.2 "a   OR b"
   162    161       1.1 "a   AND b"
          162  +    1.2 "a   OR b"
   163    163       1.3 "o"
   164    164       1.4 "b q"
   165    165       1.5 "e a e"
   166    166       1.6 "m d g q q b k b w f q q p p"
   167    167       1.7 "l o o l v v k"
   168    168       1.8 "a"
   169    169       1.9 "b"
................................................................................
   245    245       }
   246    246   
   247    247       set res [fts5_query_data $expr xx]
   248    248       do_execsql_test 1.$tn2.$tn.[llength $res].asc {
   249    249         SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
   250    250         FROM xx WHERE xx match $expr
   251    251       } $res
   252         -
   253    252   
   254    253       set res [fts5_query_data $expr xx DESC]
   255    254       do_execsql_test 1.$tn2.$tn.[llength $res].desc {
   256    255         SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
   257    256         FROM xx WHERE xx match $expr ORDER BY 1 DESC
   258    257       } $res
   259    258     }

Changes to ext/fts5/test/fts5ad.test.

     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #*************************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is testing the FTS5 module.
    13     13   #
           14  +# More specifically, the focus is on testing prefix queries, both with and
           15  +# without prefix indexes.
    14     16   #
    15     17   
    16     18   source [file join [file dirname [info script]] fts5_common.tcl]
    17     19   set testprefix fts5ad
    18     20   
    19     21   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     22   ifcapable !fts5 {

Changes to ext/fts5/test/fts5bigtok.test.

    10     10   #*************************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is testing the FTS5 module.
    13     13   #
    14     14   
    15     15   source [file join [file dirname [info script]] fts5_common.tcl]
    16     16   set testprefix fts5bigtok
           17  +return_if_no_fts5
    17     18   
    18     19   proc rndterm {} {
    19     20     set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
    20     21     set l [lindex $L [expr int(rand() * [llength $L])]]
    21     22     string repeat $l [expr int(rand() * 5) + 60]
    22     23   }
    23     24   

Changes to ext/fts5/test/fts5config.test.

   238    238     4 {detail='1,2,'}
   239    239     5 {detail=',1'}
   240    240     6 {detail=''}
   241    241   } {
   242    242     set res [list 1 {malformed detail=... directive}]
   243    243     do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
   244    244   }
          245  +
          246  +do_catchsql_test 12.1 {
          247  +  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
          248  +} {1 {SQL logic error or missing database}}
          249  +
          250  +#-------------------------------------------------------------------------
          251  +# errors in the 'usermerge' option
          252  +#
          253  +do_execsql_test 13.0 {
          254  +  CREATE VIRTUAL TABLE tt USING fts5(ttt);
          255  +}
          256  +foreach {tn val} {
          257  +  1     -1
          258  +  2     4.2
          259  +  3     17
          260  +  4     1
          261  +} {
          262  +  set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
          263  +  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
          264  +}
   245    265   
   246    266   finish_test
   247    267   

Changes to ext/fts5/test/fts5corrupt3.test.

   175    175   for {set i 1} {1} {incr i} {
   176    176     set struct [db one {SELECT block FROM t1_data WHERE id=10}]
   177    177     binary scan $struct c* var
   178    178     set end [lindex $var end]
   179    179     if {$end<=$i} break
   180    180     lset var end [expr $end - $i]
   181    181     set struct [binary format c* $var]
          182  +
          183  +  db close
          184  +  sqlite3 db test.db
          185  +
   182    186     db eval {
   183    187       BEGIN;
   184    188       UPDATE t1_data SET block = $struct WHERE id=10;
   185    189     }
   186    190     do_test 4.1.$i {
   187    191       incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
   188    192       set {} {}
................................................................................
   330    334       WHERE id>100;
   331    335   }
   332    336   do_catchsql_test 6.3.5 {
   333    337     INSERT INTO t1(t1) VALUES('integrity-check');
   334    338   } {1 {database disk image is malformed}}
   335    339   
   336    340   
   337         -}
   338         -
   339    341   #------------------------------------------------------------------------
   340    342   #
   341         -reset_db
   342    343   reset_db
   343    344   proc rnddoc {n} {
   344    345     set map [list a b c d]
   345    346     set doc [list]
   346    347     for {set i 0} {$i < $n} {incr i} {
   347    348       lappend doc "x[lindex $map [expr int(rand()*4)]]"
   348    349     }
................................................................................
   367    368       db eval {DELETE FROM t5_data WHERE rowid = $i}
   368    369       set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ]
   369    370       if {$r != "1 {database disk image is malformed}"} { error $r }
   370    371       db eval ROLLBACK  
   371    372     }
   372    373   } {}
   373    374   
          375  +}
          376  +
          377  +#------------------------------------------------------------------------
          378  +# Corruption within the structure record.
          379  +#
          380  +reset_db
          381  +do_execsql_test 8.1 {
          382  +  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
          383  +  INSERT INTO t1 VALUES('one', 'two');
          384  +}
          385  +
          386  +do_test 9.1.1 {
          387  +  set    blob "12345678"    ;# cookie
          388  +  append blob "0105"        ;# 1 level, total of 5 segments
          389  +  append blob "06"          ;# write counter
          390  +  append blob "0002"        ;# first level has 0 segments merging, 2 other.
          391  +  append blob "450108"      ;# first segment
          392  +  execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
          393  +} {}
          394  +do_catchsql_test 9.1.2 {
          395  +  SELECT * FROM t1('one AND two');
          396  +} {1 {database disk image is malformed}}
          397  +
          398  +do_test 9.2.1 {
          399  +  set    blob "12345678"    ;# cookie
          400  +  append blob "0205"        ;# 2 levels, total of 5 segments
          401  +  append blob "06"          ;# write counter
          402  +  append blob "0001"        ;# first level has 0 segments merging, 1 other.
          403  +  append blob "450108"      ;# first segment
          404  +  execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
          405  +} {}
          406  +do_catchsql_test 9.2.2 {
          407  +  SELECT * FROM t1('one AND two');
          408  +} {1 {database disk image is malformed}}
          409  +
   374    410   sqlite3_fts5_may_be_corrupt 0
   375    411   finish_test
   376    412   

Changes to ext/fts5/test/fts5detail.test.

    80     80   do_execsql_test 2.1 {
    81     81     INSERT INTO t2(t2) VALUES('integrity-check');
    82     82   }
    83     83   
    84     84   do_execsql_test 2.2 {
    85     85     SELECT fts5_test_poslist(t2) FROM t2('aa');
    86     86   } {0.0.0}
           87  +
           88  +do_execsql_test 2.3 {
           89  +  SELECT fts5_test_collist(t2) FROM t2('aa');
           90  +} {0.0}
    87     91   
    88     92   set ::pc 0
    89     93   #puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
    90     94   #exit
    91     95   
    92     96   #-------------------------------------------------------------------------
    93     97   # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs

Added ext/fts5/test/fts5determin.test.

            1  +# 2016 March 21
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +# Specifically, that the fts5 module is deterministic. At one point, when
           15  +# segment ids were allocated using sqlite3_randomness(), this was not the
           16  +# case.
           17  +#
           18  +
           19  +source [file join [file dirname [info script]] fts5_common.tcl]
           20  +set testprefix fts5aa
           21  +return_if_no_fts5 
           22  +
           23  +proc do_determin_test {tn} {
           24  +  uplevel [list
           25  +    do_execsql_test $tn {
           26  +      SELECT (SELECT md5sum(id, block) FROM t1_data)==
           27  +             (SELECT md5sum(id, block) FROM t2_data),
           28  +             (SELECT md5sum(id, block) FROM t1_data)==
           29  +             (SELECT md5sum(id, block) FROM t3_data)
           30  +    } {1 1}
           31  +  ]
           32  +}
           33  +
           34  +foreach_detail_mode $::testprefix {
           35  +  do_execsql_test 1.0 {
           36  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
           37  +    CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
           38  +    CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%);
           39  +  }
           40  +
           41  +  do_test 1.1 {
           42  +    foreach t {t1 t2 t3} {
           43  +      execsql [string map [list TBL $t] {
           44  +        INSERT INTO TBL VALUES('a b c', 'd e f');
           45  +        INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3');
           46  +        INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz');
           47  +      }]
           48  +    }
           49  +  } {}
           50  +
           51  +  do_determin_test 1.2
           52  +
           53  +  do_test 1.3 {
           54  +    foreach t {t1 t2 t3} {
           55  +      execsql [string map [list TBL $t] {
           56  +        INSERT INTO TBL(TBL) VALUES('optimize');
           57  +      }]
           58  +    }
           59  +  } {}
           60  +
           61  +  do_determin_test 1.4
           62  +}
           63  +
           64  +
           65  +finish_test
           66  +
           67  +

Changes to ext/fts5/test/fts5dlidx.test.

   174    174   
   175    175   do_execsql_test 3.2 {
   176    176     SELECT rowid FROM abc WHERE abc 
   177    177     MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' 
   178    178     ORDER BY rowid DESC;
   179    179   } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
   180    180   
   181         -do_execsql_test 3.2 {
          181  +do_execsql_test 3.3 {
   182    182     INSERT INTO abc(abc) VALUES('integrity-check');
   183    183     INSERT INTO abc(abc) VALUES('optimize');
   184    184     INSERT INTO abc(abc) VALUES('integrity-check');
   185    185   }
   186    186   
   187    187   set v [lindex $vocab 0]
   188    188   set i 0
   189    189   foreach v $vocab {
   190         -  do_execsql_test 3.3.[incr i] {
          190  +  do_execsql_test 3.4.[incr i] {
   191    191       SELECT rowid FROM abc WHERE abc MATCH $v
   192    192     } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
   193    193   }
   194    194   
   195    195   } ;# foreach_detail_mode
   196    196   
   197    197   
   198    198   
   199    199   finish_test
   200    200   

Changes to ext/fts5/test/fts5eb.test.

    29     29     do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
    30     30   }
    31     31   
    32     32   foreach {tn expr res} {
    33     33     1  {abc}                            {"abc"}
    34     34     2  {abc ""}                         {"abc"}
    35     35     3  {""}                             {}
    36         -  4  {abc OR ""}                      {"abc"}
    37         -  5  {abc NOT ""}                     {"abc"}
    38         -  6  {abc AND ""}                     {"abc"}
    39         -  7  {"" OR abc}                      {"abc"}
    40         -  8  {"" NOT abc}                     {"abc"}
    41         -  9  {"" AND abc}                     {"abc"}
           36  +  4  {abc OR ""}                      {"abc" OR ""}
           37  +  5  {abc NOT ""}                     {"abc" NOT ""}
           38  +  6  {abc AND ""}                     {"abc" AND ""}
           39  +  7  {"" OR abc}                      {"" OR "abc"}
           40  +  8  {"" NOT abc}                     {"" NOT "abc"}
           41  +  9  {"" AND abc}                     {"" AND "abc"}
    42     42     10 {abc + "" + def}                 {"abc" + "def"}
    43     43     11 {abc "" def}                     {"abc" AND "def"}
    44     44     12 {r+e OR w}                       {"r" + "e" OR "w"}
    45     45   
    46     46     13 {a AND b NOT c}                  {"a" AND ("b" NOT "c")}
    47     47     14 {a OR b NOT c}                   {"a" OR ("b" NOT "c")}
    48     48     15 {a NOT b AND c}                  {("a" NOT "b") AND "c"}

Changes to ext/fts5/test/fts5fault1.test.

    68     68   } -test {
    69     69     faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
    70     70   }
    71     71   
    72     72   reset_db
    73     73   do_execsql_test 4.0 {
    74     74     CREATE VIRTUAL TABLE t2 USING fts5(a, b);
    75         -  INSERT INTO t2 VALUES('m f a jj th q jr ar',   'hj n h h sg j i m');
           75  +  INSERT INTO t2 VALUES('m f a jj th q gi ar',   'hj n h h sg j i m');
    76     76     INSERT INTO t2 VALUES('nr s t g od j kf h',    'sb h aq rg op rb n nl');
    77     77     INSERT INTO t2 VALUES('do h h pb p p q fr',    'c rj qs or cr a l i');
    78     78     INSERT INTO t2 VALUES('lk gp t i lq mq qm p',  'h mr g f op ld aj h');
    79     79     INSERT INTO t2 VALUES('ct d sq kc qi k f j',   'sn gh c of g s qt q');
    80     80     INSERT INTO t2 VALUES('d ea d d om mp s ab',   'dm hg l df cm ft pa c');
    81     81     INSERT INTO t2 VALUES('tc dk c jn n t sr ge',  'a a kn bc n i af h');
    82     82     INSERT INTO t2 VALUES('ie ii d i b sa qo rf',  'a h m aq i b m fn');
................................................................................
    91     91     3 { f*  }           {1 3 4 5 6 8 9 10}
    92     92     4 { m OR f }        {1 4 5 8 9 10}
    93     93     5 { sn + gh }       {5}
    94     94     6 { "sn gh" }       {5}
    95     95     7 { NEAR(r a, 5) }  {9}
    96     96     8 { m* f* }         {1 4 6 8 9 10}
    97     97     9 { m* + f* }       {1 8}
           98  +  10 { c NOT p }       {5 6 7 10}
    98     99   } {
    99    100     do_faultsim_test 4.$tn -prep {
   100    101       faultsim_restore_and_reopen
   101    102     } -body "
   102    103       execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
   103    104     " -test "
   104    105       faultsim_test_result {[list 0 $res]} {1 {vtable constructor failed: t2}}

Changes to ext/fts5/test/fts5fault2.test.

    12     12   # This file is focused on OOM errors.
    13     13   #
    14     14   
    15     15   source [file join [file dirname [info script]] fts5_common.tcl]
    16     16   source $testdir/malloc_common.tcl
    17     17   set testprefix fts5fault2
    18     18   
    19         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           19  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22