/ Check-in [566b551e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge updates from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | altShellFix
Files: files | file ages | folders
SHA1:566b551e5a81440a5c8ff865ceb4422c76d67cf7
User & Date: mistachkin 2016-04-03 20:45:04
Context
2016-04-03
20:50
Replace the new fprintf() calls. check-in: f76c3a0c user: mistachkin tags: altShellFix
20:45
Merge updates from trunk. check-in: 566b551e user: mistachkin tags: altShellFix
2016-04-01
17:54
Preupdate hook documentation fixes. No changes to code. check-in: 59814f35 user: drh tags: trunk
2016-01-06
20:50
Merge updates from trunk. check-in: 5cbab637 user: mistachkin tags: altShellFix
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

    26     26   # will run on the target platform.  (BCC and TCC are usually the
    27     27   # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
    28     28   # are provide so that these aspects of the build process can be changed
    29     29   # on the "make" command-line.  Ex:  "make CC=clang CFLAGS=-fsanitize=undefined"
    30     30   #
    31     31   CC = @CC@
    32     32   CFLAGS = @CPPFLAGS@ @CFLAGS@
    33         -TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3
           33  +TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
           34  +TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
    34     35   
    35     36   # Define this for the autoconf-based build, so that the code knows it can
    36     37   # include the generated config.h
    37     38   # 
    38     39   TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
    39     40   
    40     41   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
................................................................................
    77     78   TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
    78     79   
    79     80   # Enable/disable loadable extensions, and other optional features
    80     81   # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
    81     82   # The same set of OMIT and ENABLE flags should be passed to the 
    82     83   # LEMON parser generator and the mkkeywordhash tool as well.
    83     84   OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
           85  +OPT_FEATURE_FLAGS += -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK
    84     86   
    85     87   TCC += $(OPT_FEATURE_FLAGS)
    86     88   
    87     89   # Add in any optional parameters specified on the make commane line
    88     90   # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
    89     91   TCC += $(OPTS)
    90     92   
................................................................................
   169    171            expr.lo fault.lo fkey.lo \
   170    172            fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
   171    173            fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
   172    174            fts3_tokenize_vtab.lo \
   173    175            fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
   174    176   	 fts5.lo \
   175    177            func.lo global.lo hash.lo \
   176         -         icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \
          178  +         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
   177    179            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   178    180            memjournal.lo \
   179    181            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   180    182            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   181    183            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   182         -         random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
          184  +         random.lo resolve.lo rowset.lo rtree.lo \
          185  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
   183    186            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   184    187            update.lo util.lo vacuum.lo \
   185    188            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   186    189            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   187    190            utf.lo vtab.lo
   188    191   
   189    192   # Object files for the amalgamation.
................................................................................
   220    223     $(TOP)/src/fkey.c \
   221    224     $(TOP)/src/func.c \
   222    225     $(TOP)/src/global.c \
   223    226     $(TOP)/src/hash.c \
   224    227     $(TOP)/src/hash.h \
   225    228     $(TOP)/src/hwtime.h \
   226    229     $(TOP)/src/insert.c \
   227         -  $(TOP)/src/journal.c \
   228    230     $(TOP)/src/legacy.c \
   229    231     $(TOP)/src/loadext.c \
   230    232     $(TOP)/src/main.c \
   231    233     $(TOP)/src/malloc.c \
   232    234     $(TOP)/src/mem0.c \
   233    235     $(TOP)/src/mem1.c \
   234    236     $(TOP)/src/mem2.c \
................................................................................
   339    341   SRC += \
   340    342     $(TOP)/ext/icu/sqliteicu.h \
   341    343     $(TOP)/ext/icu/icu.c
   342    344   SRC += \
   343    345     $(TOP)/ext/rtree/rtree.h \
   344    346     $(TOP)/ext/rtree/rtree.c
   345    347   SRC += \
          348  +  $(TOP)/ext/session/sqlite3session.c \
          349  +  $(TOP)/ext/session/sqlite3session.h
          350  +SRC += \
   346    351     $(TOP)/ext/rbu/sqlite3rbu.h \
   347    352     $(TOP)/ext/rbu/sqlite3rbu.c
   348    353   SRC += \
   349    354     $(TOP)/ext/misc/json1.c
   350         -
   351         -
   352    355   
   353    356   # Generated source code files
   354    357   #
   355    358   SRC += \
   356    359     keywordhash.h \
   357    360     opcodes.c \
   358    361     opcodes.h \
................................................................................
   372    375     $(TOP)/src/test6.c \
   373    376     $(TOP)/src/test7.c \
   374    377     $(TOP)/src/test8.c \
   375    378     $(TOP)/src/test9.c \
   376    379     $(TOP)/src/test_autoext.c \
   377    380     $(TOP)/src/test_async.c \
   378    381     $(TOP)/src/test_backup.c \
          382  +  $(TOP)/src/test_bestindex.c \
   379    383     $(TOP)/src/test_blob.c \
   380    384     $(TOP)/src/test_btree.c \
   381    385     $(TOP)/src/test_config.c \
   382    386     $(TOP)/src/test_demovfs.c \
   383    387     $(TOP)/src/test_devsym.c \
   384    388     $(TOP)/src/test_fs.c \
   385    389     $(TOP)/src/test_func.c \
................................................................................
   401    405     $(TOP)/src/test_syscall.c \
   402    406     $(TOP)/src/test_tclvar.c \
   403    407     $(TOP)/src/test_thread.c \
   404    408     $(TOP)/src/test_vfs.c \
   405    409     $(TOP)/src/test_windirent.c \
   406    410     $(TOP)/src/test_wsd.c       \
   407    411     $(TOP)/ext/fts3/fts3_term.c \
   408         -  $(TOP)/ext/fts3/fts3_test.c \
          412  +  $(TOP)/ext/fts3/fts3_test.c  \
          413  +  $(TOP)/ext/session/test_session.c \
   409    414     $(TOP)/ext/rbu/test_rbu.c 
   410    415   
   411    416   # Statically linked extensions
   412    417   #
   413    418   TESTSRC += \
   414    419     $(TOP)/ext/misc/amatch.c \
   415    420     $(TOP)/ext/misc/closure.c \
   416    421     $(TOP)/ext/misc/eval.c \
   417    422     $(TOP)/ext/misc/fileio.c \
   418    423     $(TOP)/ext/misc/fuzzer.c \
   419    424     $(TOP)/ext/fts5/fts5_tcl.c \
   420    425     $(TOP)/ext/fts5/fts5_test_mi.c \
          426  +  $(TOP)/ext/fts5/fts5_test_tok.c \
   421    427     $(TOP)/ext/misc/ieee754.c \
   422    428     $(TOP)/ext/misc/nextchar.c \
   423    429     $(TOP)/ext/misc/percentile.c \
   424    430     $(TOP)/ext/misc/regexp.c \
   425    431     $(TOP)/ext/misc/series.c \
   426    432     $(TOP)/ext/misc/spellfix.c \
   427    433     $(TOP)/ext/misc/totype.c \
................................................................................
   469    475     parse.c \
   470    476     $(TOP)/ext/fts3/fts3.c \
   471    477     $(TOP)/ext/fts3/fts3_aux.c \
   472    478     $(TOP)/ext/fts3/fts3_expr.c \
   473    479     $(TOP)/ext/fts3/fts3_term.c \
   474    480     $(TOP)/ext/fts3/fts3_tokenizer.c \
   475    481     $(TOP)/ext/fts3/fts3_write.c \
   476         -  $(TOP)/ext/async/sqlite3async.c
          482  +  $(TOP)/ext/async/sqlite3async.c \
          483  +  $(TOP)/ext/session/sqlite3session.c 
   477    484   
   478    485   # Header files used by all library source files.
   479    486   #
   480    487   HDR = \
   481    488      $(TOP)/src/btree.h \
   482    489      $(TOP)/src/btreeInt.h \
   483    490      $(TOP)/src/hash.h \
................................................................................
   544    551   
   545    552   # Standard options to testfixture
   546    553   #
   547    554   TESTOPTS = --verbose=file --output=test-out.txt
   548    555   
   549    556   # Extra compiler options for various shell tools
   550    557   #
   551         -SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
          558  +SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
          559  +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   552    560   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   553    561   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
   554    562   
   555    563   # This is the default Makefile target.  The objects listed here
   556    564   # are what get build when you type just "make" with no arguments.
   557    565   #
   558    566   all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
................................................................................
   570    578   libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
   571    579   	$(LTLINK) -no-undefined -o $@ tclsqlite.lo \
   572    580   		libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \
   573    581   		-rpath "$(TCLLIBDIR)" \
   574    582   		-version-info "8:6:8" \
   575    583   		-avoid-version
   576    584   
   577         -sqlite3$(TEXE):	$(TOP)/src/shell.c libsqlite3.la sqlite3.h
          585  +sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
   578    586   	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
   579         -		$(TOP)/src/shell.c libsqlite3.la \
          587  +		$(TOP)/src/shell.c sqlite3.c \
   580    588   		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
   581    589   
   582    590   sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
   583    591   	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
          592  +
          593  +srcck1$(BEXE):	$(TOP)/tool/srcck1.c
          594  +	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
          595  +
          596  +sourcetest:	srcck1$(BEXE) sqlite3.c
          597  +	./srcck1 sqlite3.c
   584    598   
   585    599   fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
   586    600   	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
   587    601   	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
   588    602   
   589    603   fuzzcheck$(TEXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
   590    604   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
................................................................................
   622    636   	mv vdbe.new tsrc/vdbe.c
   623    637   	cp fts5.c fts5.h tsrc
   624    638   	touch .target_source
   625    639   
   626    640   sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
   627    641   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
   628    642   	cp tsrc/shell.c tsrc/sqlite3ext.h .
          643  +	cp $(TOP)/ext/session/sqlite3session.h .
   629    644   
   630    645   sqlite3ext.h:	.target_source
   631    646   	cp tsrc/sqlite3ext.h .
   632    647   
   633    648   tclsqlite3.c:	sqlite3.c
   634    649   	echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
   635    650   	cat sqlite3.c >>tclsqlite3.c
................................................................................
   726    741   
   727    742   hash.lo:	$(TOP)/src/hash.c $(HDR)
   728    743   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/hash.c
   729    744   
   730    745   insert.lo:	$(TOP)/src/insert.c $(HDR)
   731    746   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c
   732    747   
   733         -journal.lo:	$(TOP)/src/journal.c $(HDR)
   734         -	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/journal.c
   735         -
   736    748   legacy.lo:	$(TOP)/src/legacy.c $(HDR)
   737    749   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c
   738    750   
   739    751   loadext.lo:	$(TOP)/src/loadext.c $(HDR)
   740    752   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/loadext.c
   741    753   
   742    754   main.lo:	$(TOP)/src/main.c $(HDR)
................................................................................
   988   1000   
   989   1001   fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
   990   1002   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
   991   1003   
   992   1004   rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
   993   1005   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
   994   1006   
         1007  +sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
         1008  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
         1009  +
   995   1010   json1.lo:	$(TOP)/ext/misc/json1.c
   996   1011   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
   997   1012   
   998   1013   # FTS5 things
   999   1014   #
  1000   1015   FTS5_SRC = \
  1001   1016      $(TOP)/ext/fts5/fts5.h \
................................................................................
  1024   1039   fts5.c: $(FTS5_SRC)
  1025   1040   	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
  1026   1041   	cp $(TOP)/ext/fts5/fts5.h .
  1027   1042   
  1028   1043   fts5.lo:	fts5.c $(HDR) $(EXTHDR)
  1029   1044   	$(LTCOMPILE) -DSQLITE_CORE -c fts5.c
  1030   1045   
         1046  +sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
         1047  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
         1048  +
  1031   1049   
  1032   1050   # Rules to build the 'testfixture' application.
  1033   1051   #
  1034   1052   # If using the amalgamation, use sqlite3.c directly to build the test
  1035   1053   # fixture.  Otherwise link against libsqlite3.la.  (This distinction is
  1036   1054   # necessary because the test fixture requires non-API symbols which are
  1037   1055   # hidden when the library is built via the amalgamation).
  1038   1056   #
  1039   1057   TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1040   1058   TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
  1041   1059   TESTFIXTURE_FLAGS += -DBUILD_sqlite
         1060  +TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
         1061  +TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
  1042   1062   
  1043   1063   TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
  1044   1064   TESTFIXTURE_SRC1 = sqlite3.c
  1045   1065   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
  1046   1066   TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
  1047   1067   
  1048   1068   testfixture$(TEXE):	$(TESTFIXTURE_SRC)
................................................................................
  1075   1095   #
  1076   1096   quicktest:	./testfixture$(TEXE)
  1077   1097   	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
  1078   1098   
  1079   1099   # This is the common case.  Run many tests that do not take too long,
  1080   1100   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1081   1101   #
  1082         -test:	$(TESTPROGS) fastfuzztest
         1102  +test:	$(TESTPROGS) sourcetest fastfuzztest
  1083   1103   	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
  1084   1104   
  1085   1105   # Run a test using valgrind.  This can take a really long time
  1086   1106   # because valgrind is so much slower than a native machine.
  1087   1107   #
  1088   1108   valgrindtest:	$(TESTPROGS) valgrindfuzz
  1089   1109   	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS)
................................................................................
  1115   1135   
  1116   1136   showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
  1117   1137   	$(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS)
  1118   1138   
  1119   1139   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1120   1140   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1121   1141   
         1142  +changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
         1143  +	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
         1144  +
  1122   1145   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
  1123   1146   	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
  1124   1147   
  1125   1148   LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
  1126   1149   	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
  1127   1150   
  1128   1151   wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
  1129   1152   	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
  1130   1153   
  1131   1154   speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
  1132   1155   	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
         1156  +
         1157  +rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
         1158  +	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
         1159  +
         1160  +loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
         1161  +	$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
  1133   1162   
  1134   1163   # This target will fail if the SQLite amalgamation contains any exported
  1135   1164   # symbols that do not begin with "sqlite3_". It is run as part of the
  1136   1165   # releasetest.tcl script.
  1137   1166   #
  1138   1167   checksymbols: sqlite3.lo
  1139         -	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
         1168  +	nm -g --defined-only sqlite3.o | egrep -v ' sqlite3(changeset|session)?_' ; test $$? -ne 0
  1140   1169   	echo '0 errors out of 1 tests'
  1141   1170   
  1142         -# Build the amalgamation-autoconf package.
         1171  +# Build the amalgamation-autoconf package.  The amalamgation-tarball target builds
         1172  +# a tarball named for the version number.  Ex:  sqlite-autoconf-3110000.tar.gz.
         1173  +# The snapshot-tarball target builds a tarball named by the SHA1 hash
  1143   1174   #
  1144   1175   amalgamation-tarball: sqlite3.c
  1145         -	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh
         1176  +	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal
         1177  +
         1178  +snapshot-tarball: sqlite3.c
         1179  +	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
  1146   1180   
  1147   1181   # The next two rules are used to support the "threadtest" target. Building
  1148   1182   # threadtest runs a few thread-safety tests that are implemented in C. This
  1149   1183   # target is invoked by the releasetest.tcl script.
  1150   1184   # 
  1151   1185   THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
  1152   1186                     $(TOP)/test/tt3_checkpoint.c \
................................................................................
  1166   1200   
  1167   1201   # Standard install and cleanup targets
  1168   1202   #
  1169   1203   lib_install:	libsqlite3.la
  1170   1204   	$(INSTALL) -d $(DESTDIR)$(libdir)
  1171   1205   	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
  1172   1206   	
  1173         -install:	sqlite3$(BEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
         1207  +install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
  1174   1208   	$(INSTALL) -d $(DESTDIR)$(bindir)
  1175         -	$(LTINSTALL) sqlite3$(BEXE) $(DESTDIR)$(bindir)
         1209  +	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
  1176   1210   	$(INSTALL) -d $(DESTDIR)$(includedir)
  1177   1211   	$(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir)
  1178   1212   	$(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir)
  1179   1213   	$(INSTALL) -d $(DESTDIR)$(pkgconfigdir)
  1180   1214   	$(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(pkgconfigdir)
  1181   1215   
  1182   1216   pkgIndex.tcl:
................................................................................
  1196   1230   	rm -f *.da *.bb *.bbg gmon.out
  1197   1231   	rm -rf quota2a quota2b quota2c
  1198   1232   	rm -rf tsrc .target_source
  1199   1233   	rm -f tclsqlite3$(TEXE)
  1200   1234   	rm -f testfixture$(TEXE) test.db
  1201   1235   	rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
  1202   1236   	rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
  1203         -	rm -f wordcount$(TEXE)
         1237  +	rm -f wordcount$(TEXE) changeset$(TEXE)
  1204   1238   	rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
  1205   1239   	rm -f sqlite3.c
  1206   1240   	rm -f sqlite3rc.h
  1207   1241   	rm -f shell.c sqlite3ext.h
  1208   1242   	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
  1209   1243   	rm -f sqlite-*-output.vsix
  1210   1244   	rm -f mptester mptester.exe
         1245  +	rm -f rbu rbu.exe
         1246  +	rm -f srcck1 srcck1.exe
  1211   1247   	rm -f fuzzershell fuzzershell.exe
  1212   1248   	rm -f fuzzcheck fuzzcheck.exe
  1213   1249   	rm -f sqldiff sqldiff.exe
  1214   1250   	rm -f fts5.* fts5parse.*
  1215   1251   
  1216   1252   distclean:	clean
  1217   1253   	rm -f config.h config.log config.status libtool Makefile sqlite3.pc

Changes to Makefile.msc.

     6      6   ###############################################################################
     7      7   
     8      8   # The toplevel directory of the source tree.  This is the directory
     9      9   # that contains this "Makefile.msc".
    10     10   #
    11     11   TOP = .
    12     12   
           13  +# <<mark>>
    13     14   # Set this non-0 to create and use the SQLite amalgamation file.
    14     15   #
    15     16   !IFNDEF USE_AMALGAMATION
    16     17   USE_AMALGAMATION = 1
    17     18   !ENDIF
           19  +# <</mark>>
    18     20   
    19     21   # Set this non-0 to enable full warnings (-W4, etc) when compiling.
    20     22   #
    21     23   !IFNDEF USE_FULLWARN
    22     24   USE_FULLWARN = 0
    23     25   !ENDIF
    24     26   
................................................................................
    64     66   # Set this non-0 to split the SQLite amalgamation file into chunks to
    65     67   # be used for debugging with Visual Studio.
    66     68   #
    67     69   !IFNDEF SPLIT_AMALGAMATION
    68     70   SPLIT_AMALGAMATION = 0
    69     71   !ENDIF
    70     72   
           73  +# <<mark>>
    71     74   # Set this non-0 to use the International Components for Unicode (ICU).
    72     75   #
    73     76   !IFNDEF USE_ICU
    74     77   USE_ICU = 0
    75     78   !ENDIF
           79  +# <</mark>>
    76     80   
    77     81   # Set this non-0 to dynamically link to the MSVC runtime library.
    78     82   #
    79     83   !IFNDEF USE_CRT_DLL
    80     84   USE_CRT_DLL = 0
    81     85   !ENDIF
    82     86   
................................................................................
   118    122   # This setting does not apply to any binaries that require Tcl to operate
   119    123   # properly (i.e. the text fixture, etc).
   120    124   #
   121    125   !IFNDEF FOR_WINRT
   122    126   FOR_WINRT = 0
   123    127   !ENDIF
   124    128   
   125         -# Set this non-0 to compile binaries suitable for the UAP environment.
          129  +# Set this non-0 to compile binaries suitable for the UWP environment.
   126    130   # This setting does not apply to any binaries that require Tcl to operate
   127    131   # properly (i.e. the text fixture, etc).
   128    132   #
   129         -!IFNDEF FOR_UAP
   130         -FOR_UAP = 0
          133  +!IFNDEF FOR_UWP
          134  +FOR_UWP = 0
   131    135   !ENDIF
   132    136   
          137  +# Set this non-0 to compile binaries suitable for the Windows 10 platform.
          138  +#
          139  +!IFNDEF FOR_WIN10
          140  +FOR_WIN10 = 0
          141  +!ENDIF
          142  +
          143  +# <<mark>>
   133    144   # Set this non-0 to skip attempting to look for and/or link with the Tcl
   134    145   # runtime library.
   135    146   #
   136    147   !IFNDEF NO_TCL
   137    148   NO_TCL = 0
   138    149   !ENDIF
          150  +# <</mark>>
   139    151   
   140    152   # Set this to non-0 to create and use PDBs.
   141    153   #
   142    154   !IFNDEF SYMBOLS
   143    155   SYMBOLS = 1
   144    156   !ENDIF
   145    157   
................................................................................
   181    193   # Enable use of available compiler optimizations?  Normally, this should be
   182    194   # non-zero.  Setting this to zero, thus disabling all compiler optimizations,
   183    195   # can be useful for testing.
   184    196   #
   185    197   !IFNDEF OPTIMIZATIONS
   186    198   OPTIMIZATIONS = 2
   187    199   !ENDIF
          200  +
          201  +# Set the source code file to be used by executables and libraries when
          202  +# they need the amalgamation.
          203  +#
          204  +!IFNDEF SQLITE3C
          205  +!IF $(SPLIT_AMALGAMATION)!=0
          206  +SQLITE3C = sqlite3-all.c
          207  +!ELSE
          208  +SQLITE3C = sqlite3.c
          209  +!ENDIF
          210  +!ENDIF
          211  +
          212  +# Set the include code file to be used by executables and libraries when
          213  +# they need SQLite.
          214  +#
          215  +!IFNDEF SQLITE3H
          216  +SQLITE3H = sqlite3.h
          217  +!ENDIF
          218  +
          219  +# This is the name to use for the SQLite dynamic link library (DLL).
          220  +#
          221  +!IFNDEF SQLITE3DLL
          222  +!IF $(FOR_WIN10)!=0
          223  +SQLITE3DLL = winsqlite3.dll
          224  +!ELSE
          225  +SQLITE3DLL = sqlite3.dll
          226  +!ENDIF
          227  +!ENDIF
          228  +
          229  +# This is the name to use for the SQLite import library (LIB).
          230  +#
          231  +!IFNDEF SQLITE3LIB
          232  +!IF $(FOR_WIN10)!=0
          233  +SQLITE3LIB = winsqlite3.lib
          234  +!ELSE
          235  +SQLITE3LIB = sqlite3.lib
          236  +!ENDIF
          237  +!ENDIF
          238  +
          239  +# This is the name to use for the SQLite shell executable (EXE).
          240  +#
          241  +!IFNDEF SQLITE3EXE
          242  +!IF $(FOR_WIN10)!=0
          243  +SQLITE3EXE = winsqlite3shell.exe
          244  +!ELSE
          245  +SQLITE3EXE = sqlite3.exe
          246  +!ENDIF
          247  +!ENDIF
          248  +
          249  +# This is the argument used to set the program database (PDB) file for the
          250  +# SQLite shell executable (EXE).
          251  +#
          252  +!IFNDEF SQLITE3EXEPDB
          253  +!IF $(FOR_WIN10)!=0
          254  +SQLITE3EXEPDB =
          255  +!ELSE
          256  +SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
          257  +!ENDIF
          258  +!ENDIF
   188    259   
   189    260   # These are the "standard" SQLite compilation options used when compiling for
   190    261   # the Windows platform.
   191    262   #
   192    263   !IFNDEF OPT_FEATURE_FLAGS
   193    264   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
   194    265   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
   195    266   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
          267  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
          268  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
          269  +!ENDIF
          270  +
          271  +# These are the "extended" SQLite compilation options used when compiling for
          272  +# the Windows 10 platform.
          273  +#
          274  +!IFNDEF EXT_FEATURE_FLAGS
          275  +!IF $(FOR_WIN10)!=0
          276  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
          277  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1
          278  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1
          279  +!ELSE
          280  +EXT_FEATURE_FLAGS =
          281  +!ENDIF
   196    282   !ENDIF
   197    283   
   198    284   ###############################################################################
   199    285   ############################### END OF OPTIONS ################################
   200    286   ###############################################################################
          287  +
          288  +# When compiling for the Windows 10 platform, the PLATFORM macro must be set
          289  +# to an appropriate value (e.g. x86, x64, arm, arm64, etc).
          290  +#
          291  +!IF $(FOR_WIN10)!=0
          292  +!IFNDEF PLATFORM
          293  +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM.
          294  +!ENDIF
          295  +!ENDIF
   201    296   
   202    297   # This assumes that MSVC is always installed in 32-bit Program Files directory
   203    298   # and sets the variable for use in locating other 32-bit installs accordingly.
   204    299   #
   205    300   PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
   206    301   PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
   207    302   
................................................................................
   208    303   # Check for the predefined command macro CC.  This should point to the compiler
   209    304   # binary for the target platform.  If it is not defined, simply define it to
   210    305   # the legacy default value 'cl.exe'.
   211    306   #
   212    307   !IFNDEF CC
   213    308   CC = cl.exe
   214    309   !ENDIF
          310  +
          311  +# Check for the predefined command macro CSC.  This should point to a working
          312  +# C Sharp compiler binary.  If it is not defined, simply define it to the
          313  +# legacy default value 'csc.exe'.
          314  +#
          315  +!IFNDEF CSC
          316  +CSC = csc.exe
          317  +!ENDIF
   215    318   
   216    319   # Check for the command macro LD.  This should point to the linker binary for
   217    320   # the target platform.  If it is not defined, simply define it to the legacy
   218    321   # default value 'link.exe'.
   219    322   #
   220    323   !IFNDEF LD
   221    324   LD = link.exe
................................................................................
   225    328   # compiler binary for the target platform.  If it is not defined, simply define
   226    329   # it to the legacy default value 'rc.exe'.
   227    330   #
   228    331   !IFNDEF RC
   229    332   RC = rc.exe
   230    333   !ENDIF
   231    334   
   232         -# Check for the MSVC runtime library path macro.  Othertise, this value will
          335  +# Check for the MSVC runtime library path macro.  Otherwise, this value will
   233    336   # default to the 'lib' directory underneath the MSVC installation directory.
   234    337   #
   235    338   !IFNDEF CRTLIBPATH
   236    339   CRTLIBPATH = $(VCINSTALLDIR)\lib
   237    340   !ENDIF
   238    341   
   239    342   CRTLIBPATH = $(CRTLIBPATH:\\=\)
................................................................................
   262    365   !ELSEIF $(XCOMPILE)!=0
   263    366   NCC = "$(VCINSTALLDIR)\bin\$(CC)"
   264    367   NCC = $(NCC:\\=\)
   265    368   !ELSE
   266    369   NCC = $(CC)
   267    370   !ENDIF
   268    371   
   269         -# Check for the MSVC native runtime library path macro.  Othertise,
          372  +# Check for the MSVC native runtime library path macro.  Otherwise,
   270    373   # this value will default to the 'lib' directory underneath the MSVC
   271    374   # installation directory.
   272    375   #
   273    376   !IFNDEF NCRTLIBPATH
   274    377   NCRTLIBPATH = $(VCINSTALLDIR)\lib
   275    378   !ENDIF
   276    379   
   277    380   NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
   278    381   
   279         -# Check for the Platform SDK library path macro.  Othertise, this
          382  +# Check for the Platform SDK library path macro.  Otherwise, this
   280    383   # value will default to the 'lib' directory underneath the Windows
   281    384   # SDK installation directory (the environment variable used appears
   282    385   # to be available when using Visual C++ 2008 or later via the
   283    386   # command line).
   284    387   #
   285    388   !IFNDEF NSDKLIBPATH
   286    389   NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
   287    390   !ENDIF
   288    391   
   289    392   NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
          393  +
          394  +# Check for the UCRT library path macro.  Otherwise, this value will
          395  +# default to the version-specific, platform-specific 'lib' directory
          396  +# underneath the Windows SDK installation directory.
          397  +#
          398  +!IFNDEF UCRTLIBPATH
          399  +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM)
          400  +!ENDIF
          401  +
          402  +UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
   290    403   
   291    404   # C compiler and options for use in building executables that
   292    405   # will run on the platform that is doing the build.
   293    406   #
   294    407   !IF $(USE_FULLWARN)!=0
   295    408   BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
   296    409   !ELSE
................................................................................
   324    437   !IF $(USE_FULLWARN)!=0
   325    438   TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
   326    439   !ELSE
   327    440   TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
   328    441   !ENDIF
   329    442   
   330    443   TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise
   331         -RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
          444  +RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS)
   332    445   
   333    446   # Check if we want to use the "stdcall" calling convention when compiling.
   334    447   # This is not supported by the compilers for non-x86 platforms.  It should
   335    448   # also be noted here that building any target with these "stdcall" options
   336    449   # will most likely fail if the Tcl library is also required.  This is due
   337    450   # to how the Tcl library functions are declared and exported (i.e. without
   338    451   # an explicit calling convention, which results in "cdecl").
   339    452   #
   340         -!IF $(USE_STDCALL)!=0
          453  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
   341    454   !IF "$(PLATFORM)"=="x86"
   342    455   CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   343    456   SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   344    457   !ELSE
   345    458   !IFNDEF PLATFORM
   346    459   CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
   347    460   SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
................................................................................
   354    467   CORE_CCONV_OPTS =
   355    468   SHELL_CCONV_OPTS =
   356    469   !ENDIF
   357    470   
   358    471   # These are additional compiler options used for the core library.
   359    472   #
   360    473   !IFNDEF CORE_COMPILE_OPTS
   361         -!IF $(DYNAMIC_SHELL)!=0
          474  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   362    475   CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport)
   363    476   !ELSE
   364    477   CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
   365    478   !ENDIF
   366    479   !ENDIF
   367    480   
   368    481   # These are the additional targets that the core library should depend on
   369    482   # when linking.
   370    483   #
   371    484   !IFNDEF CORE_LINK_DEP
   372    485   !IF $(DYNAMIC_SHELL)!=0
   373    486   CORE_LINK_DEP =
   374         -!ELSE
          487  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
   375    488   CORE_LINK_DEP = sqlite3.def
          489  +!ELSE
          490  +CORE_LINK_DEP =
   376    491   !ENDIF
   377    492   !ENDIF
   378    493   
   379    494   # These are additional linker options used for the core library.
   380    495   #
   381    496   !IFNDEF CORE_LINK_OPTS
   382    497   !IF $(DYNAMIC_SHELL)!=0
   383    498   CORE_LINK_OPTS =
   384         -!ELSE
          499  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
   385    500   CORE_LINK_OPTS = /DEF:sqlite3.def
          501  +!ELSE
          502  +CORE_LINK_OPTS =
   386    503   !ENDIF
   387    504   !ENDIF
   388    505   
   389    506   # These are additional compiler options used for the shell executable.
   390    507   #
   391    508   !IFNDEF SHELL_COMPILE_OPTS
   392         -!IF $(DYNAMIC_SHELL)!=0
   393         -SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
          509  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          510  +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
   394    511   !ELSE
   395         -SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS)
          512  +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
          513  +!ENDIF
          514  +!ENDIF
          515  +
          516  +# This is the source code that the shell executable should be compiled
          517  +# with.
          518  +#
          519  +!IFNDEF SHELL_CORE_SRC
          520  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          521  +SHELL_CORE_SRC =
          522  +!ELSE
          523  +SHELL_CORE_SRC = $(SQLITE3C)
   396    524   !ENDIF
   397    525   !ENDIF
   398    526   
   399    527   # This is the core library that the shell executable should depend on.
   400    528   #
   401    529   !IFNDEF SHELL_CORE_DEP
   402         -!IF $(DYNAMIC_SHELL)!=0
   403         -SHELL_CORE_DEP = sqlite3.dll
          530  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          531  +SHELL_CORE_DEP = $(SQLITE3DLL)
   404    532   !ELSE
   405         -SHELL_CORE_DEP = libsqlite3.lib
          533  +SHELL_CORE_DEP =
   406    534   !ENDIF
   407    535   !ENDIF
   408    536   
   409    537   # This is the core library that the shell executable should link with.
   410    538   #
   411    539   !IFNDEF SHELL_CORE_LIB
   412         -!IF $(DYNAMIC_SHELL)!=0
   413         -SHELL_CORE_LIB = sqlite3.lib
          540  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          541  +SHELL_CORE_LIB = $(SQLITE3LIB)
   414    542   !ELSE
   415         -SHELL_CORE_LIB = libsqlite3.lib
          543  +SHELL_CORE_LIB =
   416    544   !ENDIF
   417    545   !ENDIF
   418    546   
   419    547   # These are additional linker options used for the shell executable.
   420    548   #
   421    549   !IFNDEF SHELL_LINK_OPTS
   422    550   SHELL_LINK_OPTS = $(SHELL_CORE_LIB)
................................................................................
   436    564   #
   437    565   !IF $(FOR_WINRT)!=0
   438    566   TCC = $(TCC) -DSQLITE_OS_WINRT=1
   439    567   RCC = $(RCC) -DSQLITE_OS_WINRT=1
   440    568   TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
   441    569   RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
   442    570   !ENDIF
          571  +
          572  +# C compiler options for the Windows 10 platform (needs MSVC 2015).
          573  +#
          574  +!IF $(FOR_WIN10)!=0
          575  +TCC = $(TCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          576  +BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          577  +!ENDIF
   443    578   
   444    579   # Also, we need to dynamically link to the correct MSVC runtime
   445    580   # when compiling for WinRT (e.g. debug or release) OR if the
   446    581   # USE_CRT_DLL option is set to force dynamically linking to the
   447    582   # MSVC runtime library.
   448    583   #
   449    584   !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
................................................................................
   460    595   BCC = $(BCC) -MTd
   461    596   !ELSE
   462    597   TCC = $(TCC) -MT
   463    598   BCC = $(BCC) -MT
   464    599   !ENDIF
   465    600   !ENDIF
   466    601   
          602  +# <<mark>>
   467    603   # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in
   468    604   # any extension header files by default.  For non-amalgamation
   469    605   # builds, we need to make sure the compiler can find these.
   470    606   #
   471    607   !IF $(USE_AMALGAMATION)==0
   472    608   TCC = $(TCC) -I$(TOP)\ext\fts3
   473    609   RCC = $(RCC) -I$(TOP)\ext\fts3
   474    610   TCC = $(TCC) -I$(TOP)\ext\rtree
   475    611   RCC = $(RCC) -I$(TOP)\ext\rtree
          612  +TCC = $(TCC) -I$(TOP)\ext\session
          613  +RCC = $(RCC) -I$(TOP)\ext\session
   476    614   !ENDIF
   477    615   
   478    616   # The mksqlite3c.tcl script accepts some options on the command
   479    617   # line.  When compiling with debugging enabled, some of these
   480    618   # options are necessary in order to allow debugging symbols to
   481    619   # work correctly with Visual Studio when using the amalgamation.
   482    620   #
................................................................................
   483    621   !IFNDEF MKSQLITE3C_ARGS
   484    622   !IF $(DEBUG)>1
   485    623   MKSQLITE3C_ARGS = --linemacros
   486    624   !ELSE
   487    625   MKSQLITE3C_ARGS =
   488    626   !ENDIF
   489    627   !ENDIF
          628  +# <</mark>>
   490    629   
   491    630   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
   492    631   # Omitting the define will cause extra debugging code to be inserted and
   493    632   # includes extra comments when "EXPLAIN stmt" is used.
   494    633   #
   495    634   !IF $(DEBUG)==0
   496    635   TCC = $(TCC) -DNDEBUG
   497    636   BCC = $(BCC) -DNDEBUG
   498    637   RCC = $(RCC) -DNDEBUG
   499    638   !ENDIF
   500    639   
   501         -!IF $(DEBUG)>0 || $(API_ARMOR)!=0
          640  +!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0
   502    641   TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
   503    642   RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
   504    643   !ENDIF
   505    644   
   506    645   !IF $(DEBUG)>2
   507    646   TCC = $(TCC) -DSQLITE_DEBUG=1
   508    647   RCC = $(RCC) -DSQLITE_DEBUG=1
................................................................................
   547    686   #
   548    687   !IF $(DEBUG)>3
   549    688   TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
   550    689   RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
   551    690   !ENDIF
   552    691   !ENDIF
   553    692   
          693  +# <<mark>>
   554    694   # The locations of the Tcl header and library files.  Also, the library that
   555    695   # non-stubs enabled programs using Tcl must link against.  These variables
   556    696   # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
   557    697   # prior to running nmake in order to match the actual installed location and
   558    698   # version on this machine.
   559    699   #
   560    700   !IFNDEF TCLINCDIR
................................................................................
   598    738   # know the specific version we want to use.  This variable (TCLSH_CMD) may be
   599    739   # overridden via the environment prior to running nmake in order to select a
   600    740   # specific Tcl shell to use.
   601    741   #
   602    742   !IFNDEF TCLSH_CMD
   603    743   TCLSH_CMD = tclsh85
   604    744   !ENDIF
          745  +# <</mark>>
   605    746   
   606    747   # Compiler options needed for programs that use the readline() library.
   607    748   #
   608    749   !IFNDEF READLINE_FLAGS
   609    750   READLINE_FLAGS = -DHAVE_READLINE=0
   610    751   !ENDIF
   611    752   
................................................................................
   655    796   !IF $(USE_RPCRT4_LIB)!=0
   656    797   REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1
   657    798   !ENDIF
   658    799   
   659    800   # Add the required and optional SQLite compilation options into the command
   660    801   # lines used to invoke the MSVC code and resource compilers.
   661    802   #
   662         -TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS)
   663         -RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS)
          803  +TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
          804  +RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
   664    805   
   665    806   # Add in any optional parameters specified on the commane line, e.g.
   666    807   # nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1"
   667    808   #
   668    809   TCC = $(TCC) $(OPTS)
   669    810   RCC = $(RCC) $(OPTS)
   670    811   
................................................................................
   696    837   # If symbols are enabled (or compiling for debugging), enable PDBs.
   697    838   #
   698    839   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   699    840   TCC = $(TCC) -Zi
   700    841   BCC = $(BCC) -Zi
   701    842   !ENDIF
   702    843   
          844  +# <<mark>>
   703    845   # If ICU support is enabled, add the compiler options for it.
   704    846   #
   705    847   !IF $(USE_ICU)!=0
   706    848   TCC = $(TCC) -DSQLITE_ENABLE_ICU=1
   707    849   RCC = $(RCC) -DSQLITE_ENABLE_ICU=1
   708    850   TCC = $(TCC) -I$(TOP)\ext\icu
   709    851   RCC = $(RCC) -I$(TOP)\ext\icu
   710    852   TCC = $(TCC) -I$(ICUINCDIR)
   711    853   RCC = $(RCC) -I$(ICUINCDIR)
   712    854   !ENDIF
          855  +# <</mark>>
   713    856   
   714    857   # Command line prefixes for compiling code, compiling resources,
   715    858   # linking, etc.
   716    859   #
   717    860   LTCOMPILE = $(TCC) -Fo$@
   718    861   LTRCOMPILE = $(RCC) -r
   719    862   LTLIB = lib.exe
................................................................................
   783    926   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
   784    927   !ENDIF
   785    928   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
   786    929   LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
   787    930   LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
   788    931   !ENDIF
   789    932   
   790         -# When compiling for UAP, some extra linker options are also required.
          933  +# When compiling for UWP or the Windows 10 platform, some extra linker
          934  +# options are also required.
   791    935   #
   792         -!IF $(FOR_UAP)!=0
          936  +!IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0
   793    937   LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
   794    938   LTLINKOPTS = $(LTLINKOPTS) mincore.lib
   795    939   !IFDEF PSDKLIBPATH
   796    940   LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
   797    941   !ENDIF
   798    942   !ENDIF
          943  +
          944  +!IF $(FOR_WIN10)!=0
          945  +LTLINKOPTS = $(LTLINKOPTS) /guard:cf "/LIBPATH:$(UCRTLIBPATH)"
          946  +!IF $(DEBUG)>1
          947  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib
          948  +!ELSE
          949  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib
          950  +!ENDIF
          951  +!ENDIF
   799    952   
   800    953   # If either debugging or symbols are enabled, enable PDBs.
   801    954   #
   802    955   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   803    956   LDFLAGS = /DEBUG $(LDOPTS)
   804    957   !ELSE
   805    958   LDFLAGS = $(LDOPTS)
   806    959   !ENDIF
   807    960   
          961  +# <<mark>>
   808    962   # Start with the Tcl related linker options.
   809    963   #
   810    964   !IF $(NO_TCL)==0
   811    965   LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
   812    966   LTLIBS = $(LIBTCL)
   813    967   !ENDIF
   814    968   
   815    969   # If ICU support is enabled, add the linker options for it.
   816    970   #
   817    971   !IF $(USE_ICU)!=0
   818    972   LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR)
   819    973   LTLIBS = $(LTLIBS) $(LIBICU)
   820    974   !ENDIF
          975  +# <</mark>>
   821    976   
   822    977   # You should not have to change anything below this line
   823    978   ###############################################################################
   824    979   
          980  +# <<mark>>
   825    981   # Object files for the SQLite library (non-amalgamation).
   826    982   #
   827    983   LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
   828    984            backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
   829    985            callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
   830    986            expr.lo fault.lo fkey.lo \
   831    987            fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
   832    988            fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
   833    989            fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
   834    990            fts5.lo \
   835    991            func.lo global.lo hash.lo \
   836         -         icu.lo insert.lo journal.lo legacy.lo loadext.lo \
          992  +         icu.lo insert.lo legacy.lo loadext.lo \
   837    993            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   838    994            memjournal.lo \
   839    995            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   840    996            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   841    997            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   842         -         random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
          998  +         random.lo resolve.lo rowset.lo rtree.lo \
          999  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
   843   1000            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   844   1001            update.lo util.lo vacuum.lo \
   845   1002            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   846   1003            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   847   1004            utf.lo vtab.lo
         1005  +# <</mark>>
   848   1006   
   849   1007   # Object files for the amalgamation.
   850   1008   #
   851   1009   LIBOBJS1 = sqlite3.lo
   852   1010   
   853   1011   # Determine the real value of LIBOBJ based on the 'configure' script
   854   1012   #
         1013  +# <<mark>>
   855   1014   !IF $(USE_AMALGAMATION)==0
   856   1015   LIBOBJ = $(LIBOBJS0)
   857   1016   !ELSE
         1017  +# <</mark>>
   858   1018   LIBOBJ = $(LIBOBJS1)
         1019  +# <<mark>>
   859   1020   !ENDIF
         1021  +# <</mark>>
   860   1022   
   861   1023   # Determine if embedded resource compilation and usage are enabled.
   862   1024   #
   863   1025   !IF $(USE_RC)!=0
   864   1026   LIBRESOBJS = sqlite3res.lo
   865   1027   !ELSE
   866   1028   LIBRESOBJS =
   867   1029   !ENDIF
   868   1030   
   869         -# All of the source code files.
         1031  +# <<mark>>
         1032  +# Core source code files, part 1.
   870   1033   #
   871         -SRC1 = \
         1034  +SRC00 = \
   872   1035     $(TOP)\src\alter.c \
   873   1036     $(TOP)\src\analyze.c \
   874   1037     $(TOP)\src\attach.c \
   875   1038     $(TOP)\src\auth.c \
   876   1039     $(TOP)\src\backup.c \
   877   1040     $(TOP)\src\bitvec.c \
   878   1041     $(TOP)\src\btmutex.c \
   879   1042     $(TOP)\src\btree.c \
   880         -  $(TOP)\src\btree.h \
   881         -  $(TOP)\src\btreeInt.h \
   882   1043     $(TOP)\src\build.c \
   883   1044     $(TOP)\src\callback.c \
   884   1045     $(TOP)\src\complete.c \
   885   1046     $(TOP)\src\ctime.c \
   886   1047     $(TOP)\src\date.c \
   887   1048     $(TOP)\src\dbstat.c \
   888   1049     $(TOP)\src\delete.c \
   889   1050     $(TOP)\src\expr.c \
   890   1051     $(TOP)\src\fault.c \
   891   1052     $(TOP)\src\fkey.c \
   892   1053     $(TOP)\src\func.c \
   893   1054     $(TOP)\src\global.c \
   894   1055     $(TOP)\src\hash.c \
   895         -  $(TOP)\src\hash.h \
   896         -  $(TOP)\src\hwtime.h \
   897   1056     $(TOP)\src\insert.c \
   898         -  $(TOP)\src\journal.c \
   899   1057     $(TOP)\src\legacy.c \
   900   1058     $(TOP)\src\loadext.c \
   901   1059     $(TOP)\src\main.c \
   902   1060     $(TOP)\src\malloc.c \
   903   1061     $(TOP)\src\mem0.c \
   904   1062     $(TOP)\src\mem1.c \
   905   1063     $(TOP)\src\mem2.c \
   906   1064     $(TOP)\src\mem3.c \
   907   1065     $(TOP)\src\mem5.c \
   908   1066     $(TOP)\src\memjournal.c \
   909         -  $(TOP)\src\msvc.h \
   910   1067     $(TOP)\src\mutex.c \
   911         -  $(TOP)\src\mutex.h \
   912   1068     $(TOP)\src\mutex_noop.c \
   913   1069     $(TOP)\src\mutex_unix.c \
   914   1070     $(TOP)\src\mutex_w32.c \
   915   1071     $(TOP)\src\notify.c \
   916   1072     $(TOP)\src\os.c \
   917         -  $(TOP)\src\os.h \
   918         -  $(TOP)\src\os_common.h \
   919         -  $(TOP)\src\os_setup.h \
   920   1073     $(TOP)\src\os_unix.c \
   921         -  $(TOP)\src\os_win.c \
   922         -  $(TOP)\src\os_win.h
   923         -SRC2 = \
         1074  +  $(TOP)\src\os_win.c
         1075  +
         1076  +# Core source code files, part 2.
         1077  +#
         1078  +SRC01 = \
   924   1079     $(TOP)\src\pager.c \
   925         -  $(TOP)\src\pager.h \
   926         -  $(TOP)\src\parse.y \
   927   1080     $(TOP)\src\pcache.c \
   928         -  $(TOP)\src\pcache.h \
   929   1081     $(TOP)\src\pcache1.c \
   930   1082     $(TOP)\src\pragma.c \
   931         -  $(TOP)\src\pragma.h \
   932   1083     $(TOP)\src\prepare.c \
   933   1084     $(TOP)\src\printf.c \
   934   1085     $(TOP)\src\random.c \
   935   1086     $(TOP)\src\resolve.c \
   936   1087     $(TOP)\src\rowset.c \
   937   1088     $(TOP)\src\select.c \
   938   1089     $(TOP)\src\status.c \
   939         -  $(TOP)\src\shell.c \
   940         -  $(TOP)\src\sqlite.h.in \
   941         -  $(TOP)\src\sqlite3ext.h \
   942         -  $(TOP)\src\sqliteInt.h \
   943         -  $(TOP)\src\sqliteLimit.h \
   944   1090     $(TOP)\src\table.c \
   945   1091     $(TOP)\src\threads.c \
   946   1092     $(TOP)\src\tclsqlite.c \
   947   1093     $(TOP)\src\tokenize.c \
   948   1094     $(TOP)\src\treeview.c \
   949   1095     $(TOP)\src\trigger.c \
   950   1096     $(TOP)\src\utf.c \
   951   1097     $(TOP)\src\update.c \
   952   1098     $(TOP)\src\util.c \
   953   1099     $(TOP)\src\vacuum.c \
   954   1100     $(TOP)\src\vdbe.c \
   955         -  $(TOP)\src\vdbe.h \
   956   1101     $(TOP)\src\vdbeapi.c \
   957   1102     $(TOP)\src\vdbeaux.c \
   958   1103     $(TOP)\src\vdbeblob.c \
   959   1104     $(TOP)\src\vdbemem.c \
   960   1105     $(TOP)\src\vdbesort.c \
   961   1106     $(TOP)\src\vdbetrace.c \
   962         -  $(TOP)\src\vdbeInt.h \
   963   1107     $(TOP)\src\vtab.c \
   964         -  $(TOP)\src\vxworks.h \
   965   1108     $(TOP)\src\wal.c \
   966         -  $(TOP)\src\wal.h \
   967   1109     $(TOP)\src\walker.c \
   968   1110     $(TOP)\src\where.c \
   969   1111     $(TOP)\src\wherecode.c \
   970         -  $(TOP)\src\whereexpr.c \
         1112  +  $(TOP)\src\whereexpr.c
         1113  +
         1114  +# Shell source code files.
         1115  +#
         1116  +SRC02 = \
         1117  +  $(TOP)\src\shell.c
         1118  +
         1119  +# Core miscellaneous files.
         1120  +#
         1121  +SRC03 = \
         1122  +  $(TOP)\src\parse.y
         1123  +
         1124  +# Core header files, part 1.
         1125  +#
         1126  +SRC04 = \
         1127  +  $(TOP)\src\btree.h \
         1128  +  $(TOP)\src\btreeInt.h \
         1129  +  $(TOP)\src\hash.h \
         1130  +  $(TOP)\src\hwtime.h \
         1131  +  $(TOP)\src\msvc.h \
         1132  +  $(TOP)\src\mutex.h \
         1133  +  $(TOP)\src\os.h \
         1134  +  $(TOP)\src\os_common.h \
         1135  +  $(TOP)\src\os_setup.h \
         1136  +  $(TOP)\src\os_win.h
         1137  +
         1138  +# Core header files, part 2.
         1139  +#
         1140  +SRC05 = \
         1141  +  $(TOP)\src\pager.h \
         1142  +  $(TOP)\src\pcache.h \
         1143  +  $(TOP)\src\pragma.h \
         1144  +  $(TOP)\src\sqlite.h.in \
         1145  +  $(TOP)\src\sqlite3ext.h \
         1146  +  $(TOP)\src\sqliteInt.h \
         1147  +  $(TOP)\src\sqliteLimit.h \
         1148  +  $(TOP)\src\vdbe.h \
         1149  +  $(TOP)\src\vdbeInt.h \
         1150  +  $(TOP)\src\vxworks.h \
         1151  +  $(TOP)\src\wal.h \
   971   1152     $(TOP)\src\whereInt.h
   972   1153   
   973         -# Source code for extensions
         1154  +# Extension source code files, part 1.
   974   1155   #
   975         -SRC3 = \
         1156  +SRC06 = \
   976   1157     $(TOP)\ext\fts1\fts1.c \
   977         -  $(TOP)\ext\fts1\fts1.h \
   978   1158     $(TOP)\ext\fts1\fts1_hash.c \
   979         -  $(TOP)\ext\fts1\fts1_hash.h \
   980   1159     $(TOP)\ext\fts1\fts1_porter.c \
   981         -  $(TOP)\ext\fts1\fts1_tokenizer.h \
   982   1160     $(TOP)\ext\fts1\fts1_tokenizer1.c \
   983   1161     $(TOP)\ext\fts2\fts2.c \
   984         -  $(TOP)\ext\fts2\fts2.h \
   985   1162     $(TOP)\ext\fts2\fts2_hash.c \
   986         -  $(TOP)\ext\fts2\fts2_hash.h \
   987   1163     $(TOP)\ext\fts2\fts2_icu.c \
   988   1164     $(TOP)\ext\fts2\fts2_porter.c \
   989         -  $(TOP)\ext\fts2\fts2_tokenizer.h \
   990   1165     $(TOP)\ext\fts2\fts2_tokenizer.c \
   991   1166     $(TOP)\ext\fts2\fts2_tokenizer1.c
   992         -SRC4 = \
         1167  +
         1168  +# Extension source code files, part 2.
         1169  +#
         1170  +SRC07 = \
   993   1171     $(TOP)\ext\fts3\fts3.c \
   994         -  $(TOP)\ext\fts3\fts3.h \
   995         -  $(TOP)\ext\fts3\fts3Int.h \
   996   1172     $(TOP)\ext\fts3\fts3_aux.c \
   997   1173     $(TOP)\ext\fts3\fts3_expr.c \
   998   1174     $(TOP)\ext\fts3\fts3_hash.c \
   999         -  $(TOP)\ext\fts3\fts3_hash.h \
  1000   1175     $(TOP)\ext\fts3\fts3_icu.c \
  1001   1176     $(TOP)\ext\fts3\fts3_porter.c \
  1002   1177     $(TOP)\ext\fts3\fts3_snippet.c \
  1003         -  $(TOP)\ext\fts3\fts3_tokenizer.h \
  1004   1178     $(TOP)\ext\fts3\fts3_tokenizer.c \
  1005   1179     $(TOP)\ext\fts3\fts3_tokenizer1.c \
  1006   1180     $(TOP)\ext\fts3\fts3_tokenize_vtab.c \
  1007   1181     $(TOP)\ext\fts3\fts3_unicode.c \
  1008   1182     $(TOP)\ext\fts3\fts3_unicode2.c \
  1009   1183     $(TOP)\ext\fts3\fts3_write.c \
  1010         -  $(TOP)\ext\icu\sqliteicu.h \
  1011   1184     $(TOP)\ext\icu\icu.c \
  1012         -  $(TOP)\ext\rtree\rtree.h \
  1013   1185     $(TOP)\ext\rtree\rtree.c \
  1014         -  $(TOP)\ext\rbu\sqlite3rbu.h \
         1186  +  $(TOP)\ext\session\sqlite3session.c \
  1015   1187     $(TOP)\ext\rbu\sqlite3rbu.c \
  1016   1188     $(TOP)\ext\misc\json1.c
  1017   1189   
         1190  +# Extension header files, part 1.
         1191  +#
         1192  +SRC08 = \
         1193  +  $(TOP)\ext\fts1\fts1.h \
         1194  +  $(TOP)\ext\fts1\fts1_hash.h \
         1195  +  $(TOP)\ext\fts1\fts1_tokenizer.h \
         1196  +  $(TOP)\ext\fts2\fts2.h \
         1197  +  $(TOP)\ext\fts2\fts2_hash.h \
         1198  +  $(TOP)\ext\fts2\fts2_tokenizer.h
         1199  +
         1200  +# Extension header files, part 2.
         1201  +#
         1202  +SRC09 = \
         1203  +  $(TOP)\ext\fts3\fts3.h \
         1204  +  $(TOP)\ext\fts3\fts3Int.h \
         1205  +  $(TOP)\ext\fts3\fts3_hash.h \
         1206  +  $(TOP)\ext\fts3\fts3_tokenizer.h \
         1207  +  $(TOP)\ext\icu\sqliteicu.h \
         1208  +  $(TOP)\ext\rtree\rtree.h \
         1209  +  $(TOP)\ext\rbu\sqlite3rbu.h \
         1210  +  $(TOP)\ext\session\sqlite3session.h
  1018   1211   
  1019   1212   # Generated source code files
  1020   1213   #
  1021         -SRC5 = \
  1022         -  keywordhash.h \
         1214  +SRC10 = \
  1023   1215     opcodes.c \
         1216  +  parse.c
         1217  +
         1218  +# Generated header files
         1219  +#
         1220  +SRC11 = \
         1221  +  keywordhash.h \
  1024   1222     opcodes.h \
  1025         -  parse.c \
  1026   1223     parse.h \
  1027         -  sqlite3.h
         1224  +  $(SQLITE3H)
  1028   1225   
  1029   1226   # All source code files.
  1030   1227   #
  1031         -SRC = $(SRC1) $(SRC2) $(SRC3) $(SRC4) $(SRC5)
         1228  +SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
  1032   1229   
  1033   1230   # Source code to the test files.
  1034   1231   #
  1035   1232   TESTSRC = \
  1036   1233     $(TOP)\src\test1.c \
  1037   1234     $(TOP)\src\test2.c \
  1038   1235     $(TOP)\src\test3.c \
................................................................................
  1041   1238     $(TOP)\src\test6.c \
  1042   1239     $(TOP)\src\test7.c \
  1043   1240     $(TOP)\src\test8.c \
  1044   1241     $(TOP)\src\test9.c \
  1045   1242     $(TOP)\src\test_autoext.c \
  1046   1243     $(TOP)\src\test_async.c \
  1047   1244     $(TOP)\src\test_backup.c \
         1245  +  $(TOP)\src\test_bestindex.c \
  1048   1246     $(TOP)\src\test_blob.c \
  1049   1247     $(TOP)\src\test_btree.c \
  1050   1248     $(TOP)\src\test_config.c \
  1051   1249     $(TOP)\src\test_demovfs.c \
  1052   1250     $(TOP)\src\test_devsym.c \
  1053   1251     $(TOP)\src\test_fs.c \
  1054   1252     $(TOP)\src\test_func.c \
................................................................................
  1071   1269     $(TOP)\src\test_tclvar.c \
  1072   1270     $(TOP)\src\test_thread.c \
  1073   1271     $(TOP)\src\test_vfs.c \
  1074   1272     $(TOP)\src\test_windirent.c \
  1075   1273     $(TOP)\src\test_wsd.c \
  1076   1274     $(TOP)\ext\fts3\fts3_term.c \
  1077   1275     $(TOP)\ext\fts3\fts3_test.c \
  1078         -  $(TOP)\ext\rbu\test_rbu.c
         1276  +  $(TOP)\ext\rbu\test_rbu.c \
         1277  +  $(TOP)\ext\session\test_session.c
  1079   1278   
  1080         -# Statically linked extensions
         1279  +# Statically linked extensions.
  1081   1280   #
  1082   1281   TESTEXT = \
  1083   1282     $(TOP)\ext\misc\amatch.c \
  1084   1283     $(TOP)\ext\misc\closure.c \
  1085   1284     $(TOP)\ext\misc\eval.c \
  1086   1285     $(TOP)\ext\misc\fileio.c \
  1087   1286     $(TOP)\ext\misc\fuzzer.c \
  1088   1287     $(TOP)\ext\fts5\fts5_tcl.c \
  1089   1288     $(TOP)\ext\fts5\fts5_test_mi.c \
         1289  +  $(TOP)\ext\fts5\fts5_test_tok.c \
  1090   1290     $(TOP)\ext\misc\ieee754.c \
  1091   1291     $(TOP)\ext\misc\nextchar.c \
  1092   1292     $(TOP)\ext\misc\percentile.c \
  1093   1293     $(TOP)\ext\misc\regexp.c \
  1094   1294     $(TOP)\ext\misc\series.c \
  1095   1295     $(TOP)\ext\misc\spellfix.c \
  1096   1296     $(TOP)\ext\misc\totype.c \
  1097   1297     $(TOP)\ext\misc\wholenumber.c
  1098   1298   
  1099         -
  1100   1299   # Source code to the library files needed by the test fixture
         1300  +# (non-amalgamation)
  1101   1301   #
  1102   1302   TESTSRC2 = \
  1103         -  $(TOP)\src\attach.c \
  1104         -  $(TOP)\src\backup.c \
  1105         -  $(TOP)\src\bitvec.c \
  1106         -  $(TOP)\src\btree.c \
  1107         -  $(TOP)\src\build.c \
  1108         -  $(TOP)\src\ctime.c \
  1109         -  $(TOP)\src\date.c \
  1110         -  $(TOP)\src\dbstat.c \
  1111         -  $(TOP)\src\expr.c \
  1112         -  $(TOP)\src\func.c \
  1113         -  $(TOP)\src\insert.c \
  1114         -  $(TOP)\src\wal.c \
  1115         -  $(TOP)\src\main.c \
  1116         -  $(TOP)\src\mem5.c \
  1117         -  $(TOP)\src\os.c \
  1118         -  $(TOP)\src\os_unix.c \
  1119         -  $(TOP)\src\os_win.c \
  1120         -  $(TOP)\src\pager.c \
  1121         -  $(TOP)\src\pragma.c \
  1122         -  $(TOP)\src\prepare.c \
  1123         -  $(TOP)\src\printf.c \
  1124         -  $(TOP)\src\random.c \
  1125         -  $(TOP)\src\pcache.c \
  1126         -  $(TOP)\src\pcache1.c \
  1127         -  $(TOP)\src\select.c \
  1128         -  $(TOP)\src\tokenize.c \
  1129         -  $(TOP)\src\utf.c \
  1130         -  $(TOP)\src\util.c \
  1131         -  $(TOP)\src\vdbeapi.c \
  1132         -  $(TOP)\src\vdbeaux.c \
  1133         -  $(TOP)\src\vdbe.c \
  1134         -  $(TOP)\src\vdbemem.c \
  1135         -  $(TOP)\src\vdbesort.c \
  1136         -  $(TOP)\src\vdbetrace.c \
  1137         -  $(TOP)\src\where.c \
  1138         -  $(TOP)\src\wherecode.c \
  1139         -  $(TOP)\src\whereexpr.c \
  1140         -  parse.c \
  1141         -  $(TOP)\ext\fts3\fts3.c \
  1142         -  $(TOP)\ext\fts3\fts3_aux.c \
  1143         -  $(TOP)\ext\fts3\fts3_expr.c \
  1144         -  $(TOP)\ext\fts3\fts3_tokenizer.c \
  1145         -  $(TOP)\ext\fts3\fts3_tokenize_vtab.c \
  1146         -  $(TOP)\ext\fts3\fts3_unicode.c \
  1147         -  $(TOP)\ext\fts3\fts3_unicode2.c \
  1148         -  $(TOP)\ext\fts3\fts3_write.c \
         1303  +  $(SRC00) \
         1304  +  $(SRC01) \
         1305  +  $(SRC06) \
         1306  +  $(SRC07) \
         1307  +  $(SRC10) \
  1149   1308     $(TOP)\ext\async\sqlite3async.c
  1150   1309   
  1151   1310   # Header files used by all library source files.
  1152   1311   #
  1153   1312   HDR = \
  1154   1313      $(TOP)\src\btree.h \
  1155   1314      $(TOP)\src\btreeInt.h \
................................................................................
  1163   1322      $(TOP)\src\os_common.h \
  1164   1323      $(TOP)\src\os_setup.h \
  1165   1324      $(TOP)\src\os_win.h \
  1166   1325      $(TOP)\src\pager.h \
  1167   1326      $(TOP)\src\pcache.h \
  1168   1327      parse.h \
  1169   1328      $(TOP)\src\pragma.h \
  1170         -   sqlite3.h \
         1329  +   $(SQLITE3H) \
  1171   1330      $(TOP)\src\sqlite3ext.h \
  1172   1331      $(TOP)\src\sqliteInt.h \
  1173   1332      $(TOP)\src\sqliteLimit.h \
  1174   1333      $(TOP)\src\vdbe.h \
  1175   1334      $(TOP)\src\vdbeInt.h \
  1176   1335      $(TOP)\src\vxworks.h \
  1177   1336      $(TOP)\src\whereInt.h
................................................................................
  1193   1352     $(TOP)\ext\fts3\fts3_tokenizer.h
  1194   1353   EXTHDR = $(EXTHDR) \
  1195   1354     $(TOP)\ext\rtree\rtree.h
  1196   1355   EXTHDR = $(EXTHDR) \
  1197   1356     $(TOP)\ext\icu\sqliteicu.h
  1198   1357   EXTHDR = $(EXTHDR) \
  1199   1358     $(TOP)\ext\rtree\sqlite3rtree.h
         1359  +EXTHDR = $(EXTHDR) \
         1360  +  $(TOP)\ext\session\sqlite3session.h
  1200   1361   
  1201   1362   # executables needed for testing
  1202   1363   #
  1203   1364   TESTPROGS = \
  1204   1365     testfixture.exe \
  1205         -  sqlite3.exe \
         1366  +  $(SQLITE3EXE) \
  1206   1367     sqlite3_analyzer.exe \
  1207   1368     sqldiff.exe
  1208   1369   
  1209   1370   # Databases containing fuzzer test cases
  1210   1371   #
  1211   1372   FUZZDATA = \
  1212   1373     $(TOP)\test\fuzzdata1.db \
  1213   1374     $(TOP)\test\fuzzdata2.db \
  1214   1375     $(TOP)\test\fuzzdata3.db \
  1215   1376     $(TOP)\test\fuzzdata4.db
         1377  +# <</mark>>
  1216   1378   
  1217         -# Extra compiler options for various shell tools
         1379  +# Additional compiler options for the shell.  These are only effective
         1380  +# when the shell is not being dynamically linked.
  1218   1381   #
  1219         -SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
         1382  +!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
         1383  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
         1384  +!ENDIF
         1385  +
         1386  +# <<mark>>
         1387  +# Extra compiler options for various test tools.
         1388  +#
         1389  +MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
  1220   1390   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
  1221   1391   FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
  1222   1392   
  1223         -# Standard options to testfixture
         1393  +# Standard options to testfixture.
  1224   1394   #
  1225   1395   TESTOPTS = --verbose=file --output=test-out.txt
         1396  +
         1397  +# Extra targets for the "all" target that require Tcl.
         1398  +#
         1399  +!IF $(NO_TCL)==0
         1400  +ALL_TCL_TARGETS = libtclsqlite3.lib
         1401  +!ELSE
         1402  +ALL_TCL_TARGETS =
         1403  +!ENDIF
         1404  +# <</mark>>
  1226   1405   
  1227   1406   # This is the default Makefile target.  The objects listed here
  1228   1407   # are what get build when you type just "make" with no arguments.
  1229   1408   #
  1230         -all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib
         1409  +all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
  1231   1410   
         1411  +# Dynamic link library section.
         1412  +#
         1413  +dll:	$(SQLITE3DLL)
         1414  +
         1415  +# Shell executable.
         1416  +#
         1417  +shell:	$(SQLITE3EXE)
         1418  +
         1419  +# <<mark>>
  1232   1420   libsqlite3.lib:	$(LIBOBJ)
  1233   1421   	$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)
  1234   1422   
  1235   1423   libtclsqlite3.lib:	tclsqlite.lo libsqlite3.lib
  1236   1424   	$(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS)
  1237         -
  1238         -sqlite3.exe:	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h
  1239         -	$(LTLINK) $(SHELL_COMPILE_OPTS) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \
  1240         -		/link /pdb:sqlite3sh.pdb $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1241         -
  1242         -sqldiff.exe:	$(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h
  1243         -	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS)
  1244         -
  1245         -fuzzershell.exe:	$(TOP)\tool\fuzzershell.c sqlite3.c sqlite3.h
  1246         -	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) \
  1247         -	  $(TOP)\tool\fuzzershell.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS)
  1248         -
  1249         -fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c sqlite3.c sqlite3.h
  1250         -	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS)
  1251         -
  1252         -mptester.exe:	$(TOP)\mptest\mptest.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h
  1253         -	$(LTLINK) $(NO_WARN) $(SHELL_COMPILE_OPTS) $(TOP)\mptest\mptest.c \
  1254         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(SHELL_LINK_OPTS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
         1425  +# <</mark>>
         1426  +
         1427  +$(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
         1428  +	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         1429  +
         1430  +# <<block2>>
         1431  +sqlite3.def:	libsqlite3.lib
         1432  +	echo EXPORTS > sqlite3.def
         1433  +	dumpbin /all libsqlite3.lib \
         1434  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \
         1435  +		| sort >> sqlite3.def
         1436  +# <</block2>>
         1437  +
         1438  +$(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
         1439  +	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
         1440  +		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
         1441  +
         1442  +# <<mark>>
         1443  +sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
         1444  +	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1445  +
         1446  +srcck1.exe:	$(TOP)\tool\srcck1.c
         1447  +	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
         1448  +
         1449  +sourcetest:	srcck1.exe sqlite3.c
         1450  +	srcck1.exe sqlite3.c
         1451  +
         1452  +fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
         1453  +	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1454  +
         1455  +fuzzcheck.exe:	$(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H)
         1456  +	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1457  +
         1458  +mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
         1459  +	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1255   1460   
  1256   1461   MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20
  1257   1462   MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20
  1258   1463   
  1259   1464   mptest:	mptester.exe
  1260   1465   	del /Q mptest.db 2>NUL
  1261   1466   	$(MPTEST1) --journalmode DELETE
................................................................................
  1272   1477   # build on the target system.  Some of the C source code and header
  1273   1478   # files are automatically generated.  This target takes care of
  1274   1479   # all that automatic generation.
  1275   1480   #
  1276   1481   .target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c
  1277   1482   	-rmdir /Q/S tsrc 2>NUL
  1278   1483   	-mkdir tsrc
  1279         -	for %i in ($(SRC1)) do copy /Y %i tsrc
  1280         -	for %i in ($(SRC2)) do copy /Y %i tsrc
  1281         -	for %i in ($(SRC3)) do copy /Y %i tsrc
  1282         -	for %i in ($(SRC4)) do copy /Y %i tsrc
  1283         -	for %i in ($(SRC5)) do copy /Y %i tsrc
         1484  +	for %i in ($(SRC00)) do copy /Y %i tsrc
         1485  +	for %i in ($(SRC01)) do copy /Y %i tsrc
         1486  +	for %i in ($(SRC02)) do copy /Y %i tsrc
         1487  +	for %i in ($(SRC03)) do copy /Y %i tsrc
         1488  +	for %i in ($(SRC04)) do copy /Y %i tsrc
         1489  +	for %i in ($(SRC05)) do copy /Y %i tsrc
         1490  +	for %i in ($(SRC06)) do copy /Y %i tsrc
         1491  +	for %i in ($(SRC07)) do copy /Y %i tsrc
         1492  +	for %i in ($(SRC08)) do copy /Y %i tsrc
         1493  +	for %i in ($(SRC09)) do copy /Y %i tsrc
         1494  +	for %i in ($(SRC10)) do copy /Y %i tsrc
         1495  +	for %i in ($(SRC11)) do copy /Y %i tsrc
  1284   1496   	copy /Y fts5.c tsrc
  1285   1497   	copy /Y fts5.h tsrc
  1286   1498   	del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL
  1287   1499   	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new
  1288   1500   	move vdbe.new tsrc\vdbe.c
  1289   1501   	echo > .target_source
  1290   1502   
  1291   1503   sqlite3.c:	.target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl
  1292   1504   	$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS)
  1293   1505   	copy tsrc\shell.c .
         1506  +	copy $(TOP)\ext\session\sqlite3session.h .
  1294   1507   
  1295   1508   sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
  1296   1509   	$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
  1297         -
  1298         -# Set the source code file to be used by executables and libraries when
  1299         -# they need the amalgamation.
  1300         -#
  1301         -!IF $(SPLIT_AMALGAMATION)!=0
  1302         -SQLITE3C = sqlite3-all.c
  1303         -!ELSE
  1304         -SQLITE3C = sqlite3.c
  1305         -!ENDIF
         1510  +# <</mark>>
  1306   1511   
  1307   1512   # Rule to build the amalgamation
  1308   1513   #
  1309   1514   sqlite3.lo:	$(SQLITE3C)
  1310   1515   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)
  1311   1516   
         1517  +# <<mark>>
  1312   1518   # Rules to build the LEMON compiler generator
  1313   1519   #
  1314   1520   lempar.c:	$(TOP)\tool\lempar.c
  1315   1521   	copy $(TOP)\tool\lempar.c .
  1316   1522   
  1317   1523   lemon.exe:	$(TOP)\tool\lemon.c lempar.c
  1318   1524   	$(BCC) $(NO_WARN) -Daccess=_access \
................................................................................
  1325   1531   #     opcodes.lo
  1326   1532   #
  1327   1533   parse.lo:	parse.c $(HDR)
  1328   1534   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c parse.c
  1329   1535   
  1330   1536   opcodes.lo:	opcodes.c
  1331   1537   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c opcodes.c
         1538  +# <</mark>>
  1332   1539   
  1333   1540   # Rule to build the Win32 resources object file.
  1334   1541   #
  1335   1542   !IF $(USE_RC)!=0
  1336         -$(LIBRESOBJS):	$(TOP)\src\sqlite3.rc $(HDR)
         1543  +# <<block1>>
         1544  +$(LIBRESOBJS):	$(TOP)\src\sqlite3.rc $(SQLITE3H)
  1337   1545   	echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h
  1338   1546   	for /F %%V in ('type "$(TOP)\VERSION"') do ( \
  1339   1547   		echo #define SQLITE_RESOURCE_VERSION %%V \
  1340   1548   			| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \
  1341   1549   	)
  1342   1550   	echo #endif >> sqlite3rc.h
  1343   1551   	$(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc
         1552  +# <</block1>>
  1344   1553   !ENDIF
  1345   1554   
         1555  +# <<mark>>
  1346   1556   # Rules to build individual *.lo files from files in the src directory.
  1347   1557   #
  1348   1558   alter.lo:	$(TOP)\src\alter.c $(HDR)
  1349   1559   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\alter.c
  1350   1560   
  1351   1561   analyze.lo:	$(TOP)\src\analyze.c $(HDR)
  1352   1562   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\analyze.c
................................................................................
  1407   1617   
  1408   1618   hash.lo:	$(TOP)\src\hash.c $(HDR)
  1409   1619   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\hash.c
  1410   1620   
  1411   1621   insert.lo:	$(TOP)\src\insert.c $(HDR)
  1412   1622   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\insert.c
  1413   1623   
  1414         -journal.lo:	$(TOP)\src\journal.c $(HDR)
  1415         -	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\journal.c
  1416         -
  1417   1624   legacy.lo:	$(TOP)\src\legacy.c $(HDR)
  1418   1625   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c
  1419   1626   
  1420   1627   loadext.lo:	$(TOP)\src\loadext.c $(HDR)
  1421   1628   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\loadext.c
  1422   1629   
  1423   1630   main.lo:	$(TOP)\src\main.c $(HDR)
................................................................................
  1569   1776   
  1570   1777   tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR)
  1571   1778   	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1572   1779   
  1573   1780   tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR)
  1574   1781   	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1575   1782   
  1576         -tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(LIBRESOBJS)
         1783  +tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
  1577   1784   	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1578   1785   
  1579   1786   # Rules to build opcodes.c and opcodes.h
  1580   1787   #
  1581   1788   opcodes.c:	opcodes.h $(TOP)\tool\mkopcodec.tcl
  1582   1789   	$(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c
  1583   1790   
................................................................................
  1587   1794   # Rules to build parse.c and parse.h - the outputs of lemon.
  1588   1795   #
  1589   1796   parse.h:	parse.c
  1590   1797   
  1591   1798   parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl
  1592   1799   	del /Q parse.y parse.h parse.h.temp 2>NUL
  1593   1800   	copy $(TOP)\src\parse.y .
  1594         -	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
         1801  +	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y
  1595   1802   	move parse.h parse.h.temp
  1596   1803   	$(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h
  1597   1804   
  1598         -sqlite3.h:	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
  1599         -	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h
         1805  +$(SQLITE3H):	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
         1806  +	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H)
  1600   1807   
  1601         -sqlite3ext.h: .target_source
         1808  +sqlite3ext.h:	.target_source
  1602   1809   	copy tsrc\sqlite3ext.h .
  1603   1810   
  1604   1811   mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
  1605         -	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) \
         1812  +	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \
  1606   1813   		$(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
  1607   1814   
  1608   1815   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
  1609   1816   	.\mkkeywordhash.exe > keywordhash.h
  1610   1817   
  1611   1818   
  1612   1819   
................................................................................
  1671   1878   
  1672   1879   fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
  1673   1880   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
  1674   1881   
  1675   1882   rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
  1676   1883   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
  1677   1884   
         1885  +sqlite3session.lo:	$(TOP)\ext\session\sqlite3sesion.c $(HDR) $(EXTHDR)
         1886  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c
         1887  +
  1678   1888   # FTS5 things
  1679   1889   #
  1680   1890   FTS5_SRC = \
  1681   1891      $(TOP)\ext\fts5\fts5.h \
  1682   1892      $(TOP)\ext\fts5\fts5Int.h \
  1683   1893      $(TOP)\ext\fts5\fts5_aux.c \
  1684   1894      $(TOP)\ext\fts5\fts5_buffer.c \
................................................................................
  1693   1903      $(TOP)\ext\fts5\fts5_unicode2.c \
  1694   1904      $(TOP)\ext\fts5\fts5_varint.c \
  1695   1905      $(TOP)\ext\fts5\fts5_vocab.c
  1696   1906   
  1697   1907   fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
  1698   1908   	copy $(TOP)\ext\fts5\fts5parse.y .
  1699   1909   	del /Q fts5parse.h 2>NUL
  1700         -	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) fts5parse.y
         1910  +	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y
  1701   1911   
  1702         -fts5parse.h: fts5parse.c
         1912  +fts5parse.h:	fts5parse.c
  1703   1913   
  1704         -fts5.c: $(FTS5_SRC)
         1914  +fts5.c:	$(FTS5_SRC)
  1705   1915   	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
  1706   1916   	copy $(TOP)\ext\fts5\fts5.h .
  1707   1917   
  1708   1918   fts5.lo:	fts5.c $(HDR) $(EXTHDR)
  1709   1919   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
  1710   1920   
  1711   1921   fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
  1712   1922   	$(LTCOMPILE) $(NO_WARN) -c fts5.c
  1713   1923   
  1714   1924   fts5.dll:	fts5_ext.lo
  1715   1925   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo
  1716   1926   
         1927  +sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
         1928  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rbu\sqlite3rbu.c
         1929  +
  1717   1930   # Rules to build the 'testfixture' application.
  1718   1931   #
  1719   1932   # If using the amalgamation, use sqlite3.c directly to build the test
  1720   1933   # fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
  1721   1934   # necessary because the test fixture requires non-API symbols which are
  1722   1935   # hidden when the library is built via the amalgamation).
  1723   1936   #
  1724   1937   TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1725   1938   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  1726   1939   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
         1940  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
         1941  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
  1727   1942   
  1728         -TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) $(SHELL_CORE_DEP)
         1943  +TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  1729   1944   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  1730   1945   !IF $(USE_AMALGAMATION)==0
  1731   1946   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  1732   1947   !ELSE
  1733   1948   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)
  1734   1949   !ENDIF
  1735   1950   
  1736         -testfixture.exe:	$(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR)
         1951  +testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR)
  1737   1952   	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
  1738   1953   		-DBUILD_sqlite -I$(TCLINCDIR) \
  1739   1954   		$(TESTFIXTURE_SRC) \
  1740   1955   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1741   1956   
  1742         -extensiontest: testfixture.exe testloadext.dll
         1957  +extensiontest:	testfixture.exe testloadext.dll
  1743   1958   	@set PATH=$(LIBTCLPATH);$(PATH)
  1744   1959   	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
  1745   1960   
  1746   1961   fulltest:	$(TESTPROGS) fuzztest
  1747   1962   	@set PATH=$(LIBTCLPATH);$(PATH)
  1748   1963   	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
  1749   1964   
................................................................................
  1751   1966   	@set PATH=$(LIBTCLPATH);$(PATH)
  1752   1967   	.\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS)
  1753   1968   
  1754   1969   fulltestonly:	$(TESTPROGS) fuzztest
  1755   1970   	@set PATH=$(LIBTCLPATH);$(PATH)
  1756   1971   	.\testfixture.exe $(TOP)\test\full.test
  1757   1972   
  1758         -queryplantest:	testfixture.exe sqlite3.exe
         1973  +queryplantest:	testfixture.exe shell
  1759   1974   	@set PATH=$(LIBTCLPATH);$(PATH)
  1760   1975   	.\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS)
  1761   1976   
  1762   1977   fuzztest:	fuzzcheck.exe
  1763   1978   	.\fuzzcheck.exe $(FUZZDATA)
  1764   1979   
  1765   1980   fastfuzztest:	fuzzcheck.exe
  1766   1981   	.\fuzzcheck.exe --limit-mem 100M $(FUZZDATA)
  1767   1982   
  1768   1983   # Minimal testing that runs in less than 3 minutes (on a fast machine)
  1769   1984   #
  1770         -quicktest:	testfixture.exe
         1985  +quicktest:	testfixture.exe sourcetest
  1771   1986   	@set PATH=$(LIBTCLPATH);$(PATH)
  1772   1987   	.\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS)
  1773   1988   
  1774   1989   # This is the common case.  Run many tests that do not take too long,
  1775   1990   # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
  1776   1991   #
  1777         -test:	$(TESTPROGS) fastfuzztest
         1992  +test:	$(TESTPROGS) sourcetest fastfuzztest
  1778   1993   	@set PATH=$(LIBTCLPATH);$(PATH)
  1779   1994   	.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)
  1780   1995   
  1781   1996   smoketest:	$(TESTPROGS)
  1782   1997   	@set PATH=$(LIBTCLPATH);$(PATH)
  1783   1998   	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
  1784   1999   
  1785         -sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
         2000  +sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
  1786   2001   	echo #define TCLSH 2 > $@
  1787   2002   	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
  1788   2003   	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
  1789   2004   	echo static const char *tclsh_main_loop(void){ >> $@
  1790   2005   	echo static const char *zMainloop = >> $@
  1791   2006   	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
  1792   2007   	echo ; return zMainloop; } >> $@
................................................................................
  1794   2009   sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
  1795   2010   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
  1796   2011   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1797   2012   
  1798   2013   testloadext.lo:	$(TOP)\src\test_loadext.c
  1799   2014   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
  1800   2015   
  1801         -testloadext.dll: testloadext.lo
         2016  +testloadext.dll:	testloadext.lo
  1802   2017   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  1803   2018   
  1804         -showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C)
         2019  +showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
  1805   2020   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1806   2021   		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1807   2022   
  1808         -showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C)
         2023  +showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
  1809   2024   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1810   2025   		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1811   2026   
  1812         -showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C)
         2027  +showjournal.exe:	$(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H)
  1813   2028   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1814   2029   		$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1815   2030   
  1816         -showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C)
         2031  +showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  1817   2032   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1818   2033   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1819   2034   
  1820         -fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C)
         2035  +changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C)
         2036  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
         2037  +		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2038  +
         2039  +fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
  1821   2040   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1822   2041   		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1823   2042   
  1824         -rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C)
         2043  +rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
  1825   2044   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1826   2045   		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1827   2046   
  1828         -LogEst.exe:	$(TOP)\tool\logest.c sqlite3.h
         2047  +LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
  1829   2048   	$(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
  1830   2049   
  1831         -wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C)
         2050  +wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H)
  1832   2051   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1833   2052   		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1834   2053   
  1835         -speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C)
         2054  +speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H)
  1836   2055   	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1837   2056   		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1838   2057   
         2058  +rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
         2059  +	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \
         2060  +		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2061  +
         2062  +moreclean:	clean
         2063  +	del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL
         2064  +# <</mark>>
         2065  +
  1839   2066   clean:
  1840   2067   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  1841         -	del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL
  1842         -	del /Q sqlite3.h opcodes.c opcodes.h 2>NUL
         2068  +	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
         2069  +	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
         2070  +# <<mark>>
         2071  +	del /Q sqlite3.c sqlite3.h 2>NUL
         2072  +	del /Q opcodes.c opcodes.h 2>NUL
  1843   2073   	del /Q lemon.* lempar.c parse.* 2>NUL
  1844   2074   	del /Q mkkeywordhash.* keywordhash.h 2>NUL
  1845   2075   	del /Q notasharedlib.* 2>NUL
  1846   2076   	-rmdir /Q/S .deps 2>NUL
  1847   2077   	-rmdir /Q/S .libs 2>NUL
  1848   2078   	-rmdir /Q/S quota2a 2>NUL
  1849   2079   	-rmdir /Q/S quota2b 2>NUL
................................................................................
  1850   2080   	-rmdir /Q/S quota2c 2>NUL
  1851   2081   	-rmdir /Q/S tsrc 2>NUL
  1852   2082   	del /Q .target_source 2>NUL
  1853   2083   	del /Q tclsqlite3.exe 2>NUL
  1854   2084   	del /Q testloadext.dll 2>NUL
  1855   2085   	del /Q testfixture.exe test.db 2>NUL
  1856   2086   	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL
         2087  +	del /Q changeset.exe 2>NUL
  1857   2088   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  1858         -	del /Q mptester.exe wordcount.exe 2>NUL
  1859         -	del /Q sqlite3.exe sqlite3.dll sqlite3.def 2>NUL
         2089  +	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  1860   2090   	del /Q sqlite3.c sqlite3-*.c 2>NUL
  1861   2091   	del /Q sqlite3rc.h 2>NUL
  1862         -	del /Q shell.c sqlite3ext.h 2>NUL
         2092  +	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
  1863   2093   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  1864   2094   	del /Q sqlite-*-output.vsix 2>NUL
  1865   2095   	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL
  1866   2096   	del /Q fts5.* fts5parse.* 2>NUL
  1867         -
  1868         -# Dynamic link library section.
  1869         -#
  1870         -dll: sqlite3.dll
  1871         -
  1872         -sqlite3.def: libsqlite3.lib
  1873         -	echo EXPORTS > sqlite3.def
  1874         -	dumpbin /all libsqlite3.lib \
  1875         -		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \
  1876         -		| sort >> sqlite3.def
  1877         -
  1878         -sqlite3.dll: $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
  1879         -	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2097  +# <</mark>>

Changes to VERSION.

     1         -3.10.0
            1  +3.13.0

Changes to autoconf/Makefile.am.

     6      6   libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
     7      7   
     8      8   bin_PROGRAMS = sqlite3
     9      9   sqlite3_SOURCES = shell.c sqlite3.h
    10     10   EXTRA_sqlite3_SOURCES = sqlite3.c
    11     11   sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
    12     12   sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
    13         -sqlite3_CFLAGS = $(AM_CFLAGS)
           13  +sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
    14     14   
    15     15   include_HEADERS = sqlite3.h sqlite3ext.h
    16     16   
    17         -EXTRA_DIST = sqlite3.1 tea
           17  +EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
    18     18   pkgconfigdir = ${libdir}/pkgconfig
    19     19   pkgconfig_DATA = sqlite3.pc
    20     20   
    21     21   man_MANS = sqlite3.1

Added autoconf/Makefile.msc.

            1  +#### DO NOT EDIT ####
            2  +# This makefile is automatically generated from the Makefile.msc at
            3  +# the root of the canonical SQLite source tree (not the
            4  +# amalgamation tarball) using the tool/mkmsvcmin.tcl
            5  +# script.
            6  +#
            7  +
            8  +#
            9  +# nmake Makefile for SQLite
           10  +#
           11  +###############################################################################
           12  +############################## START OF OPTIONS ###############################
           13  +###############################################################################
           14  +
           15  +# The toplevel directory of the source tree.  This is the directory
           16  +# that contains this "Makefile.msc".
           17  +#
           18  +TOP = .
           19  +
           20  +
           21  +# Set this non-0 to enable full warnings (-W4, etc) when compiling.
           22  +#
           23  +!IFNDEF USE_FULLWARN
           24  +USE_FULLWARN = 0
           25  +!ENDIF
           26  +
           27  +# Set this non-0 to use "stdcall" calling convention for the core library
           28  +# and shell executable.
           29  +#
           30  +!IFNDEF USE_STDCALL
           31  +USE_STDCALL = 0
           32  +!ENDIF
           33  +
           34  +# Set this non-0 to have the shell executable link against the core dynamic
           35  +# link library.
           36  +#
           37  +!IFNDEF DYNAMIC_SHELL
           38  +DYNAMIC_SHELL = 0
           39  +!ENDIF
           40  +
           41  +# Set this non-0 to enable extra code that attempts to detect misuse of the
           42  +# SQLite API.
           43  +#
           44  +!IFNDEF API_ARMOR
           45  +API_ARMOR = 0
           46  +!ENDIF
           47  +
           48  +# If necessary, create a list of harmless compiler warnings to disable when
           49  +# compiling the various tools.  For the SQLite source code itself, warnings,
           50  +# if any, will be disabled from within it.
           51  +#
           52  +!IFNDEF NO_WARN
           53  +!IF $(USE_FULLWARN)!=0
           54  +NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206
           55  +NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706
           56  +!ENDIF
           57  +!ENDIF
           58  +
           59  +# Set this non-0 to use the library paths and other options necessary for
           60  +# Windows Phone 8.1.
           61  +#
           62  +!IFNDEF USE_WP81_OPTS
           63  +USE_WP81_OPTS = 0
           64  +!ENDIF
           65  +
           66  +# Set this non-0 to split the SQLite amalgamation file into chunks to
           67  +# be used for debugging with Visual Studio.
           68  +#
           69  +!IFNDEF SPLIT_AMALGAMATION
           70  +SPLIT_AMALGAMATION = 0
           71  +!ENDIF
           72  +
           73  +
           74  +# Set this non-0 to dynamically link to the MSVC runtime library.
           75  +#
           76  +!IFNDEF USE_CRT_DLL
           77  +USE_CRT_DLL = 0
           78  +!ENDIF
           79  +
           80  +# Set this non-0 to link to the RPCRT4 library.
           81  +#
           82  +!IFNDEF USE_RPCRT4_LIB
           83  +USE_RPCRT4_LIB = 0
           84  +!ENDIF
           85  +
           86  +# Set this non-0 to generate assembly code listings for the source code
           87  +# files.
           88  +#
           89  +!IFNDEF USE_LISTINGS
           90  +USE_LISTINGS = 0
           91  +!ENDIF
           92  +
           93  +# Set this non-0 to attempt setting the native compiler automatically
           94  +# for cross-compiling the command line tools needed during the compilation
           95  +# process.
           96  +#
           97  +!IFNDEF XCOMPILE
           98  +XCOMPILE = 0
           99  +!ENDIF
          100  +
          101  +# Set this non-0 to use the native libraries paths for cross-compiling
          102  +# the command line tools needed during the compilation process.
          103  +#
          104  +!IFNDEF USE_NATIVE_LIBPATHS
          105  +USE_NATIVE_LIBPATHS = 0
          106  +!ENDIF
          107  +
          108  +# Set this 0 to skip the compiling and embedding of version resources.
          109  +#
          110  +!IFNDEF USE_RC
          111  +USE_RC = 1
          112  +!ENDIF
          113  +
          114  +# Set this non-0 to compile binaries suitable for the WinRT environment.
          115  +# This setting does not apply to any binaries that require Tcl to operate
          116  +# properly (i.e. the text fixture, etc).
          117  +#
          118  +!IFNDEF FOR_WINRT
          119  +FOR_WINRT = 0
          120  +!ENDIF
          121  +
          122  +# Set this non-0 to compile binaries suitable for the UWP environment.
          123  +# This setting does not apply to any binaries that require Tcl to operate
          124  +# properly (i.e. the text fixture, etc).
          125  +#
          126  +!IFNDEF FOR_UWP
          127  +FOR_UWP = 0
          128  +!ENDIF
          129  +
          130  +# Set this non-0 to compile binaries suitable for the Windows 10 platform.
          131  +#
          132  +!IFNDEF FOR_WIN10
          133  +FOR_WIN10 = 0
          134  +!ENDIF
          135  +
          136  +
          137  +# Set this to non-0 to create and use PDBs.
          138  +#
          139  +!IFNDEF SYMBOLS
          140  +SYMBOLS = 1
          141  +!ENDIF
          142  +
          143  +# Set this to non-0 to use the SQLite debugging heap subsystem.
          144  +#
          145  +!IFNDEF MEMDEBUG
          146  +MEMDEBUG = 0
          147  +!ENDIF
          148  +
          149  +# Set this to non-0 to use the Win32 native heap subsystem.
          150  +#
          151  +!IFNDEF WIN32HEAP
          152  +WIN32HEAP = 0
          153  +!ENDIF
          154  +
          155  +# Set this to non-0 to enable OSTRACE() macros, which can be useful when
          156  +# debugging.
          157  +#
          158  +!IFNDEF OSTRACE
          159  +OSTRACE = 0
          160  +!ENDIF
          161  +
          162  +# Set this to one of the following values to enable various debugging
          163  +# features.  Each level includes the debugging options from the previous
          164  +# levels.  Currently, the recognized values for DEBUG are:
          165  +#
          166  +# 0 == NDEBUG: Disables assert() and other runtime diagnostics.
          167  +# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API.
          168  +# 2 == Disables NDEBUG and all optimizations and then enables PDBs.
          169  +# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code.
          170  +# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call.
          171  +# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros.
          172  +# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros.
          173  +#
          174  +!IFNDEF DEBUG
          175  +DEBUG = 0
          176  +!ENDIF
          177  +
          178  +# Enable use of available compiler optimizations?  Normally, this should be
          179  +# non-zero.  Setting this to zero, thus disabling all compiler optimizations,
          180  +# can be useful for testing.
          181  +#
          182  +!IFNDEF OPTIMIZATIONS
          183  +OPTIMIZATIONS = 2
          184  +!ENDIF
          185  +
          186  +# Set the source code file to be used by executables and libraries when
          187  +# they need the amalgamation.
          188  +#
          189  +!IFNDEF SQLITE3C
          190  +!IF $(SPLIT_AMALGAMATION)!=0
          191  +SQLITE3C = sqlite3-all.c
          192  +!ELSE
          193  +SQLITE3C = sqlite3.c
          194  +!ENDIF
          195  +!ENDIF
          196  +
          197  +# Set the include code file to be used by executables and libraries when
          198  +# they need SQLite.
          199  +#
          200  +!IFNDEF SQLITE3H
          201  +SQLITE3H = sqlite3.h
          202  +!ENDIF
          203  +
          204  +# This is the name to use for the SQLite dynamic link library (DLL).
          205  +#
          206  +!IFNDEF SQLITE3DLL
          207  +!IF $(FOR_WIN10)!=0
          208  +SQLITE3DLL = winsqlite3.dll
          209  +!ELSE
          210  +SQLITE3DLL = sqlite3.dll
          211  +!ENDIF
          212  +!ENDIF
          213  +
          214  +# This is the name to use for the SQLite import library (LIB).
          215  +#
          216  +!IFNDEF SQLITE3LIB
          217  +!IF $(FOR_WIN10)!=0
          218  +SQLITE3LIB = winsqlite3.lib
          219  +!ELSE
          220  +SQLITE3LIB = sqlite3.lib
          221  +!ENDIF
          222  +!ENDIF
          223  +
          224  +# This is the name to use for the SQLite shell executable (EXE).
          225  +#
          226  +!IFNDEF SQLITE3EXE
          227  +!IF $(FOR_WIN10)!=0
          228  +SQLITE3EXE = winsqlite3shell.exe
          229  +!ELSE
          230  +SQLITE3EXE = sqlite3.exe
          231  +!ENDIF
          232  +!ENDIF
          233  +
          234  +# This is the argument used to set the program database (PDB) file for the
          235  +# SQLite shell executable (EXE).
          236  +#
          237  +!IFNDEF SQLITE3EXEPDB
          238  +!IF $(FOR_WIN10)!=0
          239  +SQLITE3EXEPDB =
          240  +!ELSE
          241  +SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
          242  +!ENDIF
          243  +!ENDIF
          244  +
          245  +# These are the "standard" SQLite compilation options used when compiling for
          246  +# the Windows platform.
          247  +#
          248  +!IFNDEF OPT_FEATURE_FLAGS
          249  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
          250  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
          251  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
          252  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1
          253  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1
          254  +!ENDIF
          255  +
          256  +# These are the "extended" SQLite compilation options used when compiling for
          257  +# the Windows 10 platform.
          258  +#
          259  +!IFNDEF EXT_FEATURE_FLAGS
          260  +!IF $(FOR_WIN10)!=0
          261  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
          262  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1
          263  +EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1
          264  +!ELSE
          265  +EXT_FEATURE_FLAGS =
          266  +!ENDIF
          267  +!ENDIF
          268  +
          269  +###############################################################################
          270  +############################### END OF OPTIONS ################################
          271  +###############################################################################
          272  +
          273  +# When compiling for the Windows 10 platform, the PLATFORM macro must be set
          274  +# to an appropriate value (e.g. x86, x64, arm, arm64, etc).
          275  +#
          276  +!IF $(FOR_WIN10)!=0
          277  +!IFNDEF PLATFORM
          278  +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM.
          279  +!ENDIF
          280  +!ENDIF
          281  +
          282  +# This assumes that MSVC is always installed in 32-bit Program Files directory
          283  +# and sets the variable for use in locating other 32-bit installs accordingly.
          284  +#
          285  +PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
          286  +PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
          287  +
          288  +# Check for the predefined command macro CC.  This should point to the compiler
          289  +# binary for the target platform.  If it is not defined, simply define it to
          290  +# the legacy default value 'cl.exe'.
          291  +#
          292  +!IFNDEF CC
          293  +CC = cl.exe
          294  +!ENDIF
          295  +
          296  +# Check for the predefined command macro CSC.  This should point to a working
          297  +# C Sharp compiler binary.  If it is not defined, simply define it to the
          298  +# legacy default value 'csc.exe'.
          299  +#
          300  +!IFNDEF CSC
          301  +CSC = csc.exe
          302  +!ENDIF
          303  +
          304  +# Check for the command macro LD.  This should point to the linker binary for
          305  +# the target platform.  If it is not defined, simply define it to the legacy
          306  +# default value 'link.exe'.
          307  +#
          308  +!IFNDEF LD
          309  +LD = link.exe
          310  +!ENDIF
          311  +
          312  +# Check for the predefined command macro RC.  This should point to the resource
          313  +# compiler binary for the target platform.  If it is not defined, simply define
          314  +# it to the legacy default value 'rc.exe'.
          315  +#
          316  +!IFNDEF RC
          317  +RC = rc.exe
          318  +!ENDIF
          319  +
          320  +# Check for the MSVC runtime library path macro.  Otherwise, this value will
          321  +# default to the 'lib' directory underneath the MSVC installation directory.
          322  +#
          323  +!IFNDEF CRTLIBPATH
          324  +CRTLIBPATH = $(VCINSTALLDIR)\lib
          325  +!ENDIF
          326  +
          327  +CRTLIBPATH = $(CRTLIBPATH:\\=\)
          328  +
          329  +# Check for the command macro NCC.  This should point to the compiler binary
          330  +# for the platform the compilation process is taking place on.  If it is not
          331  +# defined, simply define it to have the same value as the CC macro.  When
          332  +# cross-compiling, it is suggested that this macro be modified via the command
          333  +# line (since nmake itself does not provide a built-in method to guess it).
          334  +# For example, to use the x86 compiler when cross-compiling for x64, a command
          335  +# line similar to the following could be used (all on one line):
          336  +#
          337  +#     nmake /f Makefile.msc sqlite3.dll
          338  +#           XCOMPILE=1 USE_NATIVE_LIBPATHS=1
          339  +#
          340  +# Alternatively, the full path and file name to the compiler binary for the
          341  +# platform the compilation process is taking place may be specified (all on
          342  +# one line):
          343  +#
          344  +#     nmake /f Makefile.msc sqlite3.dll
          345  +#           "NCC=""%VCINSTALLDIR%\bin\cl.exe"""
          346  +#           USE_NATIVE_LIBPATHS=1
          347  +#
          348  +!IFDEF NCC
          349  +NCC = $(NCC:\\=\)
          350  +!ELSEIF $(XCOMPILE)!=0
          351  +NCC = "$(VCINSTALLDIR)\bin\$(CC)"
          352  +NCC = $(NCC:\\=\)
          353  +!ELSE
          354  +NCC = $(CC)
          355  +!ENDIF
          356  +
          357  +# Check for the MSVC native runtime library path macro.  Otherwise,
          358  +# this value will default to the 'lib' directory underneath the MSVC
          359  +# installation directory.
          360  +#
          361  +!IFNDEF NCRTLIBPATH
          362  +NCRTLIBPATH = $(VCINSTALLDIR)\lib
          363  +!ENDIF
          364  +
          365  +NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
          366  +
          367  +# Check for the Platform SDK library path macro.  Otherwise, this
          368  +# value will default to the 'lib' directory underneath the Windows
          369  +# SDK installation directory (the environment variable used appears
          370  +# to be available when using Visual C++ 2008 or later via the
          371  +# command line).
          372  +#
          373  +!IFNDEF NSDKLIBPATH
          374  +NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
          375  +!ENDIF
          376  +
          377  +NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
          378  +
          379  +# Check for the UCRT library path macro.  Otherwise, this value will
          380  +# default to the version-specific, platform-specific 'lib' directory
          381  +# underneath the Windows SDK installation directory.
          382  +#
          383  +!IFNDEF UCRTLIBPATH
          384  +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM)
          385  +!ENDIF
          386  +
          387  +UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
          388  +
          389  +# C compiler and options for use in building executables that
          390  +# will run on the platform that is doing the build.
          391  +#
          392  +!IF $(USE_FULLWARN)!=0
          393  +BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
          394  +!ELSE
          395  +BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS)
          396  +!ENDIF
          397  +
          398  +# Check if assembly code listings should be generated for the source
          399  +# code files to be compiled.
          400  +#
          401  +!IF $(USE_LISTINGS)!=0
          402  +BCC = $(BCC) -FAcs
          403  +!ENDIF
          404  +
          405  +# Check if the native library paths should be used when compiling
          406  +# the command line tools used during the compilation process.  If
          407  +# so, set the necessary macro now.
          408  +#
          409  +!IF $(USE_NATIVE_LIBPATHS)!=0
          410  +NLTLIBPATHS = "/LIBPATH:$(NCRTLIBPATH)" "/LIBPATH:$(NSDKLIBPATH)"
          411  +
          412  +!IFDEF NUCRTLIBPATH
          413  +NUCRTLIBPATH = $(NUCRTLIBPATH:\\=\)
          414  +NLTLIBPATHS = $(NLTLIBPATHS) "/LIBPATH:$(NUCRTLIBPATH)"
          415  +!ENDIF
          416  +!ENDIF
          417  +
          418  +# C compiler and options for use in building executables that
          419  +# will run on the target platform.  (BCC and TCC are usually the
          420  +# same unless your are cross-compiling.)
          421  +#
          422  +!IF $(USE_FULLWARN)!=0
          423  +TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
          424  +!ELSE
          425  +TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
          426  +!ENDIF
          427  +
          428  +TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
          429  +RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
          430  +
          431  +# Check if we want to use the "stdcall" calling convention when compiling.
          432  +# This is not supported by the compilers for non-x86 platforms.  It should
          433  +# also be noted here that building any target with these "stdcall" options
          434  +# will most likely fail if the Tcl library is also required.  This is due
          435  +# to how the Tcl library functions are declared and exported (i.e. without
          436  +# an explicit calling convention, which results in "cdecl").
          437  +#
          438  +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
          439  +!IF "$(PLATFORM)"=="x86"
          440  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          441  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          442  +!ELSE
          443  +!IFNDEF PLATFORM
          444  +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          445  +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
          446  +!ELSE
          447  +CORE_CCONV_OPTS =
          448  +SHELL_CCONV_OPTS =
          449  +!ENDIF
          450  +!ENDIF
          451  +!ELSE
          452  +CORE_CCONV_OPTS =
          453  +SHELL_CCONV_OPTS =
          454  +!ENDIF
          455  +
          456  +# These are additional compiler options used for the core library.
          457  +#
          458  +!IFNDEF CORE_COMPILE_OPTS
          459  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          460  +CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport)
          461  +!ELSE
          462  +CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
          463  +!ENDIF
          464  +!ENDIF
          465  +
          466  +# These are the additional targets that the core library should depend on
          467  +# when linking.
          468  +#
          469  +!IFNDEF CORE_LINK_DEP
          470  +!IF $(DYNAMIC_SHELL)!=0
          471  +CORE_LINK_DEP =
          472  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
          473  +CORE_LINK_DEP = sqlite3.def
          474  +!ELSE
          475  +CORE_LINK_DEP =
          476  +!ENDIF
          477  +!ENDIF
          478  +
          479  +# These are additional linker options used for the core library.
          480  +#
          481  +!IFNDEF CORE_LINK_OPTS
          482  +!IF $(DYNAMIC_SHELL)!=0
          483  +CORE_LINK_OPTS =
          484  +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86"
          485  +CORE_LINK_OPTS = /DEF:sqlite3.def
          486  +!ELSE
          487  +CORE_LINK_OPTS =
          488  +!ENDIF
          489  +!ENDIF
          490  +
          491  +# These are additional compiler options used for the shell executable.
          492  +#
          493  +!IFNDEF SHELL_COMPILE_OPTS
          494  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          495  +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
          496  +!ELSE
          497  +SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
          498  +!ENDIF
          499  +!ENDIF
          500  +
          501  +# This is the source code that the shell executable should be compiled
          502  +# with.
          503  +#
          504  +!IFNDEF SHELL_CORE_SRC
          505  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          506  +SHELL_CORE_SRC =
          507  +!ELSE
          508  +SHELL_CORE_SRC = $(SQLITE3C)
          509  +!ENDIF
          510  +!ENDIF
          511  +
          512  +# This is the core library that the shell executable should depend on.
          513  +#
          514  +!IFNDEF SHELL_CORE_DEP
          515  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          516  +SHELL_CORE_DEP = $(SQLITE3DLL)
          517  +!ELSE
          518  +SHELL_CORE_DEP =
          519  +!ENDIF
          520  +!ENDIF
          521  +
          522  +# This is the core library that the shell executable should link with.
          523  +#
          524  +!IFNDEF SHELL_CORE_LIB
          525  +!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
          526  +SHELL_CORE_LIB = $(SQLITE3LIB)
          527  +!ELSE
          528  +SHELL_CORE_LIB =
          529  +!ENDIF
          530  +!ENDIF
          531  +
          532  +# These are additional linker options used for the shell executable.
          533  +#
          534  +!IFNDEF SHELL_LINK_OPTS
          535  +SHELL_LINK_OPTS = $(SHELL_CORE_LIB)
          536  +!ENDIF
          537  +
          538  +# Check if assembly code listings should be generated for the source
          539  +# code files to be compiled.
          540  +#
          541  +!IF $(USE_LISTINGS)!=0
          542  +TCC = $(TCC) -FAcs
          543  +!ENDIF
          544  +
          545  +# When compiling the library for use in the WinRT environment,
          546  +# the following compile-time options must be used as well to
          547  +# disable use of Win32 APIs that are not available and to enable
          548  +# use of Win32 APIs that are specific to Windows 8 and/or WinRT.
          549  +#
          550  +!IF $(FOR_WINRT)!=0
          551  +TCC = $(TCC) -DSQLITE_OS_WINRT=1
          552  +RCC = $(RCC) -DSQLITE_OS_WINRT=1
          553  +TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
          554  +RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
          555  +!ENDIF
          556  +
          557  +# C compiler options for the Windows 10 platform (needs MSVC 2015).
          558  +#
          559  +!IF $(FOR_WIN10)!=0
          560  +TCC = $(TCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          561  +BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
          562  +!ENDIF
          563  +
          564  +# Also, we need to dynamically link to the correct MSVC runtime
          565  +# when compiling for WinRT (e.g. debug or release) OR if the
          566  +# USE_CRT_DLL option is set to force dynamically linking to the
          567  +# MSVC runtime library.
          568  +#
          569  +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
          570  +!IF $(DEBUG)>1
          571  +TCC = $(TCC) -MDd
          572  +BCC = $(BCC) -MDd
          573  +!ELSE
          574  +TCC = $(TCC) -MD
          575  +BCC = $(BCC) -MD
          576  +!ENDIF
          577  +!ELSE
          578  +!IF $(DEBUG)>1
          579  +TCC = $(TCC) -MTd
          580  +BCC = $(BCC) -MTd
          581  +!ELSE
          582  +TCC = $(TCC) -MT
          583  +BCC = $(BCC) -MT
          584  +!ENDIF
          585  +!ENDIF
          586  +
          587  +
          588  +# Define -DNDEBUG to compile without debugging (i.e., for production usage)
          589  +# Omitting the define will cause extra debugging code to be inserted and
          590  +# includes extra comments when "EXPLAIN stmt" is used.
          591  +#
          592  +!IF $(DEBUG)==0
          593  +TCC = $(TCC) -DNDEBUG
          594  +BCC = $(BCC) -DNDEBUG
          595  +RCC = $(RCC) -DNDEBUG
          596  +!ENDIF
          597  +
          598  +!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0
          599  +TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
          600  +RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
          601  +!ENDIF
          602  +
          603  +!IF $(DEBUG)>2
          604  +TCC = $(TCC) -DSQLITE_DEBUG=1
          605  +RCC = $(RCC) -DSQLITE_DEBUG=1
          606  +!ENDIF
          607  +
          608  +!IF $(DEBUG)>4 || $(OSTRACE)!=0
          609  +TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
          610  +RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
          611  +!ENDIF
          612  +
          613  +!IF $(DEBUG)>5
          614  +TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE=1
          615  +RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE=1
          616  +!ENDIF
          617  +
          618  +# Prevent warnings about "insecure" MSVC runtime library functions
          619  +# being used.
          620  +#
          621  +TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
          622  +BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
          623  +RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
          624  +
          625  +# Prevent warnings about "deprecated" POSIX functions being used.
          626  +#
          627  +TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
          628  +BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
          629  +RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
          630  +
          631  +# Use the SQLite debugging heap subsystem?
          632  +#
          633  +!IF $(MEMDEBUG)!=0
          634  +TCC = $(TCC) -DSQLITE_MEMDEBUG=1
          635  +RCC = $(RCC) -DSQLITE_MEMDEBUG=1
          636  +
          637  +# Use native Win32 heap subsystem instead of malloc/free?
          638  +#
          639  +!ELSEIF $(WIN32HEAP)!=0
          640  +TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1
          641  +RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1
          642  +
          643  +# Validate the heap on every call into the native Win32 heap subsystem?
          644  +#
          645  +!IF $(DEBUG)>3
          646  +TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
          647  +RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
          648  +!ENDIF
          649  +!ENDIF
          650  +
          651  +
          652  +# Compiler options needed for programs that use the readline() library.
          653  +#
          654  +!IFNDEF READLINE_FLAGS
          655  +READLINE_FLAGS = -DHAVE_READLINE=0
          656  +!ENDIF
          657  +
          658  +# The library that programs using readline() must link against.
          659  +#
          660  +!IFNDEF LIBREADLINE
          661  +LIBREADLINE =
          662  +!ENDIF
          663  +
          664  +# Should the database engine be compiled threadsafe
          665  +#
          666  +TCC = $(TCC) -DSQLITE_THREADSAFE=1
          667  +RCC = $(RCC) -DSQLITE_THREADSAFE=1
          668  +
          669  +# Do threads override each others locks by default (1), or do we test (-1)
          670  +#
          671  +TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
          672  +RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
          673  +
          674  +# Any target libraries which libsqlite must be linked against
          675  +#
          676  +!IFNDEF TLIBS
          677  +TLIBS =
          678  +!ENDIF
          679  +
          680  +# Flags controlling use of the in memory btree implementation
          681  +#
          682  +# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to
          683  +# default to file, 2 to default to memory, and 3 to force temporary
          684  +# tables to always be in memory.
          685  +#
          686  +TCC = $(TCC) -DSQLITE_TEMP_STORE=1
          687  +RCC = $(RCC) -DSQLITE_TEMP_STORE=1
          688  +
          689  +# Enable/disable loadable extensions, and other optional features
          690  +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
          691  +# The same set of OMIT and ENABLE flags should be passed to the
          692  +# LEMON parser generator and the mkkeywordhash tool as well.
          693  +
          694  +# These are the required SQLite compilation options used when compiling for
          695  +# the Windows platform.
          696  +#
          697  +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100
          698  +
          699  +# If we are linking to the RPCRT4 library, enable features that need it.
          700  +#
          701  +!IF $(USE_RPCRT4_LIB)!=0
          702  +REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1
          703  +!ENDIF
          704  +
          705  +# Add the required and optional SQLite compilation options into the command
          706  +# lines used to invoke the MSVC code and resource compilers.
          707  +#
          708  +TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
          709  +RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
          710  +
          711  +# Add in any optional parameters specified on the commane line, e.g.
          712  +# nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1"
          713  +#
          714  +TCC = $(TCC) $(OPTS)
          715  +RCC = $(RCC) $(OPTS)
          716  +
          717  +# If compiling for debugging, add some defines.
          718  +#
          719  +!IF $(DEBUG)>1
          720  +TCC = $(TCC) -D_DEBUG
          721  +BCC = $(BCC) -D_DEBUG
          722  +RCC = $(RCC) -D_DEBUG
          723  +!ENDIF
          724  +
          725  +# If optimizations are enabled or disabled (either implicitly or
          726  +# explicitly), add the necessary flags.
          727  +#
          728  +!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
          729  +TCC = $(TCC) -Od
          730  +BCC = $(BCC) -Od
          731  +!ELSEIF $(OPTIMIZATIONS)>=3
          732  +TCC = $(TCC) -Ox
          733  +BCC = $(BCC) -Ox
          734  +!ELSEIF $(OPTIMIZATIONS)==2
          735  +TCC = $(TCC) -O2
          736  +BCC = $(BCC) -O2
          737  +!ELSEIF $(OPTIMIZATIONS)==1
          738  +TCC = $(TCC) -O1
          739  +BCC = $(BCC) -O1
          740  +!ENDIF
          741  +
          742  +# If symbols are enabled (or compiling for debugging), enable PDBs.
          743  +#
          744  +!IF $(DEBUG)>1 || $(SYMBOLS)!=0
          745  +TCC = $(TCC) -Zi
          746  +BCC = $(BCC) -Zi
          747  +!ENDIF
          748  +
          749  +
          750  +# Command line prefixes for compiling code, compiling resources,
          751  +# linking, etc.
          752  +#
          753  +LTCOMPILE = $(TCC) -Fo$@
          754  +LTRCOMPILE = $(RCC) -r
          755  +LTLIB = lib.exe
          756  +LTLINK = $(TCC) -Fe$@
          757  +
          758  +# If requested, link to the RPCRT4 library.
          759  +#
          760  +!IF $(USE_RPCRT4_LIB)!=0
          761  +LTLINK = $(LTLINK) rpcrt4.lib
          762  +!ENDIF
          763  +
          764  +# If a platform was set, force the linker to target that.
          765  +# Note that the vcvars*.bat family of batch files typically
          766  +# set this for you.  Otherwise, the linker will attempt
          767  +# to deduce the binary type based on the object files.
          768  +!IFDEF PLATFORM
          769  +LTLINKOPTS = /NOLOGO /MACHINE:$(PLATFORM)
          770  +LTLIBOPTS = /NOLOGO /MACHINE:$(PLATFORM)
          771  +!ELSE
          772  +LTLINKOPTS = /NOLOGO
          773  +LTLIBOPTS = /NOLOGO
          774  +!ENDIF
          775  +
          776  +# When compiling for use in the WinRT environment, the following
          777  +# linker option must be used to mark the executable as runnable
          778  +# only in the context of an application container.
          779  +#
          780  +!IF $(FOR_WINRT)!=0
          781  +LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER
          782  +!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0"
          783  +!IFNDEF STORELIBPATH
          784  +!IF "$(PLATFORM)"=="x86"
          785  +STORELIBPATH = $(CRTLIBPATH)\store
          786  +!ELSEIF "$(PLATFORM)"=="x64"
          787  +STORELIBPATH = $(CRTLIBPATH)\store\amd64
          788  +!ELSEIF "$(PLATFORM)"=="ARM"
          789  +STORELIBPATH = $(CRTLIBPATH)\store\arm
          790  +!ELSE
          791  +STORELIBPATH = $(CRTLIBPATH)\store
          792  +!ENDIF
          793  +!ENDIF
          794  +STORELIBPATH = $(STORELIBPATH:\\=\)
          795  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)"
          796  +!ENDIF
          797  +!ENDIF
          798  +
          799  +# When compiling for Windows Phone 8.1, an extra library path is
          800  +# required.
          801  +#
          802  +!IF $(USE_WP81_OPTS)!=0
          803  +!IFNDEF WP81LIBPATH
          804  +!IF "$(PLATFORM)"=="x86"
          805  +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
          806  +!ELSEIF "$(PLATFORM)"=="ARM"
          807  +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM
          808  +!ELSE
          809  +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
          810  +!ENDIF
          811  +!ENDIF
          812  +!ENDIF
          813  +
          814  +# When compiling for Windows Phone 8.1, some extra linker options
          815  +# are also required.
          816  +#
          817  +!IF $(USE_WP81_OPTS)!=0
          818  +!IFDEF WP81LIBPATH
          819  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
          820  +!ENDIF
          821  +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
          822  +LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
          823  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
          824  +!ENDIF
          825  +
          826  +# When compiling for UWP or the Windows 10 platform, some extra linker
          827  +# options are also required.
          828  +#
          829  +!IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0
          830  +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
          831  +LTLINKOPTS = $(LTLINKOPTS) mincore.lib
          832  +!IFDEF PSDKLIBPATH
          833  +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
          834  +!ENDIF
          835  +!ENDIF
          836  +
          837  +!IF $(FOR_WIN10)!=0
          838  +LTLINKOPTS = $(LTLINKOPTS) /guard:cf "/LIBPATH:$(UCRTLIBPATH)"
          839  +!IF $(DEBUG)>1
          840  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib
          841  +!ELSE
          842  +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib
          843  +!ENDIF
          844  +!ENDIF
          845  +
          846  +# If either debugging or symbols are enabled, enable PDBs.
          847  +#
          848  +!IF $(DEBUG)>1 || $(SYMBOLS)!=0
          849  +LDFLAGS = /DEBUG $(LDOPTS)
          850  +!ELSE
          851  +LDFLAGS = $(LDOPTS)
          852  +!ENDIF
          853  +
          854  +
          855  +# You should not have to change anything below this line
          856  +###############################################################################
          857  +
          858  +
          859  +# Object files for the amalgamation.
          860  +#
          861  +LIBOBJS1 = sqlite3.lo
          862  +
          863  +# Determine the real value of LIBOBJ based on the 'configure' script
          864  +#
          865  +LIBOBJ = $(LIBOBJS1)
          866  +
          867  +# Determine if embedded resource compilation and usage are enabled.
          868  +#
          869  +!IF $(USE_RC)!=0
          870  +LIBRESOBJS = sqlite3res.lo
          871  +!ELSE
          872  +LIBRESOBJS =
          873  +!ENDIF
          874  +
          875  +
          876  +# Additional compiler options for the shell.  These are only effective
          877  +# when the shell is not being dynamically linked.
          878  +#
          879  +!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
          880  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
          881  +!ENDIF
          882  +
          883  +
          884  +# This is the default Makefile target.  The objects listed here
          885  +# are what get build when you type just "make" with no arguments.
          886  +#
          887  +all:	dll shell
          888  +
          889  +# Dynamic link library section.
          890  +#
          891  +dll:	$(SQLITE3DLL)
          892  +
          893  +# Shell executable.
          894  +#
          895  +shell:	$(SQLITE3EXE)
          896  +
          897  +
          898  +$(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
          899  +	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
          900  +
          901  +Replace.exe:
          902  +	$(CSC) /target:exe $(TOP)\Replace.cs
          903  +
          904  +sqlite3.def:	Replace.exe $(LIBOBJ)
          905  +	echo EXPORTS > sqlite3.def
          906  +	dumpbin /all $(LIBOBJ) \
          907  +		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
          908  +		| sort >> sqlite3.def
          909  +
          910  +$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
          911  +	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
          912  +		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
          913  +
          914  +
          915  +# Rule to build the amalgamation
          916  +#
          917  +sqlite3.lo:	$(SQLITE3C)
          918  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)
          919  +
          920  +
          921  +# Rule to build the Win32 resources object file.
          922  +#
          923  +!IF $(USE_RC)!=0
          924  +_HASHCHAR=^#
          925  +!IF ![echo !IFNDEF VERSION > rcver.vc] && \
          926  +    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
          927  +    ![echo !ENDIF >> rcver.vc]
          928  +!INCLUDE rcver.vc
          929  +!ENDIF
          930  +
          931  +RESOURCE_VERSION = $(VERSION:^#=)
          932  +RESOURCE_VERSION = $(RESOURCE_VERSION:define=)
          933  +RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=)
          934  +RESOURCE_VERSION = $(RESOURCE_VERSION:"=)
          935  +RESOURCE_VERSION = $(RESOURCE_VERSION:.=,)
          936  +
          937  +$(LIBRESOBJS):	$(TOP)\sqlite3.rc rcver.vc $(SQLITE3H)
          938  +	echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h
          939  +	echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h
          940  +	echo #endif >> sqlite3rc.h
          941  +	$(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc
          942  +!ENDIF
          943  +
          944  +
          945  +clean:
          946  +	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
          947  +	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
          948  +	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL

Name change from autoconf/README to autoconf/README.txt.

     1         -
     2      1   This package contains:
     3      2   
     4         - * the SQLite library amalgamation (single file) source code distribution,
     5         - * the shell.c file used to build the sqlite3 shell too, and
     6         - * the sqlite3.h and sqlite3ext.h header files required to link programs
     7         -   and sqlite extensions against the installed libary.
     8         - * autoconf/automake installation infrastucture.
            3  + * the SQLite library amalgamation source code file: sqlite3.c
            4  + * the sqlite3.h and sqlite3ext.h header files that define the C-language
            5  +   interface to the sqlite3.c library file
            6  + * the shell.c file used to build the sqlite3 command-line shell program
            7  + * autoconf/automake installation infrastucture for building on POSIX
            8  +   compliant systems
            9  + * a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft
           10  +   Visual C++ on Windows
           11  +
           12  +SUMMARY OF HOW TO BUILD
           13  +=======================
           14  +
           15  +  Unix:      ./configure; make
           16  +  Windows:   nmake /f Makefile.msc
           17  +
           18  +BUILDING ON POSIX
           19  +=================
     9     20   
    10     21   The generic installation instructions for autoconf/automake are found
    11     22   in the INSTALL file.
    12     23   
    13     24   The following SQLite specific boolean options are supported:
    14     25   
    15     26     --enable-readline           use readline in shell tool   [default=yes]
    16     27     --enable-threadsafe         build a thread-safe library  [default=yes]
    17     28     --enable-dynamic-extensions support loadable extensions  [default=yes]
    18     29   
    19         -The default value for the CFLAGS variable (options passed to the C 
           30  +The default value for the CFLAGS variable (options passed to the C
    20     31   compiler) includes debugging symbols in the build, resulting in larger
    21     32   binaries than are necessary. Override it on the configure command
    22     33   line like this:
    23     34   
    24     35     $ CFLAGS="-Os" ./configure
    25     36   
    26     37   to produce a smaller installation footprint.
    27     38   
    28     39   Other SQLite compilation parameters can also be set using CFLAGS. For
    29     40   example:
    30     41   
    31         -  $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure
           42  +  $ CFLAGS="-Os -DSQLITE_THREADSAFE=0" ./configure
           43  +
           44  +
           45  +BUILDING WITH MICROSOFT VISUAL C++
           46  +==================================
           47  +
           48  +To compile for Windows using Microsoft Visual C++:
           49  +
           50  +  $ nmake /f Makefile.msc
           51  +
           52  +Using Microsoft Visual C++ 2005 (or later) is recommended.  Several Windows
           53  +platform variants may be built by adding additional macros to the NMAKE
           54  +command line.
           55  +
           56  +Building for WinRT 8.0
           57  +----------------------
           58  +
           59  +  FOR_WINRT=1
           60  +
           61  +Using Microsoft Visual C++ 2012 (or later) is required.  When using the
           62  +above, something like the following macro will need to be added to the
           63  +NMAKE command line as well:
           64  +
           65  +  "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86"
           66  +
           67  +Building for WinRT 8.1
           68  +----------------------
           69  +
           70  +  FOR_WINRT=1
           71  +
           72  +Using Microsoft Visual C++ 2013 (or later) is required.  When using the
           73  +above, something like the following macro will need to be added to the
           74  +NMAKE command line as well:
           75  +
           76  +  "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86"
           77  +
           78  +Building for UWP 10.0
           79  +---------------------
           80  +
           81  +  FOR_WINRT=1 FOR_UWP=1
           82  +
           83  +Using Microsoft Visual C++ 2015 (or later) is required.  When using the
           84  +above, something like the following macros will need to be added to the
           85  +NMAKE command line as well:
           86  +
           87  +  "NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
           88  +  "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
           89  +  "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86"
           90  +
           91  +Building for the Windows 10 SDK
           92  +-------------------------------
           93  +
           94  +  FOR_WIN10=1
           95  +
           96  +Using Microsoft Visual C++ 2015 (or later) is required.  When using the
           97  +above, no other macros should be needed on the NMAKE command line.
           98  +
           99  +Other preprocessor defines
          100  +--------------------------
          101  +
          102  +Additionally, preprocessor defines may be specified by using the OPTS macro
          103  +on the NMAKE command line.  However, not all possible preprocessor defines
          104  +may be specified in this manner as some require the amalgamation to be built
          105  +with them enabled (see http://www.sqlite.org/compile.html). For example, the
          106  +following will work:
          107  +
          108  +  "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1"
          109  +
          110  +However, the following will not compile unless the amalgamation was built
          111  +with it enabled:
    32    112   
          113  +  "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1"

Changes to autoconf/configure.ac.

    45     45   if test x"$enable_editline" != xno ; then
    46     46     sLIBS=$LIBS
    47     47     LIBS=""
    48     48     AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
    49     49     READLINE_LIBS=$LIBS
    50     50     if test x"$LIBS" != "x"; then
    51     51        AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
           52  +  else
           53  +    unset ac_cv_search_readline
    52     54     fi
    53     55     LIBS=$sLIBS
    54     56   fi
    55     57   if test x"$enable_readline" != xno ; then
    56     58     sLIBS=$LIBS
    57     59     LIBS=""
    58     60     AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], [])
................................................................................
    70     72   AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
    71     73     [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
    72     74     [], [enable_threadsafe=yes])
    73     75   THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
    74     76   if test x"$enable_threadsafe" != "xno"; then
    75     77     THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
    76     78     AC_SEARCH_LIBS(pthread_create, pthread)
           79  +  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
    77     80   fi
    78     81   AC_SUBST(THREADSAFE_FLAGS)
    79     82   #-----------------------------------------------------------------------
    80     83   
    81     84   #-----------------------------------------------------------------------
    82     85   #   --enable-dynamic-extensions
    83     86   #
................................................................................
   123    126   #   --enable-static-shell
   124    127   #
   125    128   AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
   126    129     [--enable-static-shell], 
   127    130     [statically link libsqlite3 into shell tool [default=yes]])], 
   128    131     [], [enable_static_shell=yes])
   129    132   if test x"$enable_static_shell" == "xyes"; then
   130         -  EXTRA_SHELL_OBJ=sqlite3.$OBJEXT
          133  +  EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT
   131    134   else
   132    135     EXTRA_SHELL_OBJ=libsqlite3.la
   133    136   fi
   134    137   AC_SUBST(EXTRA_SHELL_OBJ)
   135    138   #-----------------------------------------------------------------------
   136    139   
   137    140   AC_CHECK_FUNCS(posix_fallocate)

Changes to config.h.in.

    40     40   #undef HAVE_MALLOC_H
    41     41   
    42     42   /* Define to 1 if you have the `malloc_usable_size' function. */
    43     43   #undef HAVE_MALLOC_USABLE_SIZE
    44     44   
    45     45   /* Define to 1 if you have the <memory.h> header file. */
    46     46   #undef HAVE_MEMORY_H
           47  +
           48  +/* Define to 1 if you have the pread() function. */
           49  +#undef HAVE_PREAD
           50  +
           51  +/* Define to 1 if you have the pread64() function. */
           52  +#undef HAVE_PREAD64
           53  +
           54  +/* Define to 1 if you have the pwrite() function. */
           55  +#undef HAVE_PWRITE
           56  +
           57  +/* Define to 1 if you have the pwrite64() function. */
           58  +#undef HAVE_PWRITE64
    47     59   
    48     60   /* Define to 1 if you have the <stdint.h> header file. */
    49     61   #undef HAVE_STDINT_H
    50     62   
    51     63   /* Define to 1 if you have the <stdlib.h> header file. */
    52     64   #undef HAVE_STDLIB_H
    53     65   

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.10.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.13.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.10.0'
   730         -PACKAGE_STRING='sqlite 3.10.0'
          729  +PACKAGE_VERSION='3.13.0'
          730  +PACKAGE_STRING='sqlite 3.13.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1456   1456   #
  1457   1457   # Report the --help message.
  1458   1458   #
  1459   1459   if test "$ac_init_help" = "long"; then
  1460   1460     # Omit some internal or obsolete options to make the list less imposing.
  1461   1461     # This message is too long to be a string in the A/UX 3.1 sh.
  1462   1462     cat <<_ACEOF
  1463         -\`configure' configures sqlite 3.10.0 to adapt to many kinds of systems.
         1463  +\`configure' configures sqlite 3.13.0 to adapt to many kinds of systems.
  1464   1464   
  1465   1465   Usage: $0 [OPTION]... [VAR=VALUE]...
  1466   1466   
  1467   1467   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1468   1468   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1469   1469   
  1470   1470   Defaults for the options are specified in brackets.
................................................................................
  1521   1521     --build=BUILD     configure for building on BUILD [guessed]
  1522   1522     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1523   1523   _ACEOF
  1524   1524   fi
  1525   1525   
  1526   1526   if test -n "$ac_init_help"; then
  1527   1527     case $ac_init_help in
  1528         -     short | recursive ) echo "Configuration of sqlite 3.10.0:";;
         1528  +     short | recursive ) echo "Configuration of sqlite 3.13.0:";;
  1529   1529      esac
  1530   1530     cat <<\_ACEOF
  1531   1531   
  1532   1532   Optional Features:
  1533   1533     --disable-option-checking  ignore unrecognized --enable/--with options
  1534   1534     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1535   1535     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1642   1642       cd "$ac_pwd" || { ac_status=$?; break; }
  1643   1643     done
  1644   1644   fi
  1645   1645   
  1646   1646   test -n "$ac_init_help" && exit $ac_status
  1647   1647   if $ac_init_version; then
  1648   1648     cat <<\_ACEOF
  1649         -sqlite configure 3.10.0
         1649  +sqlite configure 3.13.0
  1650   1650   generated by GNU Autoconf 2.69
  1651   1651   
  1652   1652   Copyright (C) 2012 Free Software Foundation, Inc.
  1653   1653   This configure script is free software; the Free Software Foundation
  1654   1654   gives unlimited permission to copy, distribute and modify it.
  1655   1655   _ACEOF
  1656   1656     exit
................................................................................
  2061   2061     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2062   2062   
  2063   2063   } # ac_fn_c_check_header_mongrel
  2064   2064   cat >config.log <<_ACEOF
  2065   2065   This file contains any messages produced by compilers while
  2066   2066   running configure, to aid debugging if configure makes a mistake.
  2067   2067   
  2068         -It was created by sqlite $as_me 3.10.0, which was
         2068  +It was created by sqlite $as_me 3.13.0, which was
  2069   2069   generated by GNU Autoconf 2.69.  Invocation command line was
  2070   2070   
  2071   2071     $ $0 $@
  2072   2072   
  2073   2073   _ACEOF
  2074   2074   exec 5>>config.log
  2075   2075   {
................................................................................
 10269  10269   
 10270  10270   done
 10271  10271   
 10272  10272   
 10273  10273   #########
 10274  10274   # Figure out whether or not we have these functions
 10275  10275   #
 10276         -for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime
        10276  +for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64
 10277  10277   do :
 10278  10278     as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 10279  10279   ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
 10280  10280   if eval test \"x\$"$as_ac_var"\" = x"yes"; then :
 10281  10281     cat >>confdefs.h <<_ACEOF
 10282  10282   #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1
 10283  10283   _ACEOF
................................................................................
 10511  10511   fi
 10512  10512   rm conftest.$ac_ext
 10513  10513   LIBS=$ac_func_search_save_LIBS
 10514  10514   fi
 10515  10515   { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5
 10516  10516   $as_echo "$ac_cv_search_pthread_create" >&6; }
 10517  10517   ac_res=$ac_cv_search_pthread_create
        10518  +if test "$ac_res" != no; then :
        10519  +  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
        10520  +
        10521  +fi
        10522  +
        10523  +  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5
        10524  +$as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; }
        10525  +if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
        10526  +  $as_echo_n "(cached) " >&6
        10527  +else
        10528  +  ac_func_search_save_LIBS=$LIBS
        10529  +cat confdefs.h - <<_ACEOF >conftest.$ac_ext
        10530  +/* end confdefs.h.  */
        10531  +
        10532  +/* Override any GCC internal prototype to avoid an error.
        10533  +   Use char because int might match the return type of a GCC
        10534  +   builtin and then its argument prototype would still apply.  */
        10535  +#ifdef __cplusplus
        10536  +extern "C"
        10537  +#endif
        10538  +char pthread_mutexattr_init ();
        10539  +int
        10540  +main ()
        10541  +{
        10542  +return pthread_mutexattr_init ();
        10543  +  ;
        10544  +  return 0;
        10545  +}
        10546  +_ACEOF
        10547  +for ac_lib in '' pthread; do
        10548  +  if test -z "$ac_lib"; then
        10549  +    ac_res="none required"
        10550  +  else
        10551  +    ac_res=-l$ac_lib
        10552  +    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
        10553  +  fi
        10554  +  if ac_fn_c_try_link "$LINENO"; then :
        10555  +  ac_cv_search_pthread_mutexattr_init=$ac_res
        10556  +fi
        10557  +rm -f core conftest.err conftest.$ac_objext \
        10558  +    conftest$ac_exeext
        10559  +  if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
        10560  +  break
        10561  +fi
        10562  +done
        10563  +if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
        10564  +
        10565  +else
        10566  +  ac_cv_search_pthread_mutexattr_init=no
        10567  +fi
        10568  +rm conftest.$ac_ext
        10569  +LIBS=$ac_func_search_save_LIBS
        10570  +fi
        10571  +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5
        10572  +$as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; }
        10573  +ac_res=$ac_cv_search_pthread_mutexattr_init
 10518  10574   if test "$ac_res" != no; then :
 10519  10575     test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
 10520  10576   
 10521  10577   fi
 10522  10578   
 10523  10579   fi
 10524  10580   
................................................................................
 12019  12075   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12020  12076   
 12021  12077   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12022  12078   # Save the log message, to keep $0 and so on meaningful, and to
 12023  12079   # report actual input values of CONFIG_FILES etc. instead of their
 12024  12080   # values after options handling.
 12025  12081   ac_log="
 12026         -This file was extended by sqlite $as_me 3.10.0, which was
        12082  +This file was extended by sqlite $as_me 3.13.0, which was
 12027  12083   generated by GNU Autoconf 2.69.  Invocation command line was
 12028  12084   
 12029  12085     CONFIG_FILES    = $CONFIG_FILES
 12030  12086     CONFIG_HEADERS  = $CONFIG_HEADERS
 12031  12087     CONFIG_LINKS    = $CONFIG_LINKS
 12032  12088     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12033  12089     $ $0 $@
................................................................................
 12085  12141   
 12086  12142   Report bugs to the package provider."
 12087  12143   
 12088  12144   _ACEOF
 12089  12145   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12090  12146   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12091  12147   ac_cs_version="\\
 12092         -sqlite config.status 3.10.0
        12148  +sqlite config.status 3.13.0
 12093  12149   configured by $0, generated by GNU Autoconf 2.69,
 12094  12150     with options \\"\$ac_cs_config\\"
 12095  12151   
 12096  12152   Copyright (C) 2012 Free Software Foundation, Inc.
 12097  12153   This config.status script is free software; the Free Software Foundation
 12098  12154   gives unlimited permission to copy, distribute and modify it."
 12099  12155   

Changes to configure.ac.

   104    104   #########
   105    105   # Check for needed/wanted headers
   106    106   AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h])
   107    107   
   108    108   #########
   109    109   # Figure out whether or not we have these functions
   110    110   #
   111         -AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime])
          111  +AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64])
   112    112   
   113    113   #########
   114    114   # By default, we use the amalgamation (this may be changed below...)
   115    115   #
   116    116   USE_AMALGAMATION=1
   117    117   
   118    118   #########
................................................................................
   191    191     SQLITE_THREADSAFE=1
   192    192     AC_MSG_RESULT([yes])
   193    193   fi
   194    194   AC_SUBST(SQLITE_THREADSAFE)
   195    195   
   196    196   if test "$SQLITE_THREADSAFE" = "1"; then
   197    197     AC_SEARCH_LIBS(pthread_create, pthread)
          198  +  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
   198    199   fi
   199    200   
   200    201   ##########
   201    202   # Do we want to support release
   202    203   #
   203    204   AC_ARG_ENABLE(releasemode, 
   204    205   AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)

Changes to doc/lemon.html.

     1      1   <html>
     2      2   <head>
     3      3   <title>The Lemon Parser Generator</title>
     4      4   </head>
     5      5   <body bgcolor=white>
     6      6   <h1 align=center>The Lemon Parser Generator</h1>
     7      7   
     8         -<p>Lemon is an LALR(1) parser generator for C or C++.  
     9         -It does the same job as ``bison'' and ``yacc''.
    10         -But lemon is not another bison or yacc clone.  It
            8  +<p>Lemon is an LALR(1) parser generator for C.
            9  +It does the same job as "bison" and "yacc".
           10  +But lemon is not a bison or yacc clone.  Lemon
    11     11   uses a different grammar syntax which is designed to
    12         -reduce the number of coding errors.  Lemon also uses a more
    13         -sophisticated parsing engine that is faster than yacc and
    14         -bison and which is both reentrant and thread-safe.
    15         -Furthermore, Lemon implements features that can be used
           12  +reduce the number of coding errors.  Lemon also uses a
           13  +parsing engine that is faster than yacc and
           14  +bison and which is both reentrant and threadsafe.
           15  +(Update: Since the previous sentence was written, bison
           16  +has also been updated so that it too can generate a
           17  +reentrant and threadsafe parser.)
           18  +Lemon also implements features that can be used
    16     19   to eliminate resource leaks, making is suitable for use
    17     20   in long-running programs such as graphical user interfaces
    18     21   or embedded controllers.</p>
    19     22   
    20     23   <p>This document is an introduction to the Lemon
    21     24   parser generator.</p>
    22     25   
................................................................................
    40     43   <ul>
    41     44   <li>C code to implement the parser.
    42     45   <li>A header file defining an integer ID for each terminal symbol.
    43     46   <li>An information file that describes the states of the generated parser
    44     47       automaton.
    45     48   </ul>
    46     49   By default, all three of these output files are generated.
    47         -The header file is suppressed if the ``-m'' command-line option is
    48         -used and the report file is omitted when ``-q'' is selected.</p>
           50  +The header file is suppressed if the "-m" command-line option is
           51  +used and the report file is omitted when "-q" is selected.</p>
    49     52   
    50         -<p>The grammar specification file uses a ``.y'' suffix, by convention.
           53  +<p>The grammar specification file uses a ".y" suffix, by convention.
    51     54   In the examples used in this document, we'll assume the name of the
    52         -grammar file is ``gram.y''.  A typical use of Lemon would be the
           55  +grammar file is "gram.y".  A typical use of Lemon would be the
    53     56   following command:
    54     57   <pre>
    55     58      lemon gram.y
    56     59   </pre>
    57         -This command will generate three output files named ``gram.c'',
    58         -``gram.h'' and ``gram.out''.
           60  +This command will generate three output files named "gram.c",
           61  +"gram.h" and "gram.out".
    59     62   The first is C code to implement the parser.  The second
    60     63   is the header file that defines numerical values for all
    61     64   terminal symbols, and the last is the report that explains
    62     65   the states used by the parser automaton.</p>
    63     66   
    64     67   <h3>Command Line Options</h3>
    65     68   
................................................................................
    67     70   You can obtain a list of the available command-line options together
    68     71   with a brief explanation of what each does by typing
    69     72   <pre>
    70     73      lemon -?
    71     74   </pre>
    72     75   As of this writing, the following command-line options are supported:
    73     76   <ul>
    74         -<li><tt>-b</tt>
    75         -<li><tt>-c</tt>
    76         -<li><tt>-g</tt>
    77         -<li><tt>-m</tt>
    78         -<li><tt>-q</tt>
    79         -<li><tt>-s</tt>
    80         -<li><tt>-x</tt>
    81         -</ul>
    82         -The ``-b'' option reduces the amount of text in the report file by
    83         -printing only the basis of each parser state, rather than the full
    84         -configuration.
    85         -The ``-c'' option suppresses action table compression.  Using -c
    86         -will make the parser a little larger and slower but it will detect
    87         -syntax errors sooner.
    88         -The ``-g'' option causes no output files to be generated at all.
    89         -Instead, the input grammar file is printed on standard output but
    90         -with all comments, actions and other extraneous text deleted.  This
    91         -is a useful way to get a quick summary of a grammar.
    92         -The ``-m'' option causes the output C source file to be compatible
    93         -with the ``makeheaders'' program.
    94         -Makeheaders is a program that automatically generates header files
    95         -from C source code.  When the ``-m'' option is used, the header
    96         -file is not output since the makeheaders program will take care
    97         -of generated all header files automatically.
    98         -The ``-q'' option suppresses the report file.
    99         -Using ``-s'' causes a brief summary of parser statistics to be
   100         -printed.  Like this:
   101         -<pre>
   102         -   Parser statistics: 74 terminals, 70 nonterminals, 179 rules
   103         -                      340 states, 2026 parser table entries, 0 conflicts
   104         -</pre>
   105         -Finally, the ``-x'' option causes Lemon to print its version number
   106         -and then stops without attempting to read the grammar or generate a parser.</p>
           77  +<li><b>-b</b>
           78  +Show only the basis for each parser state in the report file.
           79  +<li><b>-c</b>
           80  +Do not compress the generated action tables.
           81  +<li><b>-D<i>name</i></b>
           82  +Define C preprocessor macro <i>name</i>.  This macro is useable by
           83  +"%ifdef" lines in the grammar file.
           84  +<li><b>-g</b>
           85  +Do not generate a parser.  Instead write the input grammar to standard
           86  +output with all comments, actions, and other extraneous text removed.
           87  +<li><b>-l</b>
           88  +Omit "#line" directives int the generated parser C code.
           89  +<li><b>-m</b>
           90  +Cause the output C source code to be compatible with the "makeheaders"
           91  +program. 
           92  +<li><b>-p</b>
           93  +Display all conflicts that are resolved by 
           94  +<a href='#precrules'>precedence rules</a>.
           95  +<li><b>-q</b>
           96  +Suppress generation of the report file.
           97  +<li><b>-r</b>
           98  +Do not sort or renumber the parser states as part of optimization.
           99  +<li><b>-s</b>
          100  +Show parser statistics before existing.
          101  +<li><b>-T<i>file</i></b>
          102  +Use <i>file</i> as the template for the generated C-code parser implementation.
          103  +<li><b>-x</b>
          104  +Print the Lemon version number.
          105  +</ul>
   107    106   
   108    107   <h3>The Parser Interface</h3>
   109    108   
   110    109   <p>Lemon doesn't generate a complete, working program.  It only generates
   111    110   a few subroutines that implement a parser.  This section describes
   112    111   the interface to those subroutines.  It is up to the programmer to
   113    112   call these subroutines in an appropriate way in order to produce a
................................................................................
   117    116   must first create the parser.
   118    117   A new parser is created as follows:
   119    118   <pre>
   120    119      void *pParser = ParseAlloc( malloc );
   121    120   </pre>
   122    121   The ParseAlloc() routine allocates and initializes a new parser and
   123    122   returns a pointer to it.
   124         -The actual data structure used to represent a parser is opaque --
          123  +The actual data structure used to represent a parser is opaque &mdash;
   125    124   its internal structure is not visible or usable by the calling routine.
   126    125   For this reason, the ParseAlloc() routine returns a pointer to void
   127    126   rather than a pointer to some particular structure.
   128    127   The sole argument to the ParseAlloc() routine is a pointer to the
   129         -subroutine used to allocate memory.  Typically this means ``malloc()''.</p>
          128  +subroutine used to allocate memory.  Typically this means malloc().</p>
   130    129   
   131    130   <p>After a program is finished using a parser, it can reclaim all
   132    131   memory allocated by that parser by calling
   133    132   <pre>
   134    133      ParseFree(pParser, free);
   135    134   </pre>
   136    135   The first argument is the same pointer returned by ParseAlloc().  The
................................................................................
   147    146   The first argument to the Parse() routine is the pointer returned by
   148    147   ParseAlloc().
   149    148   The second argument is a small positive integer that tells the parse the
   150    149   type of the next token in the data stream.
   151    150   There is one token type for each terminal symbol in the grammar.
   152    151   The gram.h file generated by Lemon contains #define statements that
   153    152   map symbolic terminal symbol names into appropriate integer values.
   154         -(A value of 0 for the second argument is a special flag to the
   155         -parser to indicate that the end of input has been reached.)
          153  +A value of 0 for the second argument is a special flag to the
          154  +parser to indicate that the end of input has been reached.
   156    155   The third argument is the value of the given token.  By default,
   157    156   the type of the third argument is integer, but the grammar will
   158    157   usually redefine this type to be some kind of structure.
   159    158   Typically the second argument will be a broad category of tokens
   160         -such as ``identifier'' or ``number'' and the third argument will
          159  +such as "identifier" or "number" and the third argument will
   161    160   be the name of the identifier or the value of the number.</p>
   162    161   
   163    162   <p>The Parse() function may have either three or four arguments,
   164         -depending on the grammar.  If the grammar specification file request
   165         -it, the Parse() function will have a fourth parameter that can be
          163  +depending on the grammar.  If the grammar specification file requests
          164  +it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>),
          165  +the Parse() function will have a fourth parameter that can be
   166    166   of any type chosen by the programmer.  The parser doesn't do anything
   167    167   with this argument except to pass it through to action routines.
   168    168   This is a convenient mechanism for passing state information down
   169    169   to the action routines without having to use global variables.</p>
   170    170   
   171    171   <p>A typical use of a Lemon parser might look something like the
   172    172   following:
................................................................................
   188    188      15    ParseFree(pParser, free );
   189    189      16    TokenizerFree(pTokenizer);
   190    190      17    return sState.treeRoot;
   191    191      18 }
   192    192   </pre>
   193    193   This example shows a user-written routine that parses a file of
   194    194   text and returns a pointer to the parse tree.
   195         -(We've omitted all error-handling from this example to keep it
          195  +(All error-handling code is omitted from this example to keep it
   196    196   simple.)
   197    197   We assume the existence of some kind of tokenizer which is created
   198    198   using TokenizerCreate() on line 8 and deleted by TokenizerFree()
   199    199   on line 16.  The GetNextToken() function on line 11 retrieves the
   200    200   next token from the input file and puts its type in the 
   201    201   integer variable hTokenId.  The sToken variable is assumed to be
   202    202   some kind of structure that contains details about each token,
................................................................................
   259    259       and bison do not.
   260    260   </ul>
   261    261   These differences may cause some initial confusion for programmers
   262    262   with prior yacc and bison experience.
   263    263   But after years of experience using Lemon, I firmly
   264    264   believe that the Lemon way of doing things is better.</p>
   265    265   
          266  +<p><i>Updated as of 2016-02-16:</i>
          267  +The text above was written in the 1990s.
          268  +We are told that Bison has lately been enhanced to support the
          269  +tokenizer-calls-parser paradigm used by Lemon, and to obviate the
          270  +need for global variables.</p>
          271  +
   266    272   <h2>Input File Syntax</h2>
   267    273   
   268    274   <p>The main purpose of the grammar specification file for Lemon is
   269    275   to define the grammar for the parser.  But the input file also
   270    276   specifies additional information Lemon requires to do its job.
   271    277   Most of the work in using Lemon is in writing an appropriate
   272    278   grammar file.</p>
................................................................................
   276    282   declaration can occur at any point in the file.
   277    283   Lemon ignores whitespace (except where it is needed to separate
   278    284   tokens) and it honors the same commenting conventions as C and C++.</p>
   279    285   
   280    286   <h3>Terminals and Nonterminals</h3>
   281    287   
   282    288   <p>A terminal symbol (token) is any string of alphanumeric
   283         -and underscore characters
          289  +and/or underscore characters
   284    290   that begins with an upper case letter.
   285    291   A terminal can contain lowercase letters after the first character,
   286    292   but the usual convention is to make terminals all upper case.
   287    293   A nonterminal, on the other hand, is any string of alphanumeric
   288    294   and underscore characters than begins with a lower case letter.
   289    295   Again, the usual convention is to make nonterminals use all lower
   290    296   case letters.</p>
................................................................................
   303    309   must have alphanumeric names.</p>
   304    310   
   305    311   <h3>Grammar Rules</h3>
   306    312   
   307    313   <p>The main component of a Lemon grammar file is a sequence of grammar
   308    314   rules.
   309    315   Each grammar rule consists of a nonterminal symbol followed by
   310         -the special symbol ``::='' and then a list of terminals and/or nonterminals.
          316  +the special symbol "::=" and then a list of terminals and/or nonterminals.
   311    317   The rule is terminated by a period.
   312    318   The list of terminals and nonterminals on the right-hand side of the
   313    319   rule can be empty.
   314    320   Rules can occur in any order, except that the left-hand side of the
   315    321   first rule is assumed to be the start symbol for the grammar (unless
   316    322   specified otherwise using the <tt>%start</tt> directive described below.)
   317    323   A typical sequence of grammar rules might look something like this:
................................................................................
   319    325     expr ::= expr PLUS expr.
   320    326     expr ::= expr TIMES expr.
   321    327     expr ::= LPAREN expr RPAREN.
   322    328     expr ::= VALUE.
   323    329   </pre>
   324    330   </p>
   325    331   
   326         -<p>There is one non-terminal in this example, ``expr'', and five
   327         -terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'',
   328         -``RPAREN'' and ``VALUE''.</p>
          332  +<p>There is one non-terminal in this example, "expr", and five
          333  +terminal symbols or tokens: "PLUS", "TIMES", "LPAREN",
          334  +"RPAREN" and "VALUE".</p>
   329    335   
   330    336   <p>Like yacc and bison, Lemon allows the grammar to specify a block
   331    337   of C code that will be executed whenever a grammar rule is reduced
   332    338   by the parser.
   333    339   In Lemon, this action is specified by putting the C code (contained
   334    340   within curly braces <tt>{...}</tt>) immediately after the
   335    341   period that closes the rule.
................................................................................
   337    343   <pre>
   338    344     expr ::= expr PLUS expr.   { printf("Doing an addition...\n"); }
   339    345   </pre>
   340    346   </p>
   341    347   
   342    348   <p>In order to be useful, grammar actions must normally be linked to
   343    349   their associated grammar rules.
   344         -In yacc and bison, this is accomplished by embedding a ``$$'' in the
          350  +In yacc and bison, this is accomplished by embedding a "$$" in the
   345    351   action to stand for the value of the left-hand side of the rule and
   346         -symbols ``$1'', ``$2'', and so forth to stand for the value of
          352  +symbols "$1", "$2", and so forth to stand for the value of
   347    353   the terminal or nonterminal at position 1, 2 and so forth on the
   348    354   right-hand side of the rule.
   349    355   This idea is very powerful, but it is also very error-prone.  The
   350    356   single most common source of errors in a yacc or bison grammar is
   351    357   to miscount the number of symbols on the right-hand side of a grammar
   352         -rule and say ``$7'' when you really mean ``$8''.</p>
          358  +rule and say "$7" when you really mean "$8".</p>
   353    359   
   354    360   <p>Lemon avoids the need to count grammar symbols by assigning symbolic
   355    361   names to each symbol in a grammar rule and then using those symbolic
   356    362   names in the action.
   357    363   In yacc or bison, one would write this:
   358    364   <pre>
   359    365     expr -> expr PLUS expr  { $$ = $1 + $3; };
................................................................................
   375    381   includes a linking symbol in parentheses but that linking symbol
   376    382   is not actually used in the reduce action, then an error message
   377    383   is generated.
   378    384   For example, the rule
   379    385   <pre>
   380    386     expr(A) ::= expr(B) PLUS expr(C).  { A = B; }
   381    387   </pre>
   382         -will generate an error because the linking symbol ``C'' is used
          388  +will generate an error because the linking symbol "C" is used
   383    389   in the grammar rule but not in the reduce action.</p>
   384    390   
   385    391   <p>The Lemon notation for linking grammar rules to reduce actions
   386    392   also facilitates the use of destructors for reclaiming memory
   387    393   allocated by the values of terminals and nonterminals on the
   388    394   right-hand side of a rule.</p>
   389    395   
          396  +<a name='precrules'></a>
   390    397   <h3>Precedence Rules</h3>
   391    398   
   392    399   <p>Lemon resolves parsing ambiguities in exactly the same way as
   393    400   yacc and bison.  A shift-reduce conflict is resolved in favor
   394    401   of the shift, and a reduce-reduce conflict is resolved by reducing
   395    402   whichever rule comes first in the grammar file.</p>
   396    403   
   397    404   <p>Just like in
   398    405   yacc and bison, Lemon allows a measure of control 
   399    406   over the resolution of paring conflicts using precedence rules.
   400    407   A precedence value can be assigned to any terminal symbol
   401         -using the %left, %right or %nonassoc directives.  Terminal symbols
          408  +using the 
          409  +<a href='#pleft'>%left</a>,
          410  +<a href='#pright'>%right</a> or
          411  +<a href='#pnonassoc'>%nonassoc</a> directives.  Terminal symbols
   402    412   mentioned in earlier directives have a lower precedence that
   403    413   terminal symbols mentioned in later directives.  For example:</p>
   404    414   
   405    415   <p><pre>
   406    416      %left AND.
   407    417      %left OR.
   408    418      %nonassoc EQ NE GT GE LT LE.
................................................................................
   514    524   
   515    525   <p>Lemon supports the following special directives:
   516    526   <ul>
   517    527   <li><tt>%code</tt>
   518    528   <li><tt>%default_destructor</tt>
   519    529   <li><tt>%default_type</tt>
   520    530   <li><tt>%destructor</tt>
          531  +<li><tt>%endif</tt>
   521    532   <li><tt>%extra_argument</tt>
          533  +<li><tt>%fallback</tt>
          534  +<li><tt>%ifdef</tt>
          535  +<li><tt>%ifndef</tt>
   522    536   <li><tt>%include</tt>
   523    537   <li><tt>%left</tt>
   524    538   <li><tt>%name</tt>
   525    539   <li><tt>%nonassoc</tt>
   526    540   <li><tt>%parse_accept</tt>
   527    541   <li><tt>%parse_failure </tt>
   528    542   <li><tt>%right</tt>
   529    543   <li><tt>%stack_overflow</tt>
   530    544   <li><tt>%stack_size</tt>
   531    545   <li><tt>%start_symbol</tt>
   532    546   <li><tt>%syntax_error</tt>
          547  +<li><tt>%token_class</tt>
   533    548   <li><tt>%token_destructor</tt>
   534    549   <li><tt>%token_prefix</tt>
   535    550   <li><tt>%token_type</tt>
   536    551   <li><tt>%type</tt>
          552  +<li><tt>%wildcard</tt>
   537    553   </ul>
   538    554   Each of these directives will be described separately in the
   539    555   following sections:</p>
   540    556   
          557  +<a name='pcode'></a>
   541    558   <h4>The <tt>%code</tt> directive</h4>
   542    559   
   543         -<p>The %code directive is used to specify addition C/C++ code that
          560  +<p>The %code directive is used to specify addition C code that
   544    561   is added to the end of the main output file.  This is similar to
   545         -the %include directive except that %include is inserted at the
   546         -beginning of the main output file.</p>
          562  +the <a href='#pinclude'>%include</a> directive except that %include
          563  +is inserted at the beginning of the main output file.</p>
   547    564   
   548    565   <p>%code is typically used to include some action routines or perhaps
   549         -a tokenizer as part of the output file.</p>
          566  +a tokenizer or even the "main()" function 
          567  +as part of the output file.</p>
   550    568   
          569  +<a name='default_destructor'></a>
   551    570   <h4>The <tt>%default_destructor</tt> directive</h4>
   552    571   
   553    572   <p>The %default_destructor directive specifies a destructor to 
   554    573   use for non-terminals that do not have their own destructor
   555    574   specified by a separate %destructor directive.  See the documentation
   556         -on the %destructor directive below for additional information.</p>
          575  +on the <a name='#destructor'>%destructor</a> directive below for
          576  +additional information.</p>
   557    577   
   558    578   <p>In some grammers, many different non-terminal symbols have the
   559    579   same datatype and hence the same destructor.  This directive is
   560    580   a convenience way to specify the same destructor for all those
   561    581   non-terminals using a single statement.</p>
   562    582   
          583  +<a name='default_type'></a>
   563    584   <h4>The <tt>%default_type</tt> directive</h4>
   564    585   
   565    586   <p>The %default_type directive specifies the datatype of non-terminal
   566    587   symbols that do no have their own datatype defined using a separate
   567         -%type directive.  See the documentation on %type below for addition
   568         -information.</p>
          588  +<a href='#ptype'>%type</a> directive.  
          589  +</p>
   569    590   
          591  +<a name='destructor'></a>
   570    592   <h4>The <tt>%destructor</tt> directive</h4>
   571    593   
   572    594   <p>The %destructor directive is used to specify a destructor for
   573    595   a non-terminal symbol.
   574         -(See also the %token_destructor directive which is used to
   575         -specify a destructor for terminal symbols.)</p>
          596  +(See also the <a href='#token_destructor'>%token_destructor</a>
          597  +directive which is used to specify a destructor for terminal symbols.)</p>
   576    598   
   577    599   <p>A non-terminal's destructor is called to dispose of the
   578    600   non-terminal's value whenever the non-terminal is popped from
   579    601   the stack.  This includes all of the following circumstances:
   580    602   <ul>
   581    603   <li> When a rule reduces and the value of a non-terminal on
   582    604        the right-hand side is not linked to C code.
................................................................................
   591    613   <pre>
   592    614      %type nt {void*}
   593    615      %destructor nt { free($$); }
   594    616      nt(A) ::= ID NUM.   { A = malloc( 100 ); }
   595    617   </pre>
   596    618   This example is a bit contrived but it serves to illustrate how
   597    619   destructors work.  The example shows a non-terminal named
   598         -``nt'' that holds values of type ``void*''.  When the rule for
   599         -an ``nt'' reduces, it sets the value of the non-terminal to
          620  +"nt" that holds values of type "void*".  When the rule for
          621  +an "nt" reduces, it sets the value of the non-terminal to
   600    622   space obtained from malloc().  Later, when the nt non-terminal
   601    623   is popped from the stack, the destructor will fire and call
   602    624   free() on this malloced space, thus avoiding a memory leak.
   603         -(Note that the symbol ``$$'' in the destructor code is replaced
          625  +(Note that the symbol "$$" in the destructor code is replaced
   604    626   by the value of the non-terminal.)</p>
   605    627   
   606    628   <p>It is important to note that the value of a non-terminal is passed
   607    629   to the destructor whenever the non-terminal is removed from the
   608    630   stack, unless the non-terminal is used in a C-code action.  If
   609    631   the non-terminal is used by C-code, then it is assumed that the
   610         -C-code will take care of destroying it if it should really
   611         -be destroyed.  More commonly, the value is used to build some
          632  +C-code will take care of destroying it.
          633  +More commonly, the value is used to build some
   612    634   larger structure and we don't want to destroy it, which is why
   613    635   the destructor is not called in this circumstance.</p>
   614    636   
   615         -<p>By appropriate use of destructors, it is possible to
   616         -build a parser using Lemon that can be used within a long-running
   617         -program, such as a GUI, that will not leak memory or other resources.
          637  +<p>Destructors help avoid memory leaks by automatically freeing
          638  +allocated objects when they go out of scope.
   618    639   To do the same using yacc or bison is much more difficult.</p>
   619    640   
          641  +<a name="extraarg"></a>
   620    642   <h4>The <tt>%extra_argument</tt> directive</h4>
   621    643   
   622    644   The %extra_argument directive instructs Lemon to add a 4th parameter
   623    645   to the parameter list of the Parse() function it generates.  Lemon
   624    646   doesn't do anything itself with this extra argument, but it does
   625    647   make the argument available to C-code action routines, destructors,
   626    648   and so forth.  For example, if the grammar file contains:</p>
   627    649   
   628    650   <p><pre>
   629    651       %extra_argument { MyStruct *pAbc }
   630    652   </pre></p>
   631    653   
   632    654   <p>Then the Parse() function generated will have an 4th parameter
   633         -of type ``MyStruct*'' and all action routines will have access to
   634         -a variable named ``pAbc'' that is the value of the 4th parameter
          655  +of type "MyStruct*" and all action routines will have access to
          656  +a variable named "pAbc" that is the value of the 4th parameter
   635    657   in the most recent call to Parse().</p>
   636    658   
          659  +<a name='pfallback'></a>
          660  +<h4>The <tt>%fallback</tt> directive</h4>
          661  +
          662  +<p>The %fallback directive specifies an alternative meaning for one
          663  +or more tokens.  The alternative meaning is tried if the original token
          664  +would have generated a syntax error.
          665  +
          666  +<p>The %fallback directive was added to support robust parsing of SQL
          667  +syntax in <a href="https://www.sqlite.org/">SQLite</a>.
          668  +The SQL language contains a large assortment of keywords, each of which
          669  +appears as a different token to the language parser.  SQL contains so
          670  +many keywords, that it can be difficult for programmers to keep up with
          671  +them all.  Programmers will, therefore, sometimes mistakenly use an
          672  +obscure language keyword for an identifier.  The %fallback directive
          673  +provides a mechanism to tell the parser:  "If you are unable to parse
          674  +this keyword, try treating it as an identifier instead."
          675  +
          676  +<p>The syntax of %fallback is as follows:
          677  +
          678  +<blockquote>
          679  +<tt>%fallback</tt>  <i>ID</i> <i>TOKEN...</i> <b>.</b>
          680  +</blockquote>
          681  +
          682  +<p>In words, the %fallback directive is followed by a list of token names
          683  +terminated by a period.  The first token name is the fallback token - the
          684  +token to which all the other tokens fall back to.  The second and subsequent
          685  +arguments are tokens which fall back to the token identified by the first
          686  +argument.
          687  +
          688  +<a name='pifdef'></a>
          689  +<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4>
          690  +
          691  +<p>The %ifdef, %ifndef, and %endif directives are similar to
          692  +#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general.
          693  +Each of these directives must begin at the left margin.  No whitespace
          694  +is allowed between the "%" and the directive name.
          695  +
          696  +<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is
          697  +ignored unless the "-DMACRO" command-line option is used.  Grammar text
          698  +betwen "%ifndef MACRO" and the next nested "%endif" is included except when
          699  +the "-DMACRO" command-line option is used.
          700  +
          701  +<p>Note that the argument to %ifdef and %ifndef must be a single 
          702  +preprocessor symbol name, not a general expression.  There is no "%else"
          703  +directive.
          704  +
          705  +
          706  +<a name='pinclude'></a>
   637    707   <h4>The <tt>%include</tt> directive</h4>
   638    708   
   639    709   <p>The %include directive specifies C code that is included at the
   640    710   top of the generated parser.  You can include any text you want --
   641    711   the Lemon parser generator copies it blindly.  If you have multiple
   642         -%include directives in your grammar file the value of the last
   643         -%include directive overwrites all the others.</p.
          712  +%include directives in your grammar file, their values are concatenated
          713  +so that all %include code ultimately appears near the top of the
          714  +generated parser, in the same order as it appeared in the grammer.</p>
   644    715   
   645    716   <p>The %include directive is very handy for getting some extra #include
   646    717   preprocessor statements at the beginning of the generated parser.
   647    718   For example:</p>
   648    719   
   649    720   <p><pre>
   650    721      %include {#include &lt;unistd.h&gt;}
   651    722   </pre></p>
   652    723   
   653    724   <p>This might be needed, for example, if some of the C actions in the
   654    725   grammar call functions that are prototyed in unistd.h.</p>
   655    726   
          727  +<a name='pleft'></a>
   656    728   <h4>The <tt>%left</tt> directive</h4>
   657    729   
   658         -The %left directive is used (along with the %right and
   659         -%nonassoc directives) to declare precedences of terminal
   660         -symbols.  Every terminal symbol whose name appears after
   661         -a %left directive but before the next period (``.'') is
          730  +The %left directive is used (along with the <a href='#pright'>%right</a> and
          731  +<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 
          732  +terminal symbols.  Every terminal symbol whose name appears after
          733  +a %left directive but before the next period (".") is
   662    734   given the same left-associative precedence value.  Subsequent
   663    735   %left directives have higher precedence.  For example:</p>
   664    736   
   665    737   <p><pre>
   666    738      %left AND.
   667    739      %left OR.
   668    740      %nonassoc EQ NE GT GE LT LE.
................................................................................
   675    747   directive.</p>
   676    748   
   677    749   <p>LALR(1) grammars can get into a situation where they require
   678    750   a large amount of stack space if you make heavy use or right-associative
   679    751   operators.  For this reason, it is recommended that you use %left
   680    752   rather than %right whenever possible.</p>
   681    753   
          754  +<a name='pname'></a>
   682    755   <h4>The <tt>%name</tt> directive</h4>
   683    756   
   684    757   <p>By default, the functions generated by Lemon all begin with the
   685         -five-character string ``Parse''.  You can change this string to something
          758  +five-character string "Parse".  You can change this string to something
   686    759   different using the %name directive.  For instance:</p>
   687    760   
   688    761   <p><pre>
   689    762      %name Abcde
   690    763   </pre></p>
   691    764   
   692    765   <p>Putting this directive in the grammar file will cause Lemon to generate
................................................................................
   697    770   <li> AbcdeTrace(), and
   698    771   <li> Abcde().
   699    772   </ul>
   700    773   The %name directive allows you to generator two or more different
   701    774   parsers and link them all into the same executable.
   702    775   </p>
   703    776   
          777  +<a name='pnonassoc'></a>
   704    778   <h4>The <tt>%nonassoc</tt> directive</h4>
   705    779   
   706    780   <p>This directive is used to assign non-associative precedence to
   707         -one or more terminal symbols.  See the section on precedence rules
   708         -or on the %left directive for additional information.</p>
          781  +one or more terminal symbols.  See the section on 
          782  +<a href='#precrules'>precedence rules</a>
          783  +or on the <a href='#pleft'>%left</a> directive for additional information.</p>
   709    784   
          785  +<a name='parse_accept'></a>
   710    786   <h4>The <tt>%parse_accept</tt> directive</h4>
   711    787   
   712    788   <p>The %parse_accept directive specifies a block of C code that is
   713         -executed whenever the parser accepts its input string.  To ``accept''
          789  +executed whenever the parser accepts its input string.  To "accept"
   714    790   an input string means that the parser was able to process all tokens
   715    791   without error.</p>
   716    792   
   717    793   <p>For example:</p>
   718    794   
   719    795   <p><pre>
   720    796      %parse_accept {
   721    797         printf("parsing complete!\n");
   722    798      }
   723    799   </pre></p>
   724    800   
   725         -
          801  +<a name='parse_failure'></a>
   726    802   <h4>The <tt>%parse_failure</tt> directive</h4>
   727    803   
   728    804   <p>The %parse_failure directive specifies a block of C code that
   729    805   is executed whenever the parser fails complete.  This code is not
   730    806   executed until the parser has tried and failed to resolve an input
   731    807   error using is usual error recovery strategy.  The routine is
   732    808   only invoked when parsing is unable to continue.</p>
................................................................................
   733    809   
   734    810   <p><pre>
   735    811      %parse_failure {
   736    812        fprintf(stderr,"Giving up.  Parser is hopelessly lost...\n");
   737    813      }
   738    814   </pre></p>
   739    815   
          816  +<a name='pright'></a>
   740    817   <h4>The <tt>%right</tt> directive</h4>
   741    818   
   742    819   <p>This directive is used to assign right-associative precedence to
   743         -one or more terminal symbols.  See the section on precedence rules
   744         -or on the %left directive for additional information.</p>
          820  +one or more terminal symbols.  See the section on 
          821  +<a href='#precrules'>precedence rules</a>
          822  +or on the <a href='#pleft'>%left</a> directive for additional information.</p>
   745    823   
          824  +<a name='stack_overflow'></a>
   746    825   <h4>The <tt>%stack_overflow</tt> directive</h4>
   747    826   
   748    827   <p>The %stack_overflow directive specifies a block of C code that
   749    828   is executed if the parser's internal stack ever overflows.  Typically
   750    829   this just prints an error message.  After a stack overflow, the parser
   751    830   will be unable to continue and must be reset.</p>
   752    831   
................................................................................
   767    846   </pre>
   768    847   Not like this:
   769    848   <pre>
   770    849      list ::= element list.      // right-recursion.  Bad!
   771    850      list ::= .
   772    851   </pre>
   773    852   
          853  +<a name='stack_size'></a>
   774    854   <h4>The <tt>%stack_size</tt> directive</h4>
   775    855   
   776    856   <p>If stack overflow is a problem and you can't resolve the trouble
   777    857   by using left-recursion, then you might want to increase the size
   778    858   of the parser's stack using this directive.  Put an positive integer
   779    859   after the %stack_size directive and Lemon will generate a parse
   780    860   with a stack of the requested size.  The default value is 100.</p>
   781    861   
   782    862   <p><pre>
   783    863      %stack_size 2000
   784    864   </pre></p>
   785    865   
          866  +<a name='start_symbol'></a>
   786    867   <h4>The <tt>%start_symbol</tt> directive</h4>
   787    868   
   788    869   <p>By default, the start-symbol for the grammar that Lemon generates
   789    870   is the first non-terminal that appears in the grammar file.  But you
   790    871   can choose a different start-symbol using the %start_symbol directive.</p>
   791    872   
   792    873   <p><pre>
   793    874      %start_symbol  prog
   794    875   </pre></p>
   795    876   
          877  +<a name='token_destructor'></a>
   796    878   <h4>The <tt>%token_destructor</tt> directive</h4>
   797    879   
   798    880   <p>The %destructor directive assigns a destructor to a non-terminal
   799    881   symbol.  (See the description of the %destructor directive above.)
   800    882   This directive does the same thing for all terminal symbols.</p>
   801    883   
   802    884   <p>Unlike non-terminal symbols which may each have a different data type
   803    885   for their values, terminals all use the same data type (defined by
   804    886   the %token_type directive) and so they use a common destructor.  Other
   805    887   than that, the token destructor works just like the non-terminal
   806    888   destructors.</p>
   807    889   
          890  +<a name='token_prefix'></a>
   808    891   <h4>The <tt>%token_prefix</tt> directive</h4>
   809    892   
   810    893   <p>Lemon generates #defines that assign small integer constants
   811    894   to each terminal symbol in the grammar.  If desired, Lemon will
   812    895   add a prefix specified by this directive
   813    896   to each of the #defines it generates.
   814    897   So if the default output of Lemon looked like this:
................................................................................
   826    909   <pre>
   827    910       #define TOKEN_AND        1
   828    911       #define TOKEN_MINUS      2
   829    912       #define TOKEN_OR         3
   830    913       #define TOKEN_PLUS       4
   831    914   </pre>
   832    915   
          916  +<a name='token_type'></a><a name='ptype'></a>
   833    917   <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4>
   834    918   
   835    919   <p>These directives are used to specify the data types for values
   836    920   on the parser's stack associated with terminal and non-terminal
   837    921   symbols.  The values of all terminal symbols must be of the same
   838    922   type.  This turns out to be the same data type as the 3rd parameter
   839    923   to the Parse() function generated by Lemon.  Typically, you will
................................................................................
   841    925   token structure.  Like this:</p>
   842    926   
   843    927   <p><pre>
   844    928      %token_type    {Token*}
   845    929   </pre></p>
   846    930   
   847    931   <p>If the data type of terminals is not specified, the default value
   848         -is ``int''.</p>
          932  +is "int".</p>
   849    933   
   850    934   <p>Non-terminal symbols can each have their own data types.  Typically
   851    935   the data type  of a non-terminal is a pointer to the root of a parse-tree
   852    936   structure that contains all information about that non-terminal.
   853    937   For example:</p>
   854    938   
   855    939   <p><pre>
................................................................................
   862    946   on what the corresponding non-terminal or terminal symbol is.  But
   863    947   the grammar designer should keep in mind that the size of the union
   864    948   will be the size of its largest element.  So if you have a single
   865    949   non-terminal whose data type requires 1K of storage, then your 100
   866    950   entry parser stack will require 100K of heap space.  If you are willing
   867    951   and able to pay that price, fine.  You just need to know.</p>
   868    952   
          953  +<a name='pwildcard'></a>
          954  +<h4>The <tt>%wildcard</tt> directive</h4>
          955  +
          956  +<p>The %wildcard directive is followed by a single token name and a
          957  +period.  This directive specifies that the identified token should 
          958  +match any input token.
          959  +
          960  +<p>When the generated parser has the choice of matching an input against
          961  +the wildcard token and some other token, the other token is always used.
          962  +The wildcard token is only matched if there are no other alternatives.
          963  +
   869    964   <h3>Error Processing</h3>
   870    965   
   871    966   <p>After extensive experimentation over several years, it has been
   872    967   discovered that the error recovery strategy used by yacc is about
   873    968   as good as it gets.  And so that is what Lemon uses.</p>
   874    969   
   875    970   <p>When a Lemon-generated parser encounters a syntax error, it
   876    971   first invokes the code specified by the %syntax_error directive, if
   877    972   any.  It then enters its error recovery strategy.  The error recovery
   878    973   strategy is to begin popping the parsers stack until it enters a
   879    974   state where it is permitted to shift a special non-terminal symbol
   880         -named ``error''.  It then shifts this non-terminal and continues
          975  +named "error".  It then shifts this non-terminal and continues
   881    976   parsing.  But the %syntax_error routine will not be called again
   882    977   until at least three new tokens have been successfully shifted.</p>
   883    978   
   884    979   <p>If the parser pops its stack until the stack is empty, and it still
   885    980   is unable to shift the error symbol, then the %parse_failed routine
   886    981   is invoked and the parser resets itself to its start state, ready
   887    982   to begin parsing a new file.  This is what will happen at the very
   888    983   first syntax error, of course, if there are no instances of the 
   889         -``error'' non-terminal in your grammar.</p>
          984  +"error" non-terminal in your grammar.</p>
   890    985   
   891    986   </body>
   892    987   </html>

Changes to ext/fts3/fts3Int.h.

    13     13   */
    14     14   #ifndef _FTSINT_H
    15     15   #define _FTSINT_H
    16     16   
    17     17   #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
    18     18   # define NDEBUG 1
    19     19   #endif
           20  +
           21  +/* FTS3/FTS4 require virtual tables */
           22  +#ifdef SQLITE_OMIT_VIRTUALTABLE
           23  +# undef SQLITE_ENABLE_FTS3
           24  +# undef SQLITE_ENABLE_FTS4
           25  +#endif
    20     26   
    21     27   /*
    22     28   ** FTS4 is really an extension for FTS3.  It is enabled using the
    23     29   ** SQLITE_ENABLE_FTS3 macro.  But to avoid confusion we also all
    24     30   ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
    25     31   */
    26     32   #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)

Changes to ext/fts3/fts3_test.c.

   522    522     Tcl_Interp *interp,
   523    523     int objc,
   524    524     Tcl_Obj *CONST objv[]
   525    525   ){
   526    526   #ifdef SQLITE_ENABLE_FTS3
   527    527     char aBuf[24];
   528    528     int rc;
   529         -  Tcl_WideInt w, w2;
          529  +  Tcl_WideInt w;
          530  +  sqlite3_int64 w2;
   530    531     int nByte, nByte2;
   531    532   
   532    533     if( objc!=2 ){
   533    534       Tcl_WrongNumArgs(interp, 1, objv, "INTEGER");
   534    535       return TCL_ERROR;
   535    536     }
   536    537   

Changes to ext/fts3/fts3_tokenizer.c.

    24     24   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    25     25   */
    26     26   #include "fts3Int.h"
    27     27   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    28     28   
    29     29   #include <assert.h>
    30     30   #include <string.h>
           31  +
           32  +/*
           33  +** Return true if the two-argument version of fts3_tokenizer()
           34  +** has been activated via a prior call to sqlite3_db_config(db,
           35  +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0);
           36  +*/
           37  +static int fts3TokenizerEnabled(sqlite3_context *context){
           38  +  sqlite3 *db = sqlite3_context_db_handle(context);
           39  +  int isEnabled = 0;
           40  +  sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled);
           41  +  return isEnabled;
           42  +}
    31     43   
    32     44   /*
    33     45   ** Implementation of the SQL scalar function for accessing the underlying 
    34     46   ** hash table. This function may be called as follows:
    35     47   **
    36     48   **   SELECT <function-name>(<key-name>);
    37     49   **   SELECT <function-name>(<key-name>, <pointer>);
................................................................................
    45     57   ** the string <key-name> must already exist in the has table. Otherwise,
    46     58   ** an error is returned.
    47     59   **
    48     60   ** Whether or not the <pointer> argument is specified, the value returned
    49     61   ** is a blob containing the pointer stored as the hash data corresponding
    50     62   ** to string <key-name> (after the hash-table is updated, if applicable).
    51     63   */
    52         -static void scalarFunc(
           64  +static void fts3TokenizerFunc(
    53     65     sqlite3_context *context,
    54     66     int argc,
    55     67     sqlite3_value **argv
    56     68   ){
    57     69     Fts3Hash *pHash;
    58     70     void *pPtr = 0;
    59     71     const unsigned char *zName;
................................................................................
    63     75   
    64     76     pHash = (Fts3Hash *)sqlite3_user_data(context);
    65     77   
    66     78     zName = sqlite3_value_text(argv[0]);
    67     79     nName = sqlite3_value_bytes(argv[0])+1;
    68     80   
    69     81     if( argc==2 ){
    70         -    void *pOld;
    71         -    int n = sqlite3_value_bytes(argv[1]);
    72         -    if( zName==0 || n!=sizeof(pPtr) ){
    73         -      sqlite3_result_error(context, "argument type mismatch", -1);
    74         -      return;
    75         -    }
    76         -    pPtr = *(void **)sqlite3_value_blob(argv[1]);
    77         -    pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
    78         -    if( pOld==pPtr ){
    79         -      sqlite3_result_error(context, "out of memory", -1);
           82  +    if( fts3TokenizerEnabled(context) ){
           83  +      void *pOld;
           84  +      int n = sqlite3_value_bytes(argv[1]);
           85  +      if( zName==0 || n!=sizeof(pPtr) ){
           86  +        sqlite3_result_error(context, "argument type mismatch", -1);
           87  +        return;
           88  +      }
           89  +      pPtr = *(void **)sqlite3_value_blob(argv[1]);
           90  +      pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
           91  +      if( pOld==pPtr ){
           92  +        sqlite3_result_error(context, "out of memory", -1);
           93  +      }
           94  +    }else{
           95  +      sqlite3_result_error(context, "fts3tokenize disabled", -1);
    80     96         return;
    81     97       }
    82     98     }else{
    83     99       if( zName ){
    84    100         pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
    85    101       }
    86    102       if( !pPtr ){
    87    103         char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
    88    104         sqlite3_result_error(context, zErr, -1);
    89    105         sqlite3_free(zErr);
    90    106         return;
    91    107       }
    92    108     }
    93         -
    94    109     sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
    95    110   }
    96    111   
    97    112   int sqlite3Fts3IsIdChar(char c){
    98    113     static const char isFtsIdChar[] = {
    99    114         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
   100    115         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
................................................................................
   345    360   
   346    361     sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
   347    362     sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
   348    363     sqlite3_step(pStmt);
   349    364   
   350    365     return sqlite3_finalize(pStmt);
   351    366   }
          367  +
   352    368   
   353    369   static
   354    370   int queryTokenizer(
   355    371     sqlite3 *db, 
   356    372     char *zName,  
   357    373     const sqlite3_tokenizer_module **pp
   358    374   ){
................................................................................
   416    432     assert( p1==p2 );
   417    433     rc = queryTokenizer(db, "nosuchtokenizer", &p2);
   418    434     assert( rc==SQLITE_ERROR );
   419    435     assert( p2==0 );
   420    436     assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
   421    437   
   422    438     /* Test the storage function */
   423         -  rc = registerTokenizer(db, "nosuchtokenizer", p1);
   424         -  assert( rc==SQLITE_OK );
   425         -  rc = queryTokenizer(db, "nosuchtokenizer", &p2);
   426         -  assert( rc==SQLITE_OK );
   427         -  assert( p2==p1 );
          439  +  if( fts3TokenizerEnabled(context) ){
          440  +    rc = registerTokenizer(db, "nosuchtokenizer", p1);
          441  +    assert( rc==SQLITE_OK );
          442  +    rc = queryTokenizer(db, "nosuchtokenizer", &p2);
          443  +    assert( rc==SQLITE_OK );
          444  +    assert( p2==p1 );
          445  +  }
   428    446   
   429    447     sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
   430    448   }
   431    449   
   432    450   #endif
   433    451   
   434    452   /*
................................................................................
   436    454   ** the hash table pointed to by argument pHash. The hash table must
   437    455   ** been initialized to use string keys, and to take a private copy 
   438    456   ** of the key when a value is inserted. i.e. by a call similar to:
   439    457   **
   440    458   **    sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
   441    459   **
   442    460   ** This function adds a scalar function (see header comment above
   443         -** scalarFunc() in this file for details) and, if ENABLE_TABLE is
          461  +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is
   444    462   ** defined at compilation time, a temporary virtual table (see header 
   445    463   ** comment above struct HashTableVtab) to the database schema. Both 
   446    464   ** provide read/write access to the contents of *pHash.
   447    465   **
   448    466   ** The third argument to this function, zName, is used as the name
   449    467   ** of both the scalar and, if created, the virtual table.
   450    468   */
................................................................................
   465    483     zTest2 = sqlite3_mprintf("%s_internal_test", zName);
   466    484     if( !zTest || !zTest2 ){
   467    485       rc = SQLITE_NOMEM;
   468    486     }
   469    487   #endif
   470    488   
   471    489     if( SQLITE_OK==rc ){
   472         -    rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
          490  +    rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0);
   473    491     }
   474    492     if( SQLITE_OK==rc ){
   475         -    rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
          493  +    rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0);
   476    494     }
   477    495   #ifdef SQLITE_TEST
   478    496     if( SQLITE_OK==rc ){
   479    497       rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0);
   480    498     }
   481    499     if( SQLITE_OK==rc ){
   482    500       rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);

Changes to ext/fts3/fts3_write.c.

   329    329   /* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'",
   330    330   
   331    331   /* This statement is used to determine which level to read the input from
   332    332   ** when performing an incremental merge. It returns the absolute level number
   333    333   ** of the oldest level in the db that contains at least ? segments. Or,
   334    334   ** if no level in the FTS index contains more than ? segments, the statement
   335    335   ** returns zero rows.  */
   336         -/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
          336  +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
          337  +         "  GROUP BY level HAVING cnt>=?"
   337    338            "  ORDER BY (level %% 1024) ASC LIMIT 1",
   338    339   
   339    340   /* Estimate the upper limit on the number of leaf nodes in a new segment
   340    341   ** created by merging the oldest :2 segments from absolute level :1. See 
   341    342   ** function sqlite3Fts3Incrmerge() for details.  */
   342    343   /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
   343    344            "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
................................................................................
  3190   3191     }
  3191   3192   
  3192   3193     if( iLevel==FTS3_SEGCURSOR_ALL ){
  3193   3194       /* This call is to merge all segments in the database to a single
  3194   3195       ** segment. The level of the new segment is equal to the numerically
  3195   3196       ** greatest segment level currently present in the database for this
  3196   3197       ** index. The idx of the new segment is always 0.  */
  3197         -    if( csr.nSegment==1 ){
         3198  +    if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){
  3198   3199         rc = SQLITE_DONE;
  3199   3200         goto finished;
  3200   3201       }
  3201   3202       iNewLevel = iMaxLevel;
  3202   3203       bIgnoreEmpty = 1;
  3203   3204   
  3204   3205     }else{
................................................................................
  4832   4833       /* Search the %_segdir table for the absolute level with the smallest
  4833   4834       ** relative level number that contains at least nMin segments, if any.
  4834   4835       ** If one is found, set iAbsLevel to the absolute level number and
  4835   4836       ** nSeg to nMin. If no level with at least nMin segments can be found, 
  4836   4837       ** set nSeg to -1.
  4837   4838       */
  4838   4839       rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
  4839         -    sqlite3_bind_int(pFindLevel, 1, nMin);
         4840  +    sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin));
  4840   4841       if( sqlite3_step(pFindLevel)==SQLITE_ROW ){
  4841   4842         iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
  4842         -      nSeg = nMin;
         4843  +      nSeg = sqlite3_column_int(pFindLevel, 1);
         4844  +      assert( nSeg>=2 );
  4843   4845       }else{
  4844   4846         nSeg = -1;
  4845   4847       }
  4846   4848       rc = sqlite3_reset(pFindLevel);
  4847   4849   
  4848   4850       /* If the hint read from the %_stat table is not empty, check if the
  4849   4851       ** last entry in it specifies a relative level smaller than or equal

Changes to ext/fts3/unicode/mkunicode.tcl.

   222    222     puts "** The results are undefined if the value passed to this function"
   223    223     puts "** is less than zero."
   224    224     puts "*/"
   225    225     puts "int ${zFunc}\(int c)\{"
   226    226     an_print_range_array $lRange
   227    227     an_print_ascii_bitmap $lRange
   228    228     puts {
   229         -  if( c<128 ){
          229  +  if( (unsigned int)c<128 ){
   230    230       return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   231         -  }else if( c<(1<<22) ){
          231  +  }else if( (unsigned int)c<(1<<22) ){
   232    232       unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   233    233       int iRes = 0;
   234    234       int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   235    235       int iLo = 0;
   236    236       while( iHi>=iLo ){
   237    237         int iTest = (iHi + iLo) / 2;
   238    238         if( key >= aEntry[iTest] ){

Changes to ext/fts5/fts5.h.

    80     80   **   *pnToken to the number of tokens in column iCol of the current row.
    81     81   **
    82     82   **   If parameter iCol is greater than or equal to the number of columns
    83     83   **   in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
    84     84   **   an OOM condition or IO error), an appropriate SQLite error code is 
    85     85   **   returned.
    86     86   **
           87  +**   This function may be quite inefficient if used with an FTS5 table
           88  +**   created with the "columnsize=0" option.
           89  +**
    87     90   ** xColumnText:
    88     91   **   This function attempts to retrieve the text of column iCol of the
    89     92   **   current document. If successful, (*pz) is set to point to a buffer
    90     93   **   containing the text in utf-8 encoding, (*pn) is set to the size in bytes
    91     94   **   (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
    92     95   **   if an error occurs, an SQLite error code is returned and the final values
    93     96   **   of (*pz) and (*pn) are undefined.
................................................................................
   100    103   **   are numbered starting from zero.
   101    104   **
   102    105   ** xInstCount:
   103    106   **   Set *pnInst to the total number of occurrences of all phrases within
   104    107   **   the query within the current row. Return SQLITE_OK if successful, or
   105    108   **   an error code (i.e. SQLITE_NOMEM) if an error occurs.
   106    109   **
          110  +**   This API can be quite slow if used with an FTS5 table created with the
          111  +**   "detail=none" or "detail=column" option. If the FTS5 table is created 
          112  +**   with either "detail=none" or "detail=column" and "content=" option 
          113  +**   (i.e. if it is a contentless table), then this API always returns 0.
          114  +**
   107    115   ** xInst:
   108    116   **   Query for the details of phrase match iIdx within the current row.
   109    117   **   Phrase matches are numbered starting from zero, so the iIdx argument
   110    118   **   should be greater than or equal to zero and smaller than the value
   111    119   **   output by xInstCount().
          120  +**
          121  +**   Usually, output parameter *piPhrase is set to the phrase number, *piCol
          122  +**   to the column in which it occurs and *piOff the token offset of the
          123  +**   first token of the phrase. The exception is if the table was created
          124  +**   with the offsets=0 option specified. In this case *piOff is always
          125  +**   set to -1.
   112    126   **
   113    127   **   Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 
   114    128   **   if an error occurs.
          129  +**
          130  +**   This API can be quite slow if used with an FTS5 table created with the
          131  +**   "detail=none" or "detail=column" option. 
   115    132   **
   116    133   ** xRowid:
   117    134   **   Returns the rowid of the current row.
   118    135   **
   119    136   ** xTokenize:
   120    137   **   Tokenize text using the tokenizer belonging to the FTS5 table.
   121    138   **
................................................................................
   192    209   **   xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
   193    210   **   to use, this API may be faster under some circumstances. To iterate 
   194    211   **   through instances of phrase iPhrase, use the following code:
   195    212   **
   196    213   **       Fts5PhraseIter iter;
   197    214   **       int iCol, iOff;
   198    215   **       for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
   199         -**           iOff>=0;
          216  +**           iCol>=0;
   200    217   **           pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
   201    218   **       ){
   202    219   **         // An instance of phrase iPhrase at offset iOff of column iCol
   203    220   **       }
   204    221   **
   205    222   **   The Fts5PhraseIter structure is defined above. Applications should not
   206    223   **   modify this structure directly - it should only be used as shown above
   207         -**   with the xPhraseFirst() and xPhraseNext() API methods.
          224  +**   with the xPhraseFirst() and xPhraseNext() API methods (and by
          225  +**   xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
          226  +**
          227  +**   This API can be quite slow if used with an FTS5 table created with the
          228  +**   "detail=none" or "detail=column" option. If the FTS5 table is created 
          229  +**   with either "detail=none" or "detail=column" and "content=" option 
          230  +**   (i.e. if it is a contentless table), then this API always iterates
          231  +**   through an empty set (all calls to xPhraseFirst() set iCol to -1).
   208    232   **
   209    233   ** xPhraseNext()
   210    234   **   See xPhraseFirst above.
          235  +**
          236  +** xPhraseFirstColumn()
          237  +**   This function and xPhraseNextColumn() are similar to the xPhraseFirst()
          238  +**   and xPhraseNext() APIs described above. The difference is that instead
          239  +**   of iterating through all instances of a phrase in the current row, these
          240  +**   APIs are used to iterate through the set of columns in the current row
          241  +**   that contain one or more instances of a specified phrase. For example:
          242  +**
          243  +**       Fts5PhraseIter iter;
          244  +**       int iCol;
          245  +**       for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
          246  +**           iCol>=0;
          247  +**           pApi->xPhraseNextColumn(pFts, &iter, &iCol)
          248  +**       ){
          249  +**         // Column iCol contains at least one instance of phrase iPhrase
          250  +**       }
          251  +**
          252  +**   This API can be quite slow if used with an FTS5 table created with the
          253  +**   "detail=none" option. If the FTS5 table is created with either 
          254  +**   "detail=none" "content=" option (i.e. if it is a contentless table), 
          255  +**   then this API always iterates through an empty set (all calls to 
          256  +**   xPhraseFirstColumn() set iCol to -1).
          257  +**
          258  +**   The information accessed using this API and its companion
          259  +**   xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
          260  +**   (or xInst/xInstCount). The chief advantage of this API is that it is
          261  +**   significantly more efficient than those alternatives when used with
          262  +**   "detail=column" tables.  
          263  +**
          264  +** xPhraseNextColumn()
          265  +**   See xPhraseFirstColumn above.
   211    266   */
   212    267   struct Fts5ExtensionApi {
   213         -  int iVersion;                   /* Currently always set to 1 */
          268  +  int iVersion;                   /* Currently always set to 3 */
   214    269   
   215    270     void *(*xUserData)(Fts5Context*);
   216    271   
   217    272     int (*xColumnCount)(Fts5Context*);
   218    273     int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
   219    274     int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
   220    275   
................................................................................
   236    291   
   237    292     int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
   238    293       int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
   239    294     );
   240    295     int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
   241    296     void *(*xGetAuxdata)(Fts5Context*, int bClear);
   242    297   
   243         -  void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
          298  +  int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
   244    299     void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
          300  +
          301  +  int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
          302  +  void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
   245    303   };
   246    304   
   247    305   /* 
   248    306   ** CUSTOM AUXILIARY FUNCTIONS
   249    307   *************************************************************************/
   250    308   
   251    309   /*************************************************************************

Changes to ext/fts5/fts5Int.h.

    22     22   #include <assert.h>
    23     23   
    24     24   #ifndef SQLITE_AMALGAMATION
    25     25   
    26     26   typedef unsigned char  u8;
    27     27   typedef unsigned int   u32;
    28     28   typedef unsigned short u16;
           29  +typedef short i16;
    29     30   typedef sqlite3_int64 i64;
    30     31   typedef sqlite3_uint64 u64;
    31     32   
    32         -#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
           33  +#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
    33     34   
    34     35   #define testcase(x)
    35     36   #define ALWAYS(x) 1
    36     37   #define NEVER(x) 0
    37     38   
    38     39   #define MIN(x,y) (((x) < (y)) ? (x) : (y))
    39     40   #define MAX(x,y) (((x) > (y)) ? (x) : (y))
................................................................................
    42     43   ** Constants for the largest and smallest possible 64-bit signed integers.
    43     44   */
    44     45   # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
    45     46   # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
    46     47   
    47     48   #endif
    48     49   
           50  +/* Truncate very long tokens to this many bytes. Hard limit is 
           51  +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
           52  +** field that occurs at the start of each leaf page (see fts5_index.c). */
           53  +#define FTS5_MAX_TOKEN_SIZE 32768
    49     54   
    50     55   /*
    51     56   ** Maximum number of prefix indexes on single FTS5 table. This must be
    52     57   ** less than 32. If it is set to anything large than that, an #error
    53     58   ** directive in fts5_index.c will cause the build to fail.
    54     59   */
    55     60   #define FTS5_MAX_PREFIX_INDEXES 31
................................................................................
    75     80   */
    76     81   #ifdef SQLITE_DEBUG
    77     82   extern int sqlite3_fts5_may_be_corrupt;
    78     83   # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
    79     84   #else
    80     85   # define assert_nc(x) assert(x)
    81     86   #endif
           87  +
           88  +/* Mark a function parameter as unused, to suppress nuisance compiler
           89  +** warnings. */
           90  +#ifndef UNUSED_PARAM
           91  +# define UNUSED_PARAM(X)  (void)(X)
           92  +#endif
           93  +
           94  +#ifndef UNUSED_PARAM2
           95  +# define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
           96  +#endif
    82     97   
    83     98   typedef struct Fts5Global Fts5Global;
    84     99   typedef struct Fts5Colset Fts5Colset;
    85    100   
    86    101   /* If a NEAR() clump or phrase may only match a specific set of columns, 
    87    102   ** then an object of the following type is used to record the set of columns.
    88    103   ** Each entry in the aiCol[] array is a column that may be matched.
................................................................................
   147    162     u8 *abUnindexed;                /* True for unindexed columns */
   148    163     int nPrefix;                    /* Number of prefix indexes */
   149    164     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   150    165     int eContent;                   /* An FTS5_CONTENT value */
   151    166     char *zContent;                 /* content table */ 
   152    167     char *zContentRowid;            /* "content_rowid=" option value */ 
   153    168     int bColumnsize;                /* "columnsize=" option value (dflt==1) */
          169  +  int eDetail;                    /* FTS5_DETAIL_XXX value */
   154    170     char *zContentExprlist;
   155    171     Fts5Tokenizer *pTok;
   156    172     fts5_tokenizer *pTokApi;
   157    173   
   158    174     /* Values loaded from the %_config table */
   159    175     int iCookie;                    /* Incremented when %_config is modified */
   160    176     int pgsz;                       /* Approximate page size used in %_data */
   161    177     int nAutomerge;                 /* 'automerge' setting */
   162    178     int nCrisisMerge;               /* Maximum allowed segments per level */
          179  +  int nUsermerge;                 /* 'usermerge' setting */
   163    180     int nHashSize;                  /* Bytes of memory for in-memory hash */
   164    181     char *zRank;                    /* Name of rank function */
   165    182     char *zRankArgs;                /* Arguments to rank function */
   166    183   
   167    184     /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   168    185     char **pzErrmsg;
   169    186   
................................................................................
   175    192   /* Current expected value of %_config table 'version' field */
   176    193   #define FTS5_CURRENT_VERSION 4
   177    194   
   178    195   #define FTS5_CONTENT_NORMAL   0
   179    196   #define FTS5_CONTENT_NONE     1
   180    197   #define FTS5_CONTENT_EXTERNAL 2
   181    198   
          199  +#define FTS5_DETAIL_FULL    0
          200  +#define FTS5_DETAIL_NONE    1
          201  +#define FTS5_DETAIL_COLUMNS 2
   182    202   
   183    203   
   184    204   
   185    205   int sqlite3Fts5ConfigParse(
   186    206       Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
   187    207   );
   188    208   void sqlite3Fts5ConfigFree(Fts5Config*);
................................................................................
   221    241   typedef struct Fts5Buffer Fts5Buffer;
   222    242   struct Fts5Buffer {
   223    243     u8 *p;
   224    244     int n;
   225    245     int nSpace;
   226    246   };
   227    247   
   228         -int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int);
          248  +int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
   229    249   void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
   230         -void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
          250  +void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
   231    251   void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
   232    252   void sqlite3Fts5BufferFree(Fts5Buffer*);
   233    253   void sqlite3Fts5BufferZero(Fts5Buffer*);
   234    254   void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
   235    255   void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
   236    256   
   237    257   char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
................................................................................
   239    259   #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
   240    260   #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
   241    261   #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
   242    262   #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
   243    263   #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
   244    264   
   245    265   #define fts5BufferGrow(pRc,pBuf,nn) ( \
   246         -  (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \
          266  +  (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
   247    267       sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
   248    268   )
   249    269   
   250    270   /* Write and decode big-endian 32-bit integer values */
   251    271   void sqlite3Fts5Put32(u8*, int);
   252    272   int sqlite3Fts5Get32(const u8*);
   253    273   
................................................................................
   274    294   int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
   275    295   
   276    296   typedef struct Fts5PoslistWriter Fts5PoslistWriter;
   277    297   struct Fts5PoslistWriter {
   278    298     i64 iPrev;
   279    299   };
   280    300   int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
          301  +void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
   281    302   
   282    303   int sqlite3Fts5PoslistNext64(
   283    304     const u8 *a, int n,             /* Buffer containing poslist */
   284    305     int *pi,                        /* IN/OUT: Offset within a[] */
   285    306     i64 *piOff                      /* IN/OUT: Current offset */
   286    307   );
   287    308   
................................................................................
   288    309   /* Malloc utility */
   289    310   void *sqlite3Fts5MallocZero(int *pRc, int nByte);
   290    311   char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
   291    312   
   292    313   /* Character set tests (like isspace(), isalpha() etc.) */
   293    314   int sqlite3Fts5IsBareword(char t);
   294    315   
          316  +
          317  +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
          318  +typedef struct Fts5Termset Fts5Termset;
          319  +int sqlite3Fts5TermsetNew(Fts5Termset**);
          320  +int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
          321  +void sqlite3Fts5TermsetFree(Fts5Termset*);
          322  +
   295    323   /*
   296    324   ** End of interface to code in fts5_buffer.c.
   297    325   **************************************************************************/
   298    326   
   299    327   /**************************************************************************
   300    328   ** Interface to code in fts5_index.c. fts5_index.c contains contains code
   301    329   ** to access the data stored in the %_data table.
   302    330   */
   303    331   
   304    332   typedef struct Fts5Index Fts5Index;
   305    333   typedef struct Fts5IndexIter Fts5IndexIter;
   306    334   
          335  +struct Fts5IndexIter {
          336  +  i64 iRowid;
          337  +  const u8 *pData;
          338  +  int nData;
          339  +  u8 bEof;
          340  +};
          341  +
          342  +#define sqlite3Fts5IterEof(x) ((x)->bEof)
          343  +
   307    344   /*
   308    345   ** Values used as part of the flags argument passed to IndexQuery().
   309    346   */
   310    347   #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
   311    348   #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
   312    349   #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
   313    350   #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
          351  +
          352  +/* The following are used internally by the fts5_index.c module. They are
          353  +** defined here only to make it easier to avoid clashes with the flags
          354  +** above. */
          355  +#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
          356  +#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
   314    357   
   315    358   /*
   316    359   ** Create/destroy an Fts5Index object.
   317    360   */
   318    361   int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
   319    362   int sqlite3Fts5IndexClose(Fts5Index *p);
   320    363   
   321    364   /*
   322         -** for(
   323         -**   sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
   324         -**   0==sqlite3Fts5IterEof(pIter);
   325         -**   sqlite3Fts5IterNext(pIter)
   326         -** ){
   327         -**   i64 iRowid = sqlite3Fts5IterRowid(pIter);
   328         -** }
          365  +** Return a simple checksum value based on the arguments.
   329    366   */
          367  +u64 sqlite3Fts5IndexEntryCksum(
          368  +  i64 iRowid, 
          369  +  int iCol, 
          370  +  int iPos, 
          371  +  int iIdx,
          372  +  const char *pTerm,
          373  +  int nTerm
          374  +);
          375  +
          376  +/*
          377  +** Argument p points to a buffer containing utf-8 text that is n bytes in 
          378  +** size. Return the number of bytes in the nChar character prefix of the
          379  +** buffer, or 0 if there are less than nChar characters in total.
          380  +*/
          381  +int sqlite3Fts5IndexCharlenToBytelen(
          382  +  const char *p, 
          383  +  int nByte, 
          384  +  int nChar
          385  +);
   330    386   
   331    387   /*
   332    388   ** Open a new iterator to iterate though all rowids that match the 
   333    389   ** specified token or token prefix.
   334    390   */
   335    391   int sqlite3Fts5IndexQuery(
   336    392     Fts5Index *p,                   /* FTS index to query */
................................................................................
   340    396     Fts5IndexIter **ppIter          /* OUT: New iterator object */
   341    397   );
   342    398   
   343    399   /*
   344    400   ** The various operations on open token or token prefix iterators opened
   345    401   ** using sqlite3Fts5IndexQuery().
   346    402   */
   347         -int sqlite3Fts5IterEof(Fts5IndexIter*);
   348    403   int sqlite3Fts5IterNext(Fts5IndexIter*);
   349    404   int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
   350         -i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
   351         -int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*);
   352         -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf);
   353    405   
   354    406   /*
   355    407   ** Close an iterator opened by sqlite3Fts5IndexQuery().
   356    408   */
   357    409   void sqlite3Fts5IterClose(Fts5IndexIter*);
   358    410   
   359    411   /*
................................................................................
   409    461   */
   410    462   int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
   411    463   int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
   412    464   
   413    465   /*
   414    466   ** Functions called by the storage module as part of integrity-check.
   415    467   */
   416         -u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int);
   417    468   int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
   418    469   
   419    470   /* 
   420    471   ** Called during virtual module initialization to register UDF 
   421    472   ** fts5_decode() with SQLite 
   422    473   */
   423    474   int sqlite3Fts5IndexInit(sqlite3*);
................................................................................
   429    480   ** this connection since it was created.
   430    481   */
   431    482   int sqlite3Fts5IndexReads(Fts5Index *p);
   432    483   
   433    484   int sqlite3Fts5IndexReinit(Fts5Index *p);
   434    485   int sqlite3Fts5IndexOptimize(Fts5Index *p);
   435    486   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
          487  +int sqlite3Fts5IndexReset(Fts5Index *p);
   436    488   
   437    489   int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
   438    490   
   439    491   /*
   440    492   ** End of interface to code in fts5_index.c.
   441    493   **************************************************************************/
   442    494   
................................................................................
   488    540   ** Interface to code in fts5_hash.c. 
   489    541   */
   490    542   typedef struct Fts5Hash Fts5Hash;
   491    543   
   492    544   /*
   493    545   ** Create a hash table, free a hash table.
   494    546   */
   495         -int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize);
          547  +int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
   496    548   void sqlite3Fts5HashFree(Fts5Hash*);
   497    549   
   498    550   int sqlite3Fts5HashWrite(
   499    551     Fts5Hash*,
   500    552     i64 iRowid,                     /* Rowid for this entry */
   501    553     int iCol,                       /* Column token appears in (-ve -> delete) */
   502    554     int iPos,                       /* Position of token within column */
................................................................................
   547    599   int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
   548    600   int sqlite3Fts5StorageClose(Fts5Storage *p);
   549    601   int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
   550    602   
   551    603   int sqlite3Fts5DropAll(Fts5Config*);
   552    604   int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
   553    605   
   554         -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
          606  +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
   555    607   int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
   556    608   int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
   557    609   
   558    610   int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
   559    611   
   560    612   int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
   561    613   void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
................................................................................
   567    619   int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
   568    620   int sqlite3Fts5StorageRollback(Fts5Storage *p);
   569    621   
   570    622   int sqlite3Fts5StorageConfigValue(
   571    623       Fts5Storage *p, const char*, sqlite3_value*, int
   572    624   );
   573    625   
   574         -int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
   575         -
   576    626   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   577    627   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
   578    628   int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   579    629   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
          630  +int sqlite3Fts5StorageReset(Fts5Storage *p);
   580    631   
   581    632   /*
   582    633   ** End of interface to code in fts5_storage.c.
   583    634   **************************************************************************/
   584    635   
   585    636   
   586    637   /**************************************************************************
................................................................................
   625    676   /* Called during startup to register a UDF with SQLite */
   626    677   int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
   627    678   
   628    679   int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
   629    680   int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
   630    681   int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
   631    682   
   632         -int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
          683  +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
          684  +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
          685  +int sqlite3Fts5ExprPopulatePoslists(
          686  +    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
          687  +);
          688  +void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
          689  +void sqlite3Fts5ExprClearEof(Fts5Expr*);
          690  +
          691  +int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
          692  +
          693  +int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
   633    694   
   634    695   /*******************************************
   635    696   ** The fts5_expr.c API above this point is used by the other hand-written
   636    697   ** C code in this module. The interfaces below this point are called by
   637    698   ** the parser code in fts5parse.y.  */
   638    699   
   639    700   void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
................................................................................
   641    702   Fts5ExprNode *sqlite3Fts5ParseNode(
   642    703     Fts5Parse *pParse,
   643    704     int eType,
   644    705     Fts5ExprNode *pLeft,
   645    706     Fts5ExprNode *pRight,
   646    707     Fts5ExprNearset *pNear
   647    708   );
          709  +
          710  +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
          711  +  Fts5Parse *pParse,
          712  +  Fts5ExprNode *pLeft,
          713  +  Fts5ExprNode *pRight
          714  +);
   648    715   
   649    716   Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   650    717     Fts5Parse *pParse, 
   651    718     Fts5ExprPhrase *pPhrase, 
   652    719     Fts5Token *pToken,
   653    720     int bPrefix
   654    721   );

Changes to ext/fts5/fts5_aux.c.

   153    153     int nToken,                     /* Size of token in bytes */
   154    154     int iStartOff,                  /* Start offset of token */
   155    155     int iEndOff                     /* End offset of token */
   156    156   ){
   157    157     HighlightContext *p = (HighlightContext*)pContext;
   158    158     int rc = SQLITE_OK;
   159    159     int iPos;
          160  +
          161  +  UNUSED_PARAM2(pToken, nToken);
   160    162   
   161    163     if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
   162    164     iPos = p->iPos++;
   163    165   
   164    166     if( p->iRangeEnd>0 ){
   165    167       if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
   166    168       if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
................................................................................
   387    389   */
   388    390   static int fts5CountCb(
   389    391     const Fts5ExtensionApi *pApi, 
   390    392     Fts5Context *pFts,
   391    393     void *pUserData                 /* Pointer to sqlite3_int64 variable */
   392    394   ){
   393    395     sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
          396  +  UNUSED_PARAM2(pApi, pFts);
   394    397     (*pn)++;
   395    398     return SQLITE_OK;
   396    399   }
   397    400   
   398    401   /*
   399    402   ** Set *ppData to point to the Fts5Bm25Data object for the current query. 
   400    403   ** If the object has not already been allocated, allocate and populate it
................................................................................
   540    543       { "snippet",   0, fts5SnippetFunction, 0 },
   541    544       { "highlight", 0, fts5HighlightFunction, 0 },
   542    545       { "bm25",      0, fts5Bm25Function,    0 },
   543    546     };
   544    547     int rc = SQLITE_OK;             /* Return code */
   545    548     int i;                          /* To iterate through builtin functions */
   546    549   
   547         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
          550  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
   548    551       rc = pApi->xCreateFunction(pApi,
   549    552           aBuiltin[i].zFunc,
   550    553           aBuiltin[i].pUserData,
   551    554           aBuiltin[i].xFunc,
   552    555           aBuiltin[i].xDestroy
   553    556       );
   554    557     }
   555    558   
   556    559     return rc;
   557    560   }
   558    561   
   559    562   

Changes to ext/fts5/fts5_buffer.c.

    11     11   ******************************************************************************
    12     12   */
    13     13   
    14     14   
    15     15   
    16     16   #include "fts5Int.h"
    17     17   
    18         -int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){
    19         -  int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
    20         -  u8 *pNew;
    21         -  while( nNew<nByte ){
    22         -    nNew = nNew * 2;
    23         -  }
    24         -  pNew = sqlite3_realloc(pBuf->p, nNew);
    25         -  if( pNew==0 ){
    26         -    *pRc = SQLITE_NOMEM;
    27         -    return 1;
    28         -  }else{
    29         -    pBuf->nSpace = nNew;
    30         -    pBuf->p = pNew;
           18  +int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
           19  +  if( (u32)pBuf->nSpace<nByte ){
           20  +    u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
           21  +    u8 *pNew;
           22  +    while( nNew<nByte ){
           23  +      nNew = nNew * 2;
           24  +    }
           25  +    pNew = sqlite3_realloc(pBuf->p, nNew);
           26  +    if( pNew==0 ){
           27  +      *pRc = SQLITE_NOMEM;
           28  +      return 1;
           29  +    }else{
           30  +      pBuf->nSpace = nNew;
           31  +      pBuf->p = pNew;
           32  +    }
    31     33     }
    32     34     return 0;
    33     35   }
    34     36   
    35     37   
    36     38   /*
    37     39   ** Encode value iVal as an SQLite varint and append it to the buffer object
................................................................................
    57     59   ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set 
    58     60   ** the error code in p. If an error has already occurred when this function
    59     61   ** is called, it is a no-op.
    60     62   */
    61     63   void sqlite3Fts5BufferAppendBlob(
    62     64     int *pRc,
    63     65     Fts5Buffer *pBuf, 
    64         -  int nData, 
           66  +  u32 nData, 
    65     67     const u8 *pData
    66     68   ){
    67         -  assert( *pRc || nData>=0 );
           69  +  assert_nc( *pRc || nData>=0 );
    68     70     if( fts5BufferGrow(pRc, pBuf, nData) ) return;
    69     71     memcpy(&pBuf->p[pBuf->n], pData, nData);
    70     72     pBuf->n += nData;
    71     73   }
    72     74   
    73     75   /*
    74     76   ** Append the nul-terminated string zStr to the buffer pBuf. This function
................................................................................
   203    205   ){
   204    206     memset(pIter, 0, sizeof(*pIter));
   205    207     pIter->a = a;
   206    208     pIter->n = n;
   207    209     sqlite3Fts5PoslistReaderNext(pIter);
   208    210     return pIter->bEof;
   209    211   }
          212  +
          213  +/*
          214  +** Append position iPos to the position list being accumulated in buffer
          215  +** pBuf, which must be already be large enough to hold the new data.
          216  +** The previous position written to this list is *piPrev. *piPrev is set
          217  +** to iPos before returning.
          218  +*/
          219  +void sqlite3Fts5PoslistSafeAppend(
          220  +  Fts5Buffer *pBuf, 
          221  +  i64 *piPrev, 
          222  +  i64 iPos
          223  +){
          224  +  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
          225  +  if( (iPos & colmask) != (*piPrev & colmask) ){
          226  +    pBuf->p[pBuf->n++] = 1;
          227  +    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
          228  +    *piPrev = (iPos & colmask);
          229  +  }
          230  +  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
          231  +  *piPrev = iPos;
          232  +}
   210    233   
   211    234   int sqlite3Fts5PoslistWriterAppend(
   212    235     Fts5Buffer *pBuf, 
   213    236     Fts5PoslistWriter *pWriter,
   214    237     i64 iPos
   215    238   ){
   216         -  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
   217         -  int rc = SQLITE_OK;
   218         -  if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
   219         -    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
   220         -      pBuf->p[pBuf->n++] = 1;
   221         -      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
   222         -      pWriter->iPrev = (iPos & colmask);
   223         -    }
   224         -    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
   225         -    pWriter->iPrev = iPos;
   226         -  }
   227         -  return rc;
          239  +  int rc = 0;   /* Initialized only to suppress erroneous warning from Clang */
          240  +  if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
          241  +  sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
          242  +  return SQLITE_OK;
   228    243   }
   229    244   
   230    245   void *sqlite3Fts5MallocZero(int *pRc, int nByte){
   231    246     void *pRet = 0;
   232    247     if( *pRc==SQLITE_OK ){
   233    248       pRet = sqlite3_malloc(nByte);
   234    249       if( pRet==0 && nByte>0 ){
................................................................................
   288    303       1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
   289    304     };
   290    305   
   291    306     return (t & 0x80) || aBareword[(int)t];
   292    307   }
   293    308   
   294    309   
          310  +/*************************************************************************
          311  +*/
          312  +typedef struct Fts5TermsetEntry Fts5TermsetEntry;
          313  +struct Fts5TermsetEntry {
          314  +  char *pTerm;
          315  +  int nTerm;
          316  +  int iIdx;                       /* Index (main or aPrefix[] entry) */
          317  +  Fts5TermsetEntry *pNext;
          318  +};
          319  +
          320  +struct Fts5Termset {
          321  +  Fts5TermsetEntry *apHash[512];
          322  +};
          323  +
          324  +int sqlite3Fts5TermsetNew(Fts5Termset **pp){
          325  +  int rc = SQLITE_OK;
          326  +  *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
          327  +  return rc;
          328  +}
          329  +
          330  +int sqlite3Fts5TermsetAdd(
          331  +  Fts5Termset *p, 
          332  +  int iIdx,
          333  +  const char *pTerm, int nTerm, 
          334  +  int *pbPresent
          335  +){
          336  +  int rc = SQLITE_OK;
          337  +  *pbPresent = 0;
          338  +  if( p ){
          339  +    int i;
          340  +    u32 hash = 13;
          341  +    Fts5TermsetEntry *pEntry;
          342  +
          343  +    /* Calculate a hash value for this term. This is the same hash checksum
          344  +    ** used by the fts5_hash.c module. This is not important for correct
          345  +    ** operation of the module, but is necessary to ensure that some tests
          346  +    ** designed to produce hash table collisions really do work.  */
          347  +    for(i=nTerm-1; i>=0; i--){
          348  +      hash = (hash << 3) ^ hash ^ pTerm[i];
          349  +    }
          350  +    hash = (hash << 3) ^ hash ^ iIdx;
          351  +    hash = hash % ArraySize(p->apHash);
          352  +
          353  +    for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
          354  +      if( pEntry->iIdx==iIdx 
          355  +          && pEntry->nTerm==nTerm 
          356  +          && memcmp(pEntry->pTerm, pTerm, nTerm)==0 
          357  +      ){
          358  +        *pbPresent = 1;
          359  +        break;
          360  +      }
          361  +    }
          362  +
          363  +    if( pEntry==0 ){
          364  +      pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
          365  +      if( pEntry ){
          366  +        pEntry->pTerm = (char*)&pEntry[1];
          367  +        pEntry->nTerm = nTerm;
          368  +        pEntry->iIdx = iIdx;
          369  +        memcpy(pEntry->pTerm, pTerm, nTerm);
          370  +        pEntry->pNext = p->apHash[hash];
          371  +        p->apHash[hash] = pEntry;
          372  +      }
          373  +    }
          374  +  }
          375  +
          376  +  return rc;
          377  +}
          378  +
          379  +void sqlite3Fts5TermsetFree(Fts5Termset *p){
          380  +  if( p ){
          381  +    u32 i;
          382  +    for(i=0; i<ArraySize(p->apHash); i++){
          383  +      Fts5TermsetEntry *pEntry = p->apHash[i];
          384  +      while( pEntry ){
          385  +        Fts5TermsetEntry *pDel = pEntry;
          386  +        pEntry = pEntry->pNext;
          387  +        sqlite3_free(pDel);
          388  +      }
          389  +    }
          390  +    sqlite3_free(p);
          391  +  }
          392  +}

Changes to ext/fts5/fts5_config.c.

    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   ** This is an SQLite module implementing full-text search.
    14     14   */
    15     15   
    16     16   
    17         -
    18     17   #include "fts5Int.h"
    19     18   
    20     19   #define FTS5_DEFAULT_PAGE_SIZE   4050
    21     20   #define FTS5_DEFAULT_AUTOMERGE      4
           21  +#define FTS5_DEFAULT_USERMERGE      4
    22     22   #define FTS5_DEFAULT_CRISISMERGE   16
    23     23   #define FTS5_DEFAULT_HASHSIZE    (1024*1024)
    24     24   
    25     25   /* Maximum allowed page size */
    26     26   #define FTS5_MAX_PAGE_SIZE (128*1024)
    27     27   
    28     28   static int fts5_iswhitespace(char x){
................................................................................
   191    191     assert( 0==fts5_iswhitespace(z[0]) );
   192    192     quote = z[0];
   193    193     if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
   194    194       fts5Dequote(z);
   195    195     }
   196    196   }
   197    197   
          198  +
          199  +struct Fts5Enum {
          200  +  const char *zName;
          201  +  int eVal;
          202  +};
          203  +typedef struct Fts5Enum Fts5Enum;
          204  +
          205  +static int fts5ConfigSetEnum(
          206  +  const Fts5Enum *aEnum, 
          207  +  const char *zEnum, 
          208  +  int *peVal
          209  +){
          210  +  int nEnum = (int)strlen(zEnum);
          211  +  int i;
          212  +  int iVal = -1;
          213  +
          214  +  for(i=0; aEnum[i].zName; i++){
          215  +    if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
          216  +      if( iVal>=0 ) return SQLITE_ERROR;
          217  +      iVal = aEnum[i].eVal;
          218  +    }
          219  +  }
          220  +
          221  +  *peVal = iVal;
          222  +  return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
          223  +}
          224  +
   198    225   /*
   199    226   ** Parse a "special" CREATE VIRTUAL TABLE directive and update
   200    227   ** configuration object pConfig as appropriate.
   201    228   **
   202    229   ** If successful, object pConfig is updated and SQLITE_OK returned. If
   203    230   ** an error occurs, an SQLite error code is returned and an error message
   204    231   ** may be left in *pzErr. It is the responsibility of the caller to
................................................................................
   248    275         }
   249    276   
   250    277         while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
   251    278           nPre = nPre*10 + (p[0] - '0');
   252    279           p++;
   253    280         }
   254    281   
   255         -      if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
          282  +      if( nPre<=0 || nPre>=1000 ){
   256    283           *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
   257    284           rc = SQLITE_ERROR;
   258    285           break;
   259    286         }
   260    287   
   261    288         pConfig->aPrefix[pConfig->nPrefix] = nPre;
   262    289         pConfig->nPrefix++;
................................................................................
   340    367         *pzErr = sqlite3_mprintf("malformed columnsize=... directive");
   341    368         rc = SQLITE_ERROR;
   342    369       }else{
   343    370         pConfig->bColumnsize = (zArg[0]=='1');
   344    371       }
   345    372       return rc;
   346    373     }
          374  +
          375  +  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
          376  +    const Fts5Enum aDetail[] = {
          377  +      { "none", FTS5_DETAIL_NONE },
          378  +      { "full", FTS5_DETAIL_FULL },
          379  +      { "columns", FTS5_DETAIL_COLUMNS },
          380  +      { 0, 0 }
          381  +    };
          382  +
          383  +    if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
          384  +      *pzErr = sqlite3_mprintf("malformed detail=... directive");
          385  +    }
          386  +    return rc;
          387  +  }
   347    388   
   348    389     *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
   349    390     return SQLITE_ERROR;
   350    391   }
   351    392   
   352    393   /*
   353    394   ** Allocate an instance of the default tokenizer ("simple") at 
................................................................................
   397    438       memcpy(zOut, zIn, nIn+1);
   398    439       if( fts5_isopenquote(zOut[0]) ){
   399    440         int ii = fts5Dequote(zOut);
   400    441         zRet = &zIn[ii];
   401    442         *pbQuoted = 1;
   402    443       }else{
   403    444         zRet = fts5ConfigSkipBareword(zIn);
   404         -      zOut[zRet-zIn] = '\0';
          445  +      if( zRet ){
          446  +        zOut[zRet-zIn] = '\0';
          447  +      }
   405    448       }
   406    449     }
   407    450   
   408    451     if( zRet==0 ){
   409    452       sqlite3_free(zOut);
   410    453     }else{
   411    454       *pzOut = zOut;
................................................................................
   496    539   
   497    540     nByte = nArg * (sizeof(char*) + sizeof(u8));
   498    541     pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
   499    542     pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
   500    543     pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
   501    544     pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
   502    545     pRet->bColumnsize = 1;
          546  +  pRet->eDetail = FTS5_DETAIL_FULL;
   503    547   #ifdef SQLITE_DEBUG
   504    548     pRet->bPrefixIndex = 1;
   505    549   #endif
   506    550     if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
   507    551       *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
   508    552       rc = SQLITE_ERROR;
   509    553     }
................................................................................
   811    855       if( nAutomerge<0 || nAutomerge>64 ){
   812    856         *pbBadkey = 1;
   813    857       }else{
   814    858         if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
   815    859         pConfig->nAutomerge = nAutomerge;
   816    860       }
   817    861     }
          862  +
          863  +  else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
          864  +    int nUsermerge = -1;
          865  +    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
          866  +      nUsermerge = sqlite3_value_int(pVal);
          867  +    }
          868  +    if( nUsermerge<2 || nUsermerge>16 ){
          869  +      *pbBadkey = 1;
          870  +    }else{
          871  +      pConfig->nUsermerge = nUsermerge;
          872  +    }
          873  +  }
   818    874   
   819    875     else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
   820    876       int nCrisisMerge = -1;
   821    877       if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
   822    878         nCrisisMerge = sqlite3_value_int(pVal);
   823    879       }
   824    880       if( nCrisisMerge<0 ){
................................................................................
   858    914     sqlite3_stmt *p = 0;
   859    915     int rc = SQLITE_OK;
   860    916     int iVersion = 0;
   861    917   
   862    918     /* Set default values */
   863    919     pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
   864    920     pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
          921  +  pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   865    922     pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   866    923     pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
   867    924   
   868    925     zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   869    926     if( zSql ){
   870    927       rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
   871    928       sqlite3_free(zSql);
................................................................................
   898    955     }
   899    956   
   900    957     if( rc==SQLITE_OK ){
   901    958       pConfig->iCookie = iCookie;
   902    959     }
   903    960     return rc;
   904    961   }
   905         -

Changes to ext/fts5/fts5_expr.c.

    36     36   #include <stdio.h>
    37     37   void sqlite3Fts5ParserTrace(FILE*, char*);
    38     38   #endif
    39     39   
    40     40   
    41     41   struct Fts5Expr {
    42     42     Fts5Index *pIndex;
           43  +  Fts5Config *pConfig;
    43     44     Fts5ExprNode *pRoot;
    44     45     int bDesc;                      /* Iterate in descending rowid order */
    45     46     int nPhrase;                    /* Number of phrases in expression */
    46     47     Fts5ExprPhrase **apExprPhrase;  /* Pointers to phrase objects */
    47     48   };
    48     49   
    49     50   /*
................................................................................
    57     58   **       FTS5_TERM                (pNear valid)
    58     59   */
    59     60   struct Fts5ExprNode {
    60     61     int eType;                      /* Node type */
    61     62     int bEof;                       /* True at EOF */
    62     63     int bNomatch;                   /* True if entry is not a match */
    63     64   
           65  +  /* Next method for this node. */
           66  +  int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
           67  +
    64     68     i64 iRowid;                     /* Current rowid */
    65     69     Fts5ExprNearset *pNear;         /* For FTS5_STRING - cluster of phrases */
    66     70   
    67     71     /* Child nodes. For a NOT node, this array always contains 2 entries. For 
    68     72     ** AND or OR nodes, it contains 2 or more entries.  */
    69     73     int nChild;                     /* Number of child nodes */
    70     74     Fts5ExprNode *apChild[1];       /* Array of child nodes */
    71     75   };
    72     76   
    73     77   #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING)
    74     78   
           79  +/*
           80  +** Invoke the xNext method of an Fts5ExprNode object. This macro should be
           81  +** used as if it has the same signature as the xNext() methods themselves.
           82  +*/
           83  +#define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d))
           84  +
    75     85   /*
    76     86   ** An instance of the following structure represents a single search term
    77     87   ** or term prefix.
    78     88   */
    79     89   struct Fts5ExprTerm {
    80     90     int bPrefix;                    /* True for a prefix term */
    81     91     char *zTerm;                    /* nul-terminated term */
................................................................................
   229    239     assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
   230    240     if( sParse.rc==SQLITE_OK ){
   231    241       *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
   232    242       if( pNew==0 ){
   233    243         sParse.rc = SQLITE_NOMEM;
   234    244         sqlite3Fts5ParseNodeFree(sParse.pExpr);
   235    245       }else{
   236         -      pNew->pRoot = sParse.pExpr;
          246  +      if( !sParse.pExpr ){
          247  +        const int nByte = sizeof(Fts5ExprNode);
          248  +        pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
          249  +        if( pNew->pRoot ){
          250  +          pNew->pRoot->bEof = 1;
          251  +        }
          252  +      }else{
          253  +        pNew->pRoot = sParse.pExpr;
          254  +      }
   237    255         pNew->pIndex = 0;
          256  +      pNew->pConfig = pConfig;
   238    257         pNew->apExprPhrase = sParse.apPhrase;
   239    258         pNew->nPhrase = sParse.nPhrase;
   240    259         sParse.apPhrase = 0;
   241    260       }
          261  +  }else{
          262  +    sqlite3Fts5ParseNodeFree(sParse.pExpr);
   242    263     }
   243    264   
   244    265     sqlite3_free(sParse.apPhrase);
   245    266     *pzErr = sParse.zErr;
   246    267     return sParse.rc;
   247    268   }
   248    269   
................................................................................
   280    301     int bRetValid = 0;
   281    302     Fts5ExprTerm *p;
   282    303   
   283    304     assert( pTerm->pSynonym );
   284    305     assert( bDesc==0 || bDesc==1 );
   285    306     for(p=pTerm; p; p=p->pSynonym){
   286    307       if( 0==sqlite3Fts5IterEof(p->pIter) ){
   287         -      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
          308  +      i64 iRowid = p->pIter->iRowid;
   288    309         if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){
   289    310           iRet = iRowid;
   290    311           bRetValid = 1;
   291    312         }
   292    313       }
   293    314     }
   294    315   
................................................................................
   295    316     if( pbEof && bRetValid==0 ) *pbEof = 1;
   296    317     return iRet;
   297    318   }
   298    319   
   299    320   /*
   300    321   ** Argument pTerm must be a synonym iterator.
   301    322   */
   302         -static int fts5ExprSynonymPoslist(
          323  +static int fts5ExprSynonymList(
   303    324     Fts5ExprTerm *pTerm, 
   304         -  Fts5Colset *pColset,
   305    325     i64 iRowid,
   306         -  int *pbDel,                     /* OUT: Caller should sqlite3_free(*pa) */
          326  +  Fts5Buffer *pBuf,               /* Use this buffer for space if required */
   307    327     u8 **pa, int *pn
   308    328   ){
   309    329     Fts5PoslistReader aStatic[4];
   310    330     Fts5PoslistReader *aIter = aStatic;
   311    331     int nIter = 0;
   312    332     int nAlloc = 4;
   313    333     int rc = SQLITE_OK;
   314    334     Fts5ExprTerm *p;
   315    335   
   316    336     assert( pTerm->pSynonym );
   317    337     for(p=pTerm; p; p=p->pSynonym){
   318    338       Fts5IndexIter *pIter = p->pIter;
   319         -    if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
   320         -      const u8 *a;
   321         -      int n;
   322         -      i64 dummy;
   323         -      rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
   324         -      if( rc!=SQLITE_OK ) goto synonym_poslist_out;
          339  +    if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){
          340  +      if( pIter->nData==0 ) continue;
   325    341         if( nIter==nAlloc ){
   326    342           int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
   327    343           Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   328    344           if( aNew==0 ){
   329    345             rc = SQLITE_NOMEM;
   330    346             goto synonym_poslist_out;
   331    347           }
   332    348           memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter);
   333    349           nAlloc = nAlloc*2;
   334    350           if( aIter!=aStatic ) sqlite3_free(aIter);
   335    351           aIter = aNew;
   336    352         }
   337         -      sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]);
          353  +      sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]);
   338    354         assert( aIter[nIter].bEof==0 );
   339    355         nIter++;
   340    356       }
   341    357     }
   342    358   
   343         -  assert( *pbDel==0 );
   344    359     if( nIter==1 ){
   345    360       *pa = (u8*)aIter[0].a;
   346    361       *pn = aIter[0].n;
   347    362     }else{
   348    363       Fts5PoslistWriter writer = {0};
   349         -    Fts5Buffer buf = {0,0,0};
   350    364       i64 iPrev = -1;
          365  +    fts5BufferZero(pBuf);
   351    366       while( 1 ){
   352    367         int i;
   353    368         i64 iMin = FTS5_LARGEST_INT64;
   354    369         for(i=0; i<nIter; i++){
   355    370           if( aIter[i].bEof==0 ){
   356    371             if( aIter[i].iPos==iPrev ){
   357    372               if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue;
................................................................................
   358    373             }
   359    374             if( aIter[i].iPos<iMin ){
   360    375               iMin = aIter[i].iPos;
   361    376             }
   362    377           }
   363    378         }
   364    379         if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break;
   365         -      rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin);
          380  +      rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin);
   366    381         iPrev = iMin;
   367    382       }
   368         -    if( rc ){
   369         -      sqlite3_free(buf.p);
   370         -    }else{
   371         -      *pa = buf.p;
   372         -      *pn = buf.n;
   373         -      *pbDel = 1;
          383  +    if( rc==SQLITE_OK ){
          384  +      *pa = pBuf->p;
          385  +      *pn = pBuf->n;
   374    386       }
   375    387     }
   376    388   
   377    389    synonym_poslist_out:
   378    390     if( aIter!=aStatic ) sqlite3_free(aIter);
   379    391     return rc;
   380    392   }
................................................................................
   389    401   **
   390    402   ** SQLITE_OK is returned if an error occurs, or an SQLite error code 
   391    403   ** otherwise. It is not considered an error code if the current rowid is 
   392    404   ** not a match.
   393    405   */
   394    406   static int fts5ExprPhraseIsMatch(
   395    407     Fts5ExprNode *pNode,            /* Node pPhrase belongs to */
   396         -  Fts5Colset *pColset,            /* Restrict matches to these columns */
   397    408     Fts5ExprPhrase *pPhrase,        /* Phrase object to initialize */
   398    409     int *pbMatch                    /* OUT: Set to true if really a match */
   399    410   ){
   400    411     Fts5PoslistWriter writer = {0};
   401    412     Fts5PoslistReader aStatic[4];
   402    413     Fts5PoslistReader *aIter = aStatic;
   403    414     int i;
   404    415     int rc = SQLITE_OK;
   405    416     
   406    417     fts5BufferZero(&pPhrase->poslist);
   407    418   
   408    419     /* If the aStatic[] array is not large enough, allocate a large array
   409    420     ** using sqlite3_malloc(). This approach could be improved upon. */
   410         -  if( pPhrase->nTerm>(int)ArraySize(aStatic) ){
          421  +  if( pPhrase->nTerm>ArraySize(aStatic) ){
   411    422       int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
   412    423       aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   413    424       if( !aIter ) return SQLITE_NOMEM;
   414    425     }
   415    426     memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
   416    427   
   417    428     /* Initialize a term iterator for each term in the phrase */
   418    429     for(i=0; i<pPhrase->nTerm; i++){
   419    430       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
   420         -    i64 dummy;
   421    431       int n = 0;
   422    432       int bFlag = 0;
   423         -    const u8 *a = 0;
          433  +    u8 *a = 0;
   424    434       if( pTerm->pSynonym ){
   425         -      rc = fts5ExprSynonymPoslist(
   426         -          pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
   427         -      );
          435  +      Fts5Buffer buf = {0, 0, 0};
          436  +      rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n);
          437  +      if( rc ){
          438  +        sqlite3_free(a);
          439  +        goto ismatch_out;
          440  +      }
          441  +      if( a==buf.p ) bFlag = 1;
   428    442       }else{
   429         -      rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
          443  +      a = (u8*)pTerm->pIter->pData;
          444  +      n = pTerm->pIter->nData;
   430    445       }
   431         -    if( rc!=SQLITE_OK ) goto ismatch_out;
   432    446       sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
   433    447       aIter[i].bFlag = (u8)bFlag;
   434    448       if( aIter[i].bEof ) goto ismatch_out;
   435    449     }
   436    450   
   437    451     while( 1 ){
   438    452       int bMatch;
................................................................................
   496    510     memset(p, 0, sizeof(Fts5LookaheadReader));
   497    511     p->a = a;
   498    512     p->n = n;
   499    513     fts5LookaheadReaderNext(p);
   500    514     return fts5LookaheadReaderNext(p);
   501    515   }
   502    516   
   503         -#if 0
   504         -static int fts5LookaheadReaderEof(Fts5LookaheadReader *p){
   505         -  return (p->iPos==FTS5_LOOKAHEAD_EOF);
   506         -}
   507         -#endif
   508         -
   509    517   typedef struct Fts5NearTrimmer Fts5NearTrimmer;
   510    518   struct Fts5NearTrimmer {
   511    519     Fts5LookaheadReader reader;     /* Input iterator */
   512    520     Fts5PoslistWriter writer;       /* Writer context */
   513    521     Fts5Buffer *pOut;               /* Output poslist */
   514    522   };
   515    523   
................................................................................
   539    547     int rc = *pRc;
   540    548     int bMatch;
   541    549   
   542    550     assert( pNear->nPhrase>1 );
   543    551   
   544    552     /* If the aStatic[] array is not large enough, allocate a large array
   545    553     ** using sqlite3_malloc(). This approach could be improved upon. */
   546         -  if( pNear->nPhrase>(int)ArraySize(aStatic) ){
          554  +  if( pNear->nPhrase>ArraySize(aStatic) ){
   547    555       int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
   548    556       a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
   549    557     }else{
   550    558       memset(aStatic, 0, sizeof(aStatic));
   551    559     }
   552    560     if( rc!=SQLITE_OK ){
   553    561       *pRc = rc;
................................................................................
   616    624       int bRet = a[0].pOut->n>0;
   617    625       *pRc = rc;
   618    626       if( a!=aStatic ) sqlite3_free(a);
   619    627       return bRet;
   620    628     }
   621    629   }
   622    630   
   623         -/*
   624         -** Advance the first term iterator in the first phrase of pNear. Set output
   625         -** variable *pbEof to true if it reaches EOF or if an error occurs.
   626         -**
   627         -** Return SQLITE_OK if successful, or an SQLite error code if an error
   628         -** occurs.
   629         -*/
   630         -static int fts5ExprNearAdvanceFirst(
   631         -  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
   632         -  Fts5ExprNode *pNode,            /* FTS5_STRING or FTS5_TERM node */
   633         -  int bFromValid,
   634         -  i64 iFrom 
   635         -){
   636         -  Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
   637         -  int rc = SQLITE_OK;
   638         -
   639         -  if( pTerm->pSynonym ){
   640         -    int bEof = 1;
   641         -    Fts5ExprTerm *p;
   642         -
   643         -    /* Find the firstest rowid any synonym points to. */
   644         -    i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
   645         -
   646         -    /* Advance each iterator that currently points to iRowid. Or, if iFrom
   647         -    ** is valid - each iterator that points to a rowid before iFrom.  */
   648         -    for(p=pTerm; p; p=p->pSynonym){
   649         -      if( sqlite3Fts5IterEof(p->pIter)==0 ){
   650         -        i64 ii = sqlite3Fts5IterRowid(p->pIter);
   651         -        if( ii==iRowid 
   652         -         || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) 
   653         -        ){
   654         -          if( bFromValid ){
   655         -            rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
   656         -          }else{
   657         -            rc = sqlite3Fts5IterNext(p->pIter);
   658         -          }
   659         -          if( rc!=SQLITE_OK ) break;
   660         -          if( sqlite3Fts5IterEof(p->pIter)==0 ){
   661         -            bEof = 0;
   662         -          }
   663         -        }else{
   664         -          bEof = 0;
   665         -        }
   666         -      }
   667         -    }
   668         -
   669         -    /* Set the EOF flag if either all synonym iterators are at EOF or an
   670         -    ** error has occurred.  */
   671         -    pNode->bEof = (rc || bEof);
   672         -  }else{
   673         -    Fts5IndexIter *pIter = pTerm->pIter;
   674         -
   675         -    assert( Fts5NodeIsString(pNode) );
   676         -    if( bFromValid ){
   677         -      rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
   678         -    }else{
   679         -      rc = sqlite3Fts5IterNext(pIter);
   680         -    }
   681         -
   682         -    pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
   683         -  }
   684         -
   685         -  return rc;
   686         -}
   687         -
   688    631   /*
   689    632   ** Advance iterator pIter until it points to a value equal to or laster
   690    633   ** than the initial value of *piLast. If this means the iterator points
   691    634   ** to a value laster than *piLast, update *piLast to the new lastest value.
   692    635   **
   693    636   ** If the iterator reaches EOF, set *pbEof to true before returning. If
   694    637   ** an error occurs, set *pRc to an error code. If either *pbEof or *pRc
................................................................................
   700    643     i64 *piLast,                    /* IN/OUT: Lastest rowid seen so far */
   701    644     int *pRc,                       /* OUT: Error code */
   702    645     int *pbEof                      /* OUT: Set to true if EOF */
   703    646   ){
   704    647     i64 iLast = *piLast;
   705    648     i64 iRowid;
   706    649   
   707         -  iRowid = sqlite3Fts5IterRowid(pIter);
          650  +  iRowid = pIter->iRowid;
   708    651     if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
   709    652       int rc = sqlite3Fts5IterNextFrom(pIter, iLast);
   710    653       if( rc || sqlite3Fts5IterEof(pIter) ){
   711    654         *pRc = rc;
   712    655         *pbEof = 1;
   713    656         return 1;
   714    657       }
   715         -    iRowid = sqlite3Fts5IterRowid(pIter);
          658  +    iRowid = pIter->iRowid;
   716    659       assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) );
   717    660     }
   718    661     *piLast = iRowid;
   719    662   
   720    663     return 0;
   721    664   }
   722    665   
................................................................................
   729    672     int rc = SQLITE_OK;
   730    673     i64 iLast = *piLast;
   731    674     Fts5ExprTerm *p;
   732    675     int bEof = 0;
   733    676   
   734    677     for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
   735    678       if( sqlite3Fts5IterEof(p->pIter)==0 ){
   736         -      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
          679  +      i64 iRowid = p->pIter->iRowid;
   737    680         if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
   738    681           rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
   739    682         }
   740    683       }
   741    684     }
   742    685   
   743    686     if( rc!=SQLITE_OK ){
................................................................................
   753    696   static int fts5ExprNearTest(
   754    697     int *pRc,
   755    698     Fts5Expr *pExpr,                /* Expression that pNear is a part of */
   756    699     Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_STRING) */
   757    700   ){
   758    701     Fts5ExprNearset *pNear = pNode->pNear;
   759    702     int rc = *pRc;
   760         -  int i;
   761         -
   762         -  /* Check that each phrase in the nearset matches the current row.
   763         -  ** Populate the pPhrase->poslist buffers at the same time. If any
   764         -  ** phrase is not a match, break out of the loop early.  */
   765         -  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   766         -    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   767         -    if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
   768         -      int bMatch = 0;
   769         -      rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
   770         -      if( bMatch==0 ) break;
   771         -    }else{
   772         -      rc = sqlite3Fts5IterPoslistBuffer(
   773         -          pPhrase->aTerm[0].pIter, &pPhrase->poslist
   774         -      );
   775         -    }
   776         -  }
   777         -
   778         -  *pRc = rc;
   779         -  if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
   780         -    return 1;
   781         -  }
   782         -
   783         -  return 0;
   784         -}
   785         -
   786         -static int fts5ExprTokenTest(
   787         -  Fts5Expr *pExpr,                /* Expression that pNear is a part of */
   788         -  Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_TERM) */
   789         -){
   790         -  /* As this "NEAR" object is actually a single phrase that consists 
   791         -  ** of a single term only, grab pointers into the poslist managed by the
   792         -  ** fts5_index.c iterator object. This is much faster than synthesizing 
   793         -  ** a new poslist the way we have to for more complicated phrase or NEAR
   794         -  ** expressions.  */
   795         -  Fts5ExprNearset *pNear = pNode->pNear;
   796         -  Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
   797         -  Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
   798         -  Fts5Colset *pColset = pNear->pColset;
   799         -  int rc;
   800         -
   801         -  assert( pNode->eType==FTS5_TERM );
   802         -  assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
   803         -  assert( pPhrase->aTerm[0].pSynonym==0 );
   804         -
   805         -  rc = sqlite3Fts5IterPoslist(pIter, pColset, 
   806         -      (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid
   807         -  );
   808         -  pNode->bNomatch = (pPhrase->poslist.n==0);
   809         -  return rc;
   810         -}
   811         -
   812         -/*
   813         -** All individual term iterators in pNear are guaranteed to be valid when
   814         -** this function is called. This function checks if all term iterators
   815         -** point to the same rowid, and if not, advances them until they do.
   816         -** If an EOF is reached before this happens, *pbEof is set to true before
   817         -** returning.
   818         -**
   819         -** SQLITE_OK is returned if an error occurs, or an SQLite error code 
   820         -** otherwise. It is not considered an error code if an iterator reaches
   821         -** EOF.
   822         -*/
   823         -static int fts5ExprNearNextMatch(
   824         -  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
   825         -  Fts5ExprNode *pNode
   826         -){
   827         -  Fts5ExprNearset *pNear = pNode->pNear;
   828         -  Fts5ExprPhrase *pLeft = pNear->apPhrase[0];
   829         -  int rc = SQLITE_OK;
   830         -  i64 iLast;                      /* Lastest rowid any iterator points to */
   831         -  int i, j;                       /* Phrase and token index, respectively */
   832         -  int bMatch;                     /* True if all terms are at the same rowid */
   833         -  const int bDesc = pExpr->bDesc;
   834         -
   835         -  /* Check that this node should not be FTS5_TERM */
   836         -  assert( pNear->nPhrase>1 
   837         -       || pNear->apPhrase[0]->nTerm>1 
   838         -       || pNear->apPhrase[0]->aTerm[0].pSynonym
   839         -  );
   840         -
   841         -  /* Initialize iLast, the "lastest" rowid any iterator points to. If the
   842         -  ** iterator skips through rowids in the default ascending order, this means
   843         -  ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
   844         -  ** means the minimum rowid.  */
   845         -  if( pLeft->aTerm[0].pSynonym ){
   846         -    iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
   847         -  }else{
   848         -    iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
   849         -  }
   850         -
   851         -  do {
   852         -    bMatch = 1;
   853         -    for(i=0; i<pNear->nPhrase; i++){
          703  +
          704  +  if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){
          705  +    Fts5ExprTerm *pTerm;
          706  +    Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
          707  +    pPhrase->poslist.n = 0;
          708  +    for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
          709  +      Fts5IndexIter *pIter = pTerm->pIter;
          710  +      if( sqlite3Fts5IterEof(pIter)==0 ){
          711  +        if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){
          712  +          pPhrase->poslist.n = 1;
          713  +        }
          714  +      }
          715  +    }
          716  +    return pPhrase->poslist.n;
          717  +  }else{
          718  +    int i;
          719  +
          720  +    /* Check that each phrase in the nearset matches the current row.
          721  +    ** Populate the pPhrase->poslist buffers at the same time. If any
          722  +    ** phrase is not a match, break out of the loop early.  */
          723  +    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   854    724         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   855         -      for(j=0; j<pPhrase->nTerm; j++){
   856         -        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
   857         -        if( pTerm->pSynonym ){
   858         -          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
   859         -          if( iRowid==iLast ) continue;
   860         -          bMatch = 0;
   861         -          if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
   862         -            pNode->bEof = 1;
   863         -            return rc;
   864         -          }
   865         -        }else{
   866         -          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
   867         -          i64 iRowid = sqlite3Fts5IterRowid(pIter);
   868         -          if( iRowid==iLast ) continue;
   869         -          bMatch = 0;
   870         -          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
   871         -            return rc;
   872         -          }
   873         -        }
   874         -      }
   875         -    }
   876         -  }while( bMatch==0 );
   877         -
   878         -  pNode->iRowid = iLast;
   879         -  pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
   880         -
   881         -  return rc;
   882         -}
          725  +      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
          726  +        int bMatch = 0;
          727  +        rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
          728  +        if( bMatch==0 ) break;
          729  +      }else{
          730  +        Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
          731  +        fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData);
          732  +      }
          733  +    }
          734  +
          735  +    *pRc = rc;
          736  +    if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
          737  +      return 1;
          738  +    }
          739  +    return 0;
          740  +  }
          741  +}
          742  +
   883    743   
   884    744   /*
   885    745   ** Initialize all term iterators in the pNear object. If any term is found
   886    746   ** to match no documents at all, return immediately without initializing any
   887    747   ** further iterators.
   888    748   */
   889    749   static int fts5ExprNearInitAll(
................................................................................
   890    750     Fts5Expr *pExpr,
   891    751     Fts5ExprNode *pNode
   892    752   ){
   893    753     Fts5ExprNearset *pNear = pNode->pNear;
   894    754     int i, j;
   895    755     int rc = SQLITE_OK;
   896    756   
          757  +  assert( pNode->bNomatch==0 );
   897    758     for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   898    759       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   899    760       for(j=0; j<pPhrase->nTerm; j++){
   900    761         Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
   901    762         Fts5ExprTerm *p;
   902    763         int bEof = 1;
   903    764   
................................................................................
   924    785           return rc;
   925    786         }
   926    787       }
   927    788     }
   928    789   
   929    790     return rc;
   930    791   }
   931         -
   932         -/* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */
   933         -static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*);
   934         -
   935    792   
   936    793   /*
   937    794   ** If pExpr is an ASC iterator, this function returns a value with the
   938    795   ** same sign as:
   939    796   **
   940    797   **   (iLhs - iRhs)
   941    798   **
................................................................................
   957    814       return (iLhs < iRhs);
   958    815     }
   959    816   }
   960    817   
   961    818   static void fts5ExprSetEof(Fts5ExprNode *pNode){
   962    819     int i;
   963    820     pNode->bEof = 1;
          821  +  pNode->bNomatch = 0;
   964    822     for(i=0; i<pNode->nChild; i++){
   965    823       fts5ExprSetEof(pNode->apChild[i]);
   966    824     }
   967    825   }
   968    826   
   969    827   static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){
   970    828     if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
................................................................................
   979    837       for(i=0; i<pNode->nChild; i++){
   980    838         fts5ExprNodeZeroPoslist(pNode->apChild[i]);
   981    839       }
   982    840     }
   983    841   }
   984    842   
   985    843   
   986         -static int fts5ExprNodeNext(Fts5Expr*, Fts5ExprNode*, int, i64);
          844  +
          845  +/*
          846  +** Compare the values currently indicated by the two nodes as follows:
          847  +**
          848  +**    res = (*p1) - (*p2)
          849  +**
          850  +** Nodes that point to values that come later in the iteration order are
          851  +** considered to be larger. Nodes at EOF are the largest of all.
          852  +**
          853  +** This means that if the iteration order is ASC, then numerically larger
          854  +** rowids are considered larger. Or if it is the default DESC, numerically
          855  +** smaller rowids are larger.
          856  +*/
          857  +static int fts5NodeCompare(
          858  +  Fts5Expr *pExpr,
          859  +  Fts5ExprNode *p1, 
          860  +  Fts5ExprNode *p2
          861  +){
          862  +  if( p2->bEof ) return -1;
          863  +  if( p1->bEof ) return +1;
          864  +  return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
          865  +}
          866  +
          867  +/*
          868  +** All individual term iterators in pNear are guaranteed to be valid when
          869  +** this function is called. This function checks if all term iterators
          870  +** point to the same rowid, and if not, advances them until they do.
          871  +** If an EOF is reached before this happens, *pbEof is set to true before
          872  +** returning.
          873  +**
          874  +** SQLITE_OK is returned if an error occurs, or an SQLite error code 
          875  +** otherwise. It is not considered an error code if an iterator reaches
          876  +** EOF.
          877  +*/
          878  +static int fts5ExprNodeTest_STRING(
          879  +  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
          880  +  Fts5ExprNode *pNode
          881  +){
          882  +  Fts5ExprNearset *pNear = pNode->pNear;
          883  +  Fts5ExprPhrase *pLeft = pNear->apPhrase[0];
          884  +  int rc = SQLITE_OK;
          885  +  i64 iLast;                      /* Lastest rowid any iterator points to */
          886  +  int i, j;                       /* Phrase and token index, respectively */
          887  +  int bMatch;                     /* True if all terms are at the same rowid */
          888  +  const int bDesc = pExpr->bDesc;
          889  +
          890  +  /* Check that this node should not be FTS5_TERM */
          891  +  assert( pNear->nPhrase>1 
          892  +       || pNear->apPhrase[0]->nTerm>1 
          893  +       || pNear->apPhrase[0]->aTerm[0].pSynonym
          894  +  );
          895  +
          896  +  /* Initialize iLast, the "lastest" rowid any iterator points to. If the
          897  +  ** iterator skips through rowids in the default ascending order, this means
          898  +  ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
          899  +  ** means the minimum rowid.  */
          900  +  if( pLeft->aTerm[0].pSynonym ){
          901  +    iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
          902  +  }else{
          903  +    iLast = pLeft->aTerm[0].pIter->iRowid;
          904  +  }
          905  +
          906  +  do {
          907  +    bMatch = 1;
          908  +    for(i=0; i<pNear->nPhrase; i++){
          909  +      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
          910  +      for(j=0; j<pPhrase->nTerm; j++){
          911  +        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
          912  +        if( pTerm->pSynonym ){
          913  +          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
          914  +          if( iRowid==iLast ) continue;
          915  +          bMatch = 0;
          916  +          if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
          917  +            pNode->bNomatch = 0;
          918  +            pNode->bEof = 1;
          919  +            return rc;
          920  +          }
          921  +        }else{
          922  +          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
          923  +          if( pIter->iRowid==iLast ) continue;
          924  +          bMatch = 0;
          925  +          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
          926  +            return rc;
          927  +          }
          928  +        }
          929  +      }
          930  +    }
          931  +  }while( bMatch==0 );
          932  +
          933  +  pNode->iRowid = iLast;
          934  +  pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK);
          935  +  assert( pNode->bEof==0 || pNode->bNomatch==0 );
          936  +
          937  +  return rc;
          938  +}
          939  +
          940  +/*
          941  +** Advance the first term iterator in the first phrase of pNear. Set output
          942  +** variable *pbEof to true if it reaches EOF or if an error occurs.
          943  +**
          944  +** Return SQLITE_OK if successful, or an SQLite error code if an error
          945  +** occurs.
          946  +*/
          947  +static int fts5ExprNodeNext_STRING(
          948  +  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
          949  +  Fts5ExprNode *pNode,            /* FTS5_STRING or FTS5_TERM node */
          950  +  int bFromValid,
          951  +  i64 iFrom 
          952  +){
          953  +  Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
          954  +  int rc = SQLITE_OK;
          955  +
          956  +  pNode->bNomatch = 0;
          957  +  if( pTerm->pSynonym ){
          958  +    int bEof = 1;
          959  +    Fts5ExprTerm *p;
          960  +
          961  +    /* Find the firstest rowid any synonym points to. */
          962  +    i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
          963  +
          964  +    /* Advance each iterator that currently points to iRowid. Or, if iFrom
          965  +    ** is valid - each iterator that points to a rowid before iFrom.  */
          966  +    for(p=pTerm; p; p=p->pSynonym){
          967  +      if( sqlite3Fts5IterEof(p->pIter)==0 ){
          968  +        i64 ii = p->pIter->iRowid;
          969  +        if( ii==iRowid 
          970  +         || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) 
          971  +        ){
          972  +          if( bFromValid ){
          973  +            rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
          974  +          }else{
          975  +            rc = sqlite3Fts5IterNext(p->pIter);
          976  +          }
          977  +          if( rc!=SQLITE_OK ) break;
          978  +          if( sqlite3Fts5IterEof(p->pIter)==0 ){
          979  +            bEof = 0;
          980  +          }
          981  +        }else{
          982  +          bEof = 0;
          983  +        }
          984  +      }
          985  +    }
          986  +
          987  +    /* Set the EOF flag if either all synonym iterators are at EOF or an
          988  +    ** error has occurred.  */
          989  +    pNode->bEof = (rc || bEof);
          990  +  }else{
          991  +    Fts5IndexIter *pIter = pTerm->pIter;
          992  +
          993  +    assert( Fts5NodeIsString(pNode) );
          994  +    if( bFromValid ){
          995  +      rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
          996  +    }else{
          997  +      rc = sqlite3Fts5IterNext(pIter);
          998  +    }
          999  +
         1000  +    pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
         1001  +  }
         1002  +
         1003  +  if( pNode->bEof==0 ){
         1004  +    assert( rc==SQLITE_OK );
         1005  +    rc = fts5ExprNodeTest_STRING(pExpr, pNode);
         1006  +  }
         1007  +
         1008  +  return rc;
         1009  +}
         1010  +
         1011  +
         1012  +static int fts5ExprNodeTest_TERM(
         1013  +  Fts5Expr *pExpr,                /* Expression that pNear is a part of */
         1014  +  Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_TERM) */
         1015  +){
         1016  +  /* As this "NEAR" object is actually a single phrase that consists 
         1017  +  ** of a single term only, grab pointers into the poslist managed by the
         1018  +  ** fts5_index.c iterator object. This is much faster than synthesizing 
         1019  +  ** a new poslist the way we have to for more complicated phrase or NEAR
         1020  +  ** expressions.  */
         1021  +  Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
         1022  +  Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
         1023  +
         1024  +  assert( pNode->eType==FTS5_TERM );
         1025  +  assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 );
         1026  +  assert( pPhrase->aTerm[0].pSynonym==0 );
         1027  +
         1028  +  pPhrase->poslist.n = pIter->nData;
         1029  +  if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){
         1030  +    pPhrase->poslist.p = (u8*)pIter->pData;
         1031  +  }
         1032  +  pNode->iRowid = pIter->iRowid;
         1033  +  pNode->bNomatch = (pPhrase->poslist.n==0);
         1034  +  return SQLITE_OK;
         1035  +}
         1036  +
         1037  +/*
         1038  +** xNext() method for a node of type FTS5_TERM.
         1039  +*/
         1040  +static int fts5ExprNodeNext_TERM(
         1041  +  Fts5Expr *pExpr, 
         1042  +  Fts5ExprNode *pNode,
         1043  +  int bFromValid,
         1044  +  i64 iFrom
         1045  +){
         1046  +  int rc;
         1047  +  Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
         1048  +
         1049  +  assert( pNode->bEof==0 );
         1050  +  if( bFromValid ){
         1051  +    rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
         1052  +  }else{
         1053  +    rc = sqlite3Fts5IterNext(pIter);
         1054  +  }
         1055  +  if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){
         1056  +    rc = fts5ExprNodeTest_TERM(pExpr, pNode);
         1057  +  }else{
         1058  +    pNode->bEof = 1;
         1059  +    pNode->bNomatch = 0;
         1060  +  }
         1061  +  return rc;
         1062  +}
         1063  +
         1064  +static void fts5ExprNodeTest_OR(
         1065  +  Fts5Expr *pExpr,                /* Expression of which pNode is a part */
         1066  +  Fts5ExprNode *pNode             /* Expression node to test */
         1067  +){
         1068  +  Fts5ExprNode *pNext = pNode->apChild[0];
         1069  +  int i;
         1070  +
         1071  +  for(i=1; i<pNode->nChild; i++){
         1072  +    Fts5ExprNode *pChild = pNode->apChild[i];
         1073  +    int cmp = fts5NodeCompare(pExpr, pNext, pChild);
         1074  +    if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){
         1075  +      pNext = pChild;
         1076  +    }
         1077  +  }
         1078  +  pNode->iRowid = pNext->iRowid;
         1079  +  pNode->bEof = pNext->bEof;
         1080  +  pNode->bNomatch = pNext->bNomatch;
         1081  +}
         1082  +
         1083  +static int fts5ExprNodeNext_OR(
         1084  +  Fts5Expr *pExpr, 
         1085  +  Fts5ExprNode *pNode,
         1086  +  int bFromValid,
         1087  +  i64 iFrom
         1088  +){
         1089  +  int i;
         1090  +  i64 iLast = pNode->iRowid;
         1091  +
         1092  +  for(i=0; i<pNode->nChild; i++){
         1093  +    Fts5ExprNode *p1 = pNode->apChild[i];
         1094  +    assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 );
         1095  +    if( p1->bEof==0 ){
         1096  +      if( (p1->iRowid==iLast) 
         1097  +       || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
         1098  +      ){
         1099  +        int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
         1100  +        if( rc!=SQLITE_OK ) return rc;
         1101  +      }
         1102  +    }
         1103  +  }
         1104  +
         1105  +  fts5ExprNodeTest_OR(pExpr, pNode);
         1106  +  return SQLITE_OK;
         1107  +}
   987   1108   
   988   1109   /*
   989   1110   ** Argument pNode is an FTS5_AND node.
   990   1111   */
   991         -static int fts5ExprAndNextRowid(
         1112  +static int fts5ExprNodeTest_AND(
   992   1113     Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
   993   1114     Fts5ExprNode *pAnd              /* FTS5_AND node to advance */
   994   1115   ){
   995   1116     int iChild;
   996   1117     i64 iLast = pAnd->iRowid;
   997   1118     int rc = SQLITE_OK;
   998   1119     int bMatch;
................................................................................
   999   1120   
  1000   1121     assert( pAnd->bEof==0 );
  1001   1122     do {
  1002   1123       pAnd->bNomatch = 0;
  1003   1124       bMatch = 1;
  1004   1125       for(iChild=0; iChild<pAnd->nChild; iChild++){
  1005   1126         Fts5ExprNode *pChild = pAnd->apChild[iChild];
  1006         -      if( 0 && pChild->eType==FTS5_STRING ){
  1007         -        /* TODO */
  1008         -      }else{
  1009         -        int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
  1010         -        if( cmp>0 ){
  1011         -          /* Advance pChild until it points to iLast or laster */
  1012         -          rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
  1013         -          if( rc!=SQLITE_OK ) return rc;
  1014         -        }
         1127  +      int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
         1128  +      if( cmp>0 ){
         1129  +        /* Advance pChild until it points to iLast or laster */
         1130  +        rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast);
         1131  +        if( rc!=SQLITE_OK ) return rc;
  1015   1132         }
  1016   1133   
  1017   1134         /* If the child node is now at EOF, so is the parent AND node. Otherwise,
  1018   1135         ** the child node is guaranteed to have advanced at least as far as
  1019   1136         ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
  1020   1137         ** new lastest rowid seen so far.  */
  1021   1138         assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 );
................................................................................
  1037   1154     if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){
  1038   1155       fts5ExprNodeZeroPoslist(pAnd);
  1039   1156     }
  1040   1157     pAnd->iRowid = iLast;
  1041   1158     return SQLITE_OK;
  1042   1159   }
  1043   1160   
  1044         -
  1045         -/*
  1046         -** Compare the values currently indicated by the two nodes as follows:
  1047         -**
  1048         -**    res = (*p1) - (*p2)
  1049         -**
  1050         -** Nodes that point to values that come later in the iteration order are
  1051         -** considered to be larger. Nodes at EOF are the largest of all.
  1052         -**
  1053         -** This means that if the iteration order is ASC, then numerically larger
  1054         -** rowids are considered larger. Or if it is the default DESC, numerically
  1055         -** smaller rowids are larger.
  1056         -*/
  1057         -static int fts5NodeCompare(
  1058         -  Fts5Expr *pExpr,
  1059         -  Fts5ExprNode *p1, 
  1060         -  Fts5ExprNode *p2
  1061         -){
  1062         -  if( p2->bEof ) return -1;
  1063         -  if( p1->bEof ) return +1;
  1064         -  return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
  1065         -}
  1066         -
  1067         -/*
  1068         -** Advance node iterator pNode, part of expression pExpr. If argument
  1069         -** bFromValid is zero, then pNode is advanced exactly once. Or, if argument
  1070         -** bFromValid is non-zero, then pNode is advanced until it is at or past
  1071         -** rowid value iFrom. Whether "past" means "less than" or "greater than"
  1072         -** depends on whether this is an ASC or DESC iterator.
  1073         -*/
  1074         -static int fts5ExprNodeNext(
         1161  +static int fts5ExprNodeNext_AND(
         1162  +  Fts5Expr *pExpr, 
         1163  +  Fts5ExprNode *pNode,
         1164  +  int bFromValid,
         1165  +  i64 iFrom
         1166  +){
         1167  +  int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
         1168  +  if( rc==SQLITE_OK ){
         1169  +    rc = fts5ExprNodeTest_AND(pExpr, pNode);
         1170  +  }
         1171  +  return rc;
         1172  +}
         1173  +
         1174  +static int fts5ExprNodeTest_NOT(
         1175  +  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
         1176  +  Fts5ExprNode *pNode             /* FTS5_NOT node to advance */
         1177  +){
         1178  +  int rc = SQLITE_OK;
         1179  +  Fts5ExprNode *p1 = pNode->apChild[0];
         1180  +  Fts5ExprNode *p2 = pNode->apChild[1];
         1181  +  assert( pNode->nChild==2 );
         1182  +
         1183  +  while( rc==SQLITE_OK && p1->bEof==0 ){
         1184  +    int cmp = fts5NodeCompare(pExpr, p1, p2);
         1185  +    if( cmp>0 ){
         1186  +      rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid);
         1187  +      cmp = fts5NodeCompare(pExpr, p1, p2);
         1188  +    }
         1189  +    assert( rc!=SQLITE_OK || cmp<=0 );
         1190  +    if( cmp || p2->bNomatch ) break;
         1191  +    rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
         1192  +  }
         1193  +  pNode->bEof = p1->bEof;
         1194  +  pNode->bNomatch = p1->bNomatch;
         1195  +  pNode->iRowid = p1->iRowid;
         1196  +  if( p1->bEof ){
         1197  +    fts5ExprNodeZeroPoslist(p2);
         1198  +  }
         1199  +  return rc;
         1200  +}
         1201  +
         1202  +static int fts5ExprNodeNext_NOT(
  1075   1203     Fts5Expr *pExpr, 
  1076   1204     Fts5ExprNode *pNode,
  1077   1205     int bFromValid,
  1078   1206     i64 iFrom
  1079   1207   ){
  1080         -  int rc = SQLITE_OK;
  1081         -
  1082         -  if( pNode->bEof==0 ){
  1083         -    switch( pNode->eType ){
  1084         -      case FTS5_STRING: {
  1085         -        rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom);
  1086         -        break;
  1087         -      };
  1088         -
  1089         -      case FTS5_TERM: {
  1090         -        Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
  1091         -        if( bFromValid ){
  1092         -          rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
  1093         -        }else{
  1094         -          rc = sqlite3Fts5IterNext(pIter);
  1095         -        }
  1096         -        if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){
  1097         -          assert( rc==SQLITE_OK );
  1098         -          rc = fts5ExprTokenTest(pExpr, pNode);
  1099         -        }else{
  1100         -          pNode->bEof = 1;
  1101         -        }
  1102         -        return rc;
  1103         -      };
  1104         -
  1105         -      case FTS5_AND: {
  1106         -        Fts5ExprNode *pLeft = pNode->apChild[0];
  1107         -        rc = fts5ExprNodeNext(pExpr, pLeft, bFromValid, iFrom);
  1108         -        break;
  1109         -      }
  1110         -
  1111         -      case FTS5_OR: {
  1112         -        int i;
  1113         -        i64 iLast = pNode->iRowid;
  1114         -
  1115         -        for(i=0; rc==SQLITE_OK && i<pNode->nChild; i++){
  1116         -          Fts5ExprNode *p1 = pNode->apChild[i];
  1117         -          assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 );
  1118         -          if( p1->bEof==0 ){
  1119         -            if( (p1->iRowid==iLast) 
  1120         -             || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
  1121         -            ){
  1122         -              rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom);
  1123         -            }
  1124         -          }
  1125         -        }
  1126         -
  1127         -        break;
  1128         -      }
  1129         -
  1130         -      default: assert( pNode->eType==FTS5_NOT ); {
  1131         -        assert( pNode->nChild==2 );
  1132         -        rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
  1133         -        break;
  1134         -      }
  1135         -    }
  1136         -
  1137         -    if( rc==SQLITE_OK ){
  1138         -      rc = fts5ExprNodeNextMatch(pExpr, pNode);
  1139         -    }
         1208  +  int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom);
         1209  +  if( rc==SQLITE_OK ){
         1210  +    rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  1140   1211     }
  1141         -
  1142         -  /* Assert that if bFromValid was true, either:
  1143         -  **
  1144         -  **   a) an error occurred, or
  1145         -  **   b) the node is now at EOF, or
  1146         -  **   c) the node is now at or past rowid iFrom.
  1147         -  */
  1148         -  assert( bFromValid==0 
  1149         -      || rc!=SQLITE_OK                                                  /* a */
  1150         -      || pNode->bEof                                                    /* b */
  1151         -      || pNode->iRowid==iFrom || pExpr->bDesc==(pNode->iRowid<iFrom)    /* c */
  1152         -  );
  1153         -
  1154   1212     return rc;
  1155   1213   }
  1156         -
  1157   1214   
  1158   1215   /*
  1159   1216   ** If pNode currently points to a match, this function returns SQLITE_OK
  1160   1217   ** without modifying it. Otherwise, pNode is advanced until it does point
  1161   1218   ** to a match or EOF is reached.
  1162   1219   */
  1163         -static int fts5ExprNodeNextMatch(
         1220  +static int fts5ExprNodeTest(
  1164   1221     Fts5Expr *pExpr,                /* Expression of which pNode is a part */
  1165   1222     Fts5ExprNode *pNode             /* Expression node to test */
  1166   1223   ){
  1167   1224     int rc = SQLITE_OK;
  1168   1225     if( pNode->bEof==0 ){
  1169   1226       switch( pNode->eType ){
  1170   1227   
  1171   1228         case FTS5_STRING: {
  1172         -        /* Advance the iterators until they all point to the same rowid */
  1173         -        rc = fts5ExprNearNextMatch(pExpr, pNode);
         1229  +        rc = fts5ExprNodeTest_STRING(pExpr, pNode);
  1174   1230           break;
  1175   1231         }
  1176   1232   
  1177   1233         case FTS5_TERM: {
  1178         -        rc = fts5ExprTokenTest(pExpr, pNode);
         1234  +        rc = fts5ExprNodeTest_TERM(pExpr, pNode);
  1179   1235           break;
  1180   1236         }
  1181   1237   
  1182   1238         case FTS5_AND: {
  1183         -        rc = fts5ExprAndNextRowid(pExpr, pNode);
         1239  +        rc = fts5ExprNodeTest_AND(pExpr, pNode);
  1184   1240           break;
  1185   1241         }
  1186   1242   
  1187   1243         case FTS5_OR: {
  1188         -        Fts5ExprNode *pNext = pNode->apChild[0];
  1189         -        int i;
  1190         -
  1191         -        for(i=1; i<pNode->nChild; i++){
  1192         -          Fts5ExprNode *pChild = pNode->apChild[i];
  1193         -          int cmp = fts5NodeCompare(pExpr, pNext, pChild);
  1194         -          if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){
  1195         -            pNext = pChild;
  1196         -          }
  1197         -        }
  1198         -        pNode->iRowid = pNext->iRowid;
  1199         -        pNode->bEof = pNext->bEof;
  1200         -        pNode->bNomatch = pNext->bNomatch;
         1244  +        fts5ExprNodeTest_OR(pExpr, pNode);
  1201   1245           break;
  1202   1246         }
  1203   1247   
  1204   1248         default: assert( pNode->eType==FTS5_NOT ); {
  1205         -        Fts5ExprNode *p1 = pNode->apChild[0];
  1206         -        Fts5ExprNode *p2 = pNode->apChild[1];
  1207         -        assert( pNode->nChild==2 );
  1208         -
  1209         -        while( rc==SQLITE_OK && p1->bEof==0 ){
  1210         -          int cmp = fts5NodeCompare(pExpr, p1, p2);
  1211         -          if( cmp>0 ){
  1212         -            rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid);
  1213         -            cmp = fts5NodeCompare(pExpr, p1, p2);
  1214         -          }
  1215         -          assert( rc!=SQLITE_OK || cmp<=0 );
  1216         -          if( cmp || p2->bNomatch ) break;
  1217         -          rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
  1218         -        }
  1219         -        pNode->bEof = p1->bEof;
  1220         -        pNode->iRowid = p1->iRowid;
         1249  +        rc = fts5ExprNodeTest_NOT(pExpr, pNode);
  1221   1250           break;
  1222   1251         }
  1223   1252       }
  1224   1253     }
  1225   1254     return rc;
  1226   1255   }
  1227   1256   
................................................................................
  1232   1261   **
  1233   1262   ** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise.
  1234   1263   ** It is not an error if there are no matches.
  1235   1264   */
  1236   1265   static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
  1237   1266     int rc = SQLITE_OK;
  1238   1267     pNode->bEof = 0;
         1268  +  pNode->bNomatch = 0;
  1239   1269   
  1240   1270     if( Fts5NodeIsString(pNode) ){
  1241   1271       /* Initialize all term iterators in the NEAR object. */
  1242   1272       rc = fts5ExprNearInitAll(pExpr, pNode);
         1273  +  }else if( pNode->xNext==0 ){
         1274  +    pNode->bEof = 1;
  1243   1275     }else{
  1244   1276       int i;
         1277  +    int nEof = 0;
  1245   1278       for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){
         1279  +      Fts5ExprNode *pChild = pNode->apChild[i];
  1246   1280         rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]);
         1281  +      assert( pChild->bEof==0 || pChild->bEof==1 );
         1282  +      nEof += pChild->bEof;
  1247   1283       }
  1248   1284       pNode->iRowid = pNode->apChild[0]->iRowid;
         1285  +
         1286  +    switch( pNode->eType ){
         1287  +      case FTS5_AND:
         1288  +        if( nEof>0 ) fts5ExprSetEof(pNode);
         1289  +        break;
         1290  +
         1291  +      case FTS5_OR:
         1292  +        if( pNode->nChild==nEof ) fts5ExprSetEof(pNode);
         1293  +        break;
         1294  +
         1295  +      default:
         1296  +        assert( pNode->eType==FTS5_NOT );
         1297  +        pNode->bEof = pNode->apChild[0]->bEof;
         1298  +        break;
         1299  +    }
  1249   1300     }
  1250   1301   
  1251   1302     if( rc==SQLITE_OK ){
  1252         -    rc = fts5ExprNodeNextMatch(pExpr, pNode);
         1303  +    rc = fts5ExprNodeTest(pExpr, pNode);
  1253   1304     }
  1254   1305     return rc;
  1255   1306   }
  1256   1307   
  1257   1308   
  1258   1309   /*
  1259   1310   ** Begin iterating through the set of documents in index pIdx matched by
................................................................................
  1268   1319   ** equal to iFirst.
  1269   1320   **
  1270   1321   ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
  1271   1322   ** is not considered an error if the query does not match any documents.
  1272   1323   */
  1273   1324   int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
  1274   1325     Fts5ExprNode *pRoot = p->pRoot;
  1275         -  int rc = SQLITE_OK;
  1276         -  if( pRoot ){
  1277         -    p->pIndex = pIdx;
  1278         -    p->bDesc = bDesc;
  1279         -    rc = fts5ExprNodeFirst(p, pRoot);
         1326  +  int rc;                         /* Return code */
  1280   1327   
  1281         -    /* If not at EOF but the current rowid occurs earlier than iFirst in
  1282         -    ** the iteration order, move to document iFirst or later. */
  1283         -    if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
  1284         -      rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
  1285         -    }
         1328  +  p->pIndex = pIdx;
         1329  +  p->bDesc = bDesc;
         1330  +  rc = fts5ExprNodeFirst(p, pRoot);
  1286   1331   
  1287         -    /* If the iterator is not at a real match, skip forward until it is. */
  1288         -    while( pRoot->bNomatch && rc==SQLITE_OK && pRoot->bEof==0 ){
  1289         -      rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  1290         -    }
         1332  +  /* If not at EOF but the current rowid occurs earlier than iFirst in
         1333  +  ** the iteration order, move to document iFirst or later. */
         1334  +  if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
         1335  +    rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
         1336  +  }
         1337  +
         1338  +  /* If the iterator is not at a real match, skip forward until it is. */
         1339  +  while( pRoot->bNomatch ){
         1340  +    assert( pRoot->bEof==0 && rc==SQLITE_OK );
         1341  +    rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  1291   1342     }
  1292   1343     return rc;
  1293   1344   }
  1294   1345   
  1295   1346   /*
  1296   1347   ** Move to the next document 
  1297   1348   **
  1298   1349   ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
  1299   1350   ** is not considered an error if the query does not match any documents.
  1300   1351   */
  1301   1352   int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){
  1302   1353     int rc;
  1303   1354     Fts5ExprNode *pRoot = p->pRoot;
         1355  +  assert( pRoot->bEof==0 && pRoot->bNomatch==0 );
  1304   1356     do {
  1305   1357       rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  1306         -  }while( pRoot->bNomatch && pRoot->bEof==0 && rc==SQLITE_OK );
         1358  +    assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) );
         1359  +  }while( pRoot->bNomatch );
  1307   1360     if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
  1308   1361       pRoot->bEof = 1;
  1309   1362     }
  1310   1363     return rc;
  1311   1364   }
  1312   1365   
  1313   1366   int sqlite3Fts5ExprEof(Fts5Expr *p){
  1314         -  return (p->pRoot==0 || p->pRoot->bEof);
         1367  +  return p->pRoot->bEof;
  1315   1368   }
  1316   1369   
  1317   1370   i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
  1318   1371     return p->pRoot->iRowid;
  1319   1372   }
  1320   1373   
  1321   1374   static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
................................................................................
  1332   1385       int i;
  1333   1386       for(i=0; i<pPhrase->nTerm; i++){
  1334   1387         Fts5ExprTerm *pSyn;
  1335   1388         Fts5ExprTerm *pNext;
  1336   1389         Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
  1337   1390         sqlite3_free(pTerm->zTerm);
  1338   1391         sqlite3Fts5IterClose(pTerm->pIter);
  1339         -
  1340   1392         for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
  1341   1393           pNext = pSyn->pSynonym;
  1342   1394           sqlite3Fts5IterClose(pSyn->pIter);
         1395  +        fts5BufferFree((Fts5Buffer*)&pSyn[1]);
  1343   1396           sqlite3_free(pSyn);
  1344   1397         }
  1345   1398       }
  1346   1399       if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
  1347   1400       sqlite3_free(pPhrase);
  1348   1401     }
  1349   1402   }
................................................................................
  1390   1443     }
  1391   1444   
  1392   1445     if( pRet==0 ){
  1393   1446       assert( pParse->rc!=SQLITE_OK );
  1394   1447       sqlite3Fts5ParseNearsetFree(pNear);
  1395   1448       sqlite3Fts5ParsePhraseFree(pPhrase);
  1396   1449     }else{
         1450  +    if( pRet->nPhrase>0 ){
         1451  +      Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
         1452  +      assert( pLast==pParse->apPhrase[pParse->nPhrase-2] );
         1453  +      if( pPhrase->nTerm==0 ){
         1454  +        fts5ExprPhraseFree(pPhrase);
         1455  +        pRet->nPhrase--;
         1456  +        pParse->nPhrase--;
         1457  +        pPhrase = pLast;
         1458  +      }else if( pLast->nTerm==0 ){
         1459  +        fts5ExprPhraseFree(pLast);
         1460  +        pParse->apPhrase[pParse->nPhrase-2] = pPhrase;
         1461  +        pParse->nPhrase--;
         1462  +        pRet->nPhrase--;
         1463  +      }
         1464  +    }
  1397   1465       pRet->apPhrase[pRet->nPhrase++] = pPhrase;
  1398   1466     }
  1399   1467     return pRet;
  1400   1468   }
  1401   1469   
  1402   1470   typedef struct TokenCtx TokenCtx;
  1403   1471   struct TokenCtx {
................................................................................
  1416   1484     int iUnused1,                   /* Start offset of token */
  1417   1485     int iUnused2                    /* End offset of token */
  1418   1486   ){
  1419   1487     int rc = SQLITE_OK;
  1420   1488     const int SZALLOC = 8;
  1421   1489     TokenCtx *pCtx = (TokenCtx*)pContext;
  1422   1490     Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
         1491  +
         1492  +  UNUSED_PARAM2(iUnused1, iUnused2);
  1423   1493   
  1424   1494     /* If an error has already occurred, this is a no-op */
  1425   1495     if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
         1496  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  1426   1497   
  1427         -  assert( pPhrase==0 || pPhrase->nTerm>0 );
  1428         -  if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
         1498  +  if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
  1429   1499       Fts5ExprTerm *pSyn;
  1430         -    int nByte = sizeof(Fts5ExprTerm) + nToken+1;
         1500  +    int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
  1431   1501       pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
  1432   1502       if( pSyn==0 ){
  1433   1503         rc = SQLITE_NOMEM;
  1434   1504       }else{
  1435   1505         memset(pSyn, 0, nByte);
  1436         -      pSyn->zTerm = (char*)&pSyn[1];
         1506  +      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
  1437   1507         memcpy(pSyn->zTerm, pToken, nToken);
  1438   1508         pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
  1439   1509         pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
  1440   1510       }
  1441   1511     }else{
  1442   1512       Fts5ExprTerm *pTerm;
  1443   1513       if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
................................................................................
  1522   1592       rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
  1523   1593     }
  1524   1594     sqlite3_free(z);
  1525   1595     if( rc || (rc = sCtx.rc) ){
  1526   1596       pParse->rc = rc;
  1527   1597       fts5ExprPhraseFree(sCtx.pPhrase);
  1528   1598       sCtx.pPhrase = 0;
  1529         -  }else if( sCtx.pPhrase ){
         1599  +  }else{
  1530   1600   
  1531   1601       if( pAppend==0 ){
  1532   1602         if( (pParse->nPhrase % 8)==0 ){
  1533   1603           int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
  1534   1604           Fts5ExprPhrase **apNew;
  1535   1605           apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
  1536   1606           if( apNew==0 ){
................................................................................
  1539   1609             return 0;
  1540   1610           }
  1541   1611           pParse->apPhrase = apNew;
  1542   1612         }
  1543   1613         pParse->nPhrase++;
  1544   1614       }
  1545   1615   
         1616  +    if( sCtx.pPhrase==0 ){
         1617  +      /* This happens when parsing a token or quoted phrase that contains
         1618  +      ** no token characters at all. (e.g ... MATCH '""'). */
         1619  +      sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
         1620  +    }else if( sCtx.pPhrase->nTerm ){
         1621  +      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
         1622  +    }
  1546   1623       pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  1547         -    assert( sCtx.pPhrase->nTerm>0 );
  1548         -    sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
  1549   1624     }
  1550   1625   
  1551   1626     return sCtx.pPhrase;
  1552   1627   }
  1553   1628   
  1554   1629   /*
  1555   1630   ** Create a new FTS5 expression by cloning phrase iPhrase of the
  1556   1631   ** expression passed as the second argument.
  1557   1632   */
  1558   1633   int sqlite3Fts5ExprClonePhrase(
  1559         -  Fts5Config *pConfig,
  1560   1634     Fts5Expr *pExpr, 
  1561   1635     int iPhrase, 
  1562   1636     Fts5Expr **ppNew
  1563   1637   ){
  1564   1638     int rc = SQLITE_OK;             /* Return code */
  1565   1639     Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
  1566   1640     int i;                          /* Used to iterate through phrase terms */
  1567         -
  1568   1641     Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
  1569         -
  1570   1642     TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
  1571   1643   
  1572         -
  1573   1644     pOrig = pExpr->apExprPhrase[iPhrase];
  1574         -
  1575   1645     pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
  1576   1646     if( rc==SQLITE_OK ){
  1577   1647       pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, 
  1578   1648           sizeof(Fts5ExprPhrase*));
  1579   1649     }
  1580   1650     if( rc==SQLITE_OK ){
  1581   1651       pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, 
................................................................................
  1599   1669         sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  1600   1670       }
  1601   1671     }
  1602   1672   
  1603   1673     if( rc==SQLITE_OK ){
  1604   1674       /* All the allocations succeeded. Put the expression object together. */
  1605   1675       pNew->pIndex = pExpr->pIndex;
         1676  +    pNew->pConfig = pExpr->pConfig;
  1606   1677       pNew->nPhrase = 1;
  1607   1678       pNew->apExprPhrase[0] = sCtx.pPhrase;
  1608   1679       pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
  1609   1680       pNew->pRoot->pNear->nPhrase = 1;
  1610   1681       sCtx.pPhrase->pNode = pNew->pRoot;
  1611   1682   
  1612   1683       if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
  1613   1684         pNew->pRoot->eType = FTS5_TERM;
         1685  +      pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
  1614   1686       }else{
  1615   1687         pNew->pRoot->eType = FTS5_STRING;
         1688  +      pNew->pRoot->xNext = fts5ExprNodeNext_STRING;
  1616   1689       }
  1617   1690     }else{
  1618   1691       sqlite3Fts5ExprFree(pNew);
  1619   1692       fts5ExprPhraseFree(sCtx.pPhrase);
  1620   1693       pNew = 0;
  1621   1694     }
  1622   1695   
................................................................................
  1639   1712   }
  1640   1713   
  1641   1714   void sqlite3Fts5ParseSetDistance(
  1642   1715     Fts5Parse *pParse, 
  1643   1716     Fts5ExprNearset *pNear,
  1644   1717     Fts5Token *p
  1645   1718   ){
  1646         -  int nNear = 0;
  1647         -  int i;
  1648         -  if( p->n ){
  1649         -    for(i=0; i<p->n; i++){
  1650         -      char c = (char)p->p[i];
  1651         -      if( c<'0' || c>'9' ){
  1652         -        sqlite3Fts5ParseError(
  1653         -            pParse, "expected integer, got \"%.*s\"", p->n, p->p
  1654         -        );
  1655         -        return;
         1719  +  if( pNear ){
         1720  +    int nNear = 0;
         1721  +    int i;
         1722  +    if( p->n ){
         1723  +      for(i=0; i<p->n; i++){
         1724  +        char c = (char)p->p[i];
         1725  +        if( c<'0' || c>'9' ){
         1726  +          sqlite3Fts5ParseError(
         1727  +              pParse, "expected integer, got \"%.*s\"", p->n, p->p
         1728  +              );
         1729  +          return;
         1730  +        }
         1731  +        nNear = nNear * 10 + (p->p[i] - '0');
  1656   1732         }
  1657         -      nNear = nNear * 10 + (p->p[i] - '0');
         1733  +    }else{
         1734  +      nNear = FTS5_DEFAULT_NEARDIST;
  1658   1735       }
  1659         -  }else{
  1660         -    nNear = FTS5_DEFAULT_NEARDIST;
         1736  +    pNear->nNear = nNear;
  1661   1737     }
  1662         -  pNear->nNear = nNear;
  1663   1738   }
  1664   1739   
  1665   1740   /*
  1666   1741   ** The second argument passed to this function may be NULL, or it may be
  1667   1742   ** an existing Fts5Colset object. This function returns a pointer to
  1668   1743   ** a new colset object containing the contents of (p) with new value column
  1669   1744   ** number iCol appended. 
................................................................................
  1740   1815   }
  1741   1816   
  1742   1817   void sqlite3Fts5ParseSetColset(
  1743   1818     Fts5Parse *pParse, 
  1744   1819     Fts5ExprNearset *pNear, 
  1745   1820     Fts5Colset *pColset 
  1746   1821   ){
         1822  +  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1823  +    pParse->rc = SQLITE_ERROR;
         1824  +    pParse->zErr = sqlite3_mprintf(
         1825  +      "fts5: column queries are not supported (detail=none)"
         1826  +    );
         1827  +    sqlite3_free(pColset);
         1828  +    return;
         1829  +  }
         1830  +
  1747   1831     if( pNear ){
  1748   1832       pNear->pColset = pColset;
  1749   1833     }else{
  1750   1834       sqlite3_free(pColset);
  1751   1835     }
  1752   1836   }
         1837  +
         1838  +static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
         1839  +  switch( pNode->eType ){
         1840  +    case FTS5_STRING: {
         1841  +      Fts5ExprNearset *pNear = pNode->pNear;
         1842  +      if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
         1843  +       && pNear->apPhrase[0]->aTerm[0].pSynonym==0
         1844  +      ){
         1845  +        pNode->eType = FTS5_TERM;
         1846  +        pNode->xNext = fts5ExprNodeNext_TERM;
         1847  +      }else{
         1848  +        pNode->xNext = fts5ExprNodeNext_STRING;
         1849  +      }
         1850  +      break;
         1851  +    };
         1852  +
         1853  +    case FTS5_OR: {
         1854  +      pNode->xNext = fts5ExprNodeNext_OR;
         1855  +      break;
         1856  +    };
         1857  +
         1858  +    case FTS5_AND: {
         1859  +      pNode->xNext = fts5ExprNodeNext_AND;
         1860  +      break;
         1861  +    };
         1862  +
         1863  +    default: assert( pNode->eType==FTS5_NOT ); {
         1864  +      pNode->xNext = fts5ExprNodeNext_NOT;
         1865  +      break;
         1866  +    };
         1867  +  }
         1868  +}
  1753   1869   
  1754   1870   static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
  1755   1871     if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
  1756   1872       int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
  1757   1873       memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
  1758   1874       p->nChild += pSub->nChild;
  1759   1875       sqlite3_free(pSub);
................................................................................
  1796   1912   
  1797   1913       nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1);
  1798   1914       pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
  1799   1915   
  1800   1916       if( pRet ){
  1801   1917         pRet->eType = eType;
  1802   1918         pRet->pNear = pNear;
         1919  +      fts5ExprAssignXNext(pRet);
  1803   1920         if( eType==FTS5_STRING ){
  1804   1921           int iPhrase;
  1805   1922           for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
  1806   1923             pNear->apPhrase[iPhrase]->pNode = pRet;
         1924  +          if( pNear->apPhrase[iPhrase]->nTerm==0 ){
         1925  +            pRet->xNext = 0;
         1926  +            pRet->eType = FTS5_EOF;
         1927  +          }
  1807   1928           }
  1808         -        if( pNear->nPhrase==1 
  1809         -         && pNear->apPhrase[0]->nTerm==1 
  1810         -         && pNear->apPhrase[0]->aTerm[0].pSynonym==0
         1929  +
         1930  +        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
         1931  +         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
  1811   1932           ){
  1812         -          pRet->eType = FTS5_TERM;
         1933  +          assert( pParse->rc==SQLITE_OK );
         1934  +          pParse->rc = SQLITE_ERROR;
         1935  +          assert( pParse->zErr==0 );
         1936  +          pParse->zErr = sqlite3_mprintf(
         1937  +              "fts5: %s queries are not supported (detail!=full)", 
         1938  +              pNear->nPhrase==1 ? "phrase": "NEAR"
         1939  +          );
         1940  +          sqlite3_free(pRet);
         1941  +          pRet = 0;
  1813   1942           }
         1943  +
  1814   1944         }else{
  1815   1945           fts5ExprAddChildren(pRet, pLeft);
  1816   1946           fts5ExprAddChildren(pRet, pRight);
  1817   1947         }
  1818   1948       }
  1819   1949     }
  1820   1950   
  1821   1951     if( pRet==0 ){
  1822   1952       assert( pParse->rc!=SQLITE_OK );
  1823   1953       sqlite3Fts5ParseNodeFree(pLeft);
  1824   1954       sqlite3Fts5ParseNodeFree(pRight);
  1825   1955       sqlite3Fts5ParseNearsetFree(pNear);
  1826   1956     }
         1957  +  return pRet;
         1958  +}
         1959  +
         1960  +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
         1961  +  Fts5Parse *pParse,              /* Parse context */
         1962  +  Fts5ExprNode *pLeft,            /* Left hand child expression */
         1963  +  Fts5ExprNode *pRight            /* Right hand child expression */
         1964  +){
         1965  +  Fts5ExprNode *pRet = 0;
         1966  +  Fts5ExprNode *pPrev;
         1967  +
         1968  +  if( pParse->rc ){
         1969  +    sqlite3Fts5ParseNodeFree(pLeft);
         1970  +    sqlite3Fts5ParseNodeFree(pRight);
         1971  +  }else{
         1972  +
         1973  +    assert( pLeft->eType==FTS5_STRING 
         1974  +        || pLeft->eType==FTS5_TERM
         1975  +        || pLeft->eType==FTS5_EOF
         1976  +        || pLeft->eType==FTS5_AND
         1977  +    );
         1978  +    assert( pRight->eType==FTS5_STRING 
         1979  +        || pRight->eType==FTS5_TERM 
         1980  +        || pRight->eType==FTS5_EOF 
         1981  +    );
         1982  +
         1983  +    if( pLeft->eType==FTS5_AND ){
         1984  +      pPrev = pLeft->apChild[pLeft->nChild-1];
         1985  +    }else{
         1986  +      pPrev = pLeft;
         1987  +    }
         1988  +    assert( pPrev->eType==FTS5_STRING 
         1989  +        || pPrev->eType==FTS5_TERM 
         1990  +        || pPrev->eType==FTS5_EOF 
         1991  +        );
         1992  +
         1993  +    if( pRight->eType==FTS5_EOF ){
         1994  +      assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
         1995  +      sqlite3Fts5ParseNodeFree(pRight);
         1996  +      pRet = pLeft;
         1997  +      pParse->nPhrase--;
         1998  +    }
         1999  +    else if( pPrev->eType==FTS5_EOF ){
         2000  +      Fts5ExprPhrase **ap;
         2001  +
         2002  +      if( pPrev==pLeft ){
         2003  +        pRet = pRight;
         2004  +      }else{
         2005  +        pLeft->apChild[pLeft->nChild-1] = pRight;
         2006  +        pRet = pLeft;
         2007  +      }
         2008  +
         2009  +      ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase];
         2010  +      assert( ap[0]==pPrev->pNear->apPhrase[0] );
         2011  +      memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase);
         2012  +      pParse->nPhrase--;
         2013  +
         2014  +      sqlite3Fts5ParseNodeFree(pPrev);
         2015  +    }
         2016  +    else{
         2017  +      pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0);
         2018  +    }
         2019  +  }
         2020  +
  1827   2021     return pRet;
  1828   2022   }
  1829   2023   
  1830   2024   static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
  1831   2025     int nByte = 0;
  1832   2026     Fts5ExprTerm *p;
  1833   2027     char *zQuoted;
................................................................................
  1919   2113       for(i=0; i<pNear->nPhrase; i++){
  1920   2114         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
  1921   2115   
  1922   2116         zRet = fts5PrintfAppend(zRet, " {");
  1923   2117         for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
  1924   2118           char *zTerm = pPhrase->aTerm[iTerm].zTerm;
  1925   2119           zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
         2120  +        if( pPhrase->aTerm[iTerm].bPrefix ){
         2121  +          zRet = fts5PrintfAppend(zRet, "*");
         2122  +        }
  1926   2123         }
  1927   2124   
  1928   2125         if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
  1929   2126         if( zRet==0 ) return 0;
  1930   2127       }
  1931   2128   
  1932   2129     }else{
................................................................................
  1954   2151     }
  1955   2152   
  1956   2153     return zRet;
  1957   2154   }
  1958   2155   
  1959   2156   static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
  1960   2157     char *zRet = 0;
         2158  +  if( pExpr->eType==0 ){
         2159  +    return sqlite3_mprintf("\"\"");
         2160  +  }else
  1961   2161     if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){
  1962   2162       Fts5ExprNearset *pNear = pExpr->pNear;
  1963   2163       int i; 
  1964   2164       int iTerm;
  1965   2165   
  1966   2166       if( pNear->pColset ){
  1967   2167         int iCol = pNear->pColset->aiCol[0];
................................................................................
  2014   2214       for(i=0; i<pExpr->nChild; i++){
  2015   2215         char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]);
  2016   2216         if( z==0 ){
  2017   2217           sqlite3_free(zRet);
  2018   2218           zRet = 0;
  2019   2219         }else{
  2020   2220           int e = pExpr->apChild[i]->eType;
  2021         -        int b = (e!=FTS5_STRING && e!=FTS5_TERM);
         2221  +        int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF);
  2022   2222           zRet = fts5PrintfAppend(zRet, "%s%s%z%s", 
  2023   2223               (i==0 ? "" : zOp),
  2024   2224               (b?"(":""), z, (b?")":"")
  2025   2225           );
  2026   2226         }
  2027   2227         if( zRet==0 ) break;
  2028   2228       }
................................................................................
  2086   2286   
  2087   2287     rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  2088   2288     if( rc==SQLITE_OK ){
  2089   2289       rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
  2090   2290     }
  2091   2291     if( rc==SQLITE_OK ){
  2092   2292       char *zText;
  2093         -    if( pExpr->pRoot==0 ){
         2293  +    if( pExpr->pRoot->xNext==0 ){
  2094   2294         zText = sqlite3_mprintf("");
  2095   2295       }else if( bTcl ){
  2096   2296         zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
  2097   2297       }else{
  2098   2298         zText = fts5ExprPrint(pConfig, pExpr->pRoot);
  2099   2299       }
  2100   2300       if( zText==0 ){
................................................................................
  2186   2386       { "fts5_isalnum",  fts5ExprIsAlnum },
  2187   2387       { "fts5_fold",     fts5ExprFold },
  2188   2388     };
  2189   2389     int i;
  2190   2390     int rc = SQLITE_OK;
  2191   2391     void *pCtx = (void*)pGlobal;
  2192   2392   
  2193         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aFunc); i++){
         2393  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){
  2194   2394       struct Fts5ExprFunc *p = &aFunc[i];
  2195   2395       rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
  2196   2396     }
  2197   2397   
  2198   2398     /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
  2199   2399   #ifndef NDEBUG
  2200   2400     (void)sqlite3Fts5ParserTrace;
................................................................................
  2231   2431       nRet = pPhrase->poslist.n;
  2232   2432     }else{
  2233   2433       *pa = 0;
  2234   2434       nRet = 0;
  2235   2435     }
  2236   2436     return nRet;
  2237   2437   }
         2438  +
         2439  +struct Fts5PoslistPopulator {
         2440  +  Fts5PoslistWriter writer;
         2441  +  int bOk;                        /* True if ok to populate */
         2442  +  int bMiss;
         2443  +};
         2444  +
         2445  +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
         2446  +  Fts5PoslistPopulator *pRet;
         2447  +  pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
         2448  +  if( pRet ){
         2449  +    int i;
         2450  +    memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
         2451  +    for(i=0; i<pExpr->nPhrase; i++){
         2452  +      Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
         2453  +      Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
         2454  +      assert( pExpr->apExprPhrase[i]->nTerm==1 );
         2455  +      if( bLive && 
         2456  +          (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
         2457  +      ){
         2458  +        pRet[i].bMiss = 1;
         2459  +      }else{
         2460  +        pBuf->n = 0;
         2461  +      }
         2462  +    }
         2463  +  }
         2464  +  return pRet;
         2465  +}
         2466  +
         2467  +struct Fts5ExprCtx {
         2468  +  Fts5Expr *pExpr;
         2469  +  Fts5PoslistPopulator *aPopulator;
         2470  +  i64 iOff;
         2471  +};
         2472  +typedef struct Fts5ExprCtx Fts5ExprCtx;
         2473  +
         2474  +/*
         2475  +** TODO: Make this more efficient!
         2476  +*/
         2477  +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
         2478  +  int i;
         2479  +  for(i=0; i<pColset->nCol; i++){
         2480  +    if( pColset->aiCol[i]==iCol ) return 1;
         2481  +  }
         2482  +  return 0;
         2483  +}
         2484  +
         2485  +static int fts5ExprPopulatePoslistsCb(
         2486  +  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
         2487  +  int tflags,                /* Mask of FTS5_TOKEN_* flags */
         2488  +  const char *pToken,        /* Pointer to buffer containing token */
         2489  +  int nToken,                /* Size of token in bytes */
         2490  +  int iUnused1,              /* Byte offset of token within input text */
         2491  +  int iUnused2               /* Byte offset of end of token within input text */
         2492  +){
         2493  +  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
         2494  +  Fts5Expr *pExpr = p->pExpr;
         2495  +  int i;
         2496  +
         2497  +  UNUSED_PARAM2(iUnused1, iUnused2);
         2498  +
         2499  +  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
         2500  +  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
         2501  +  for(i=0; i<pExpr->nPhrase; i++){
         2502  +    Fts5ExprTerm *pTerm;
         2503  +    if( p->aPopulator[i].bOk==0 ) continue;
         2504  +    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
         2505  +      int nTerm = (int)strlen(pTerm->zTerm);
         2506  +      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
         2507  +       && memcmp(pTerm->zTerm, pToken, nTerm)==0
         2508  +      ){
         2509  +        int rc = sqlite3Fts5PoslistWriterAppend(
         2510  +            &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
         2511  +        );
         2512  +        if( rc ) return rc;
         2513  +        break;
         2514  +      }
         2515  +    }
         2516  +  }
         2517  +  return SQLITE_OK;
         2518  +}
         2519  +
         2520  +int sqlite3Fts5ExprPopulatePoslists(
         2521  +  Fts5Config *pConfig,
         2522  +  Fts5Expr *pExpr, 
         2523  +  Fts5PoslistPopulator *aPopulator,
         2524  +  int iCol, 
         2525  +  const char *z, int n
         2526  +){
         2527  +  int i;
         2528  +  Fts5ExprCtx sCtx;
         2529  +  sCtx.pExpr = pExpr;
         2530  +  sCtx.aPopulator = aPopulator;
         2531  +  sCtx.iOff = (((i64)iCol) << 32) - 1;
         2532  +
         2533  +  for(i=0; i<pExpr->nPhrase; i++){
         2534  +    Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
         2535  +    Fts5Colset *pColset = pNode->pNear->pColset;
         2536  +    if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) 
         2537  +     || aPopulator[i].bMiss
         2538  +    ){
         2539  +      aPopulator[i].bOk = 0;
         2540  +    }else{
         2541  +      aPopulator[i].bOk = 1;
         2542  +    }
         2543  +  }
         2544  +
         2545  +  return sqlite3Fts5Tokenize(pConfig, 
         2546  +      FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
         2547  +  );
         2548  +}
         2549  +
         2550  +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
         2551  +  if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){
         2552  +    pNode->pNear->apPhrase[0]->poslist.n = 0;
         2553  +  }else{
         2554  +    int i;
         2555  +    for(i=0; i<pNode->nChild; i++){
         2556  +      fts5ExprClearPoslists(pNode->apChild[i]);
         2557  +    }
         2558  +  }
         2559  +}
         2560  +
         2561  +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
         2562  +  pNode->iRowid = iRowid;
         2563  +  pNode->bEof = 0;
         2564  +  switch( pNode->eType ){
         2565  +    case FTS5_TERM:
         2566  +    case FTS5_STRING:
         2567  +      return (pNode->pNear->apPhrase[0]->poslist.n>0);
         2568  +
         2569  +    case FTS5_AND: {
         2570  +      int i;
         2571  +      for(i=0; i<pNode->nChild; i++){
         2572  +        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
         2573  +          fts5ExprClearPoslists(pNode);
         2574  +          return 0;
         2575  +        }
         2576  +      }
         2577  +      break;
         2578  +    }
         2579  +
         2580  +    case FTS5_OR: {
         2581  +      int i;
         2582  +      int bRet = 0;
         2583  +      for(i=0; i<pNode->nChild; i++){
         2584  +        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
         2585  +          bRet = 1;
         2586  +        }
         2587  +      }
         2588  +      return bRet;
         2589  +    }
         2590  +
         2591  +    default: {
         2592  +      assert( pNode->eType==FTS5_NOT );
         2593  +      if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
         2594  +          || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
         2595  +        ){
         2596  +        fts5ExprClearPoslists(pNode);
         2597  +        return 0;
         2598  +      }
         2599  +      break;
         2600  +    }
         2601  +  }
         2602  +  return 1;
         2603  +}
         2604  +
         2605  +void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
         2606  +  fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
         2607  +}
         2608  +
         2609  +static void fts5ExprClearEof(Fts5ExprNode *pNode){
         2610  +  int i;
         2611  +  for(i=0; i<pNode->nChild; i++){
         2612  +    fts5ExprClearEof(pNode->apChild[i]);
         2613  +  }
         2614  +  pNode->bEof = 0;
         2615  +}
         2616  +void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
         2617  +  fts5ExprClearEof(pExpr->pRoot);
         2618  +}
         2619  +
         2620  +/*
         2621  +** This function is only called for detail=columns tables. 
         2622  +*/
         2623  +int sqlite3Fts5ExprPhraseCollist(
         2624  +  Fts5Expr *pExpr, 
         2625  +  int iPhrase, 
         2626  +  const u8 **ppCollist, 
         2627  +  int *pnCollist
         2628  +){
         2629  +  Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
         2630  +  Fts5ExprNode *pNode = pPhrase->pNode;
         2631  +  int rc = SQLITE_OK;
         2632  +
         2633  +  assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
         2634  +  assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         2635  +
         2636  +  if( pNode->bEof==0 
         2637  +   && pNode->iRowid==pExpr->pRoot->iRowid 
         2638  +   && pPhrase->poslist.n>0
         2639  +  ){
         2640  +    Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
         2641  +    if( pTerm->pSynonym ){
         2642  +      Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1];
         2643  +      rc = fts5ExprSynonymList(
         2644  +          pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
         2645  +      );
         2646  +    }else{
         2647  +      *ppCollist = pPhrase->aTerm[0].pIter->pData;
         2648  +      *pnCollist = pPhrase->aTerm[0].pIter->nData;
         2649  +    }
         2650  +  }else{
         2651  +    *ppCollist = 0;
         2652  +    *pnCollist = 0;
         2653  +  }
         2654  +
         2655  +  return rc;
         2656  +}
         2657  +

Changes to ext/fts5/fts5_hash.c.

    22     22   ** This file contains the implementation of an in-memory hash table used
    23     23   ** to accumuluate "term -> doclist" content before it is flused to a level-0
    24     24   ** segment.
    25     25   */
    26     26   
    27     27   
    28     28   struct Fts5Hash {
           29  +  int eDetail;                    /* Copy of Fts5Config.eDetail */
    29     30     int *pnByte;                    /* Pointer to bytes counter */
    30     31     int nEntry;                     /* Number of entries currently in hash */
    31     32     int nSlot;                      /* Size of aSlot[] array */
    32     33     Fts5HashEntry *pScan;           /* Current ordered scan item */
    33     34     Fts5HashEntry **aSlot;          /* Array of hash slots */
    34     35   };
    35     36   
................................................................................
    57     58   struct Fts5HashEntry {
    58     59     Fts5HashEntry *pHashNext;       /* Next hash entry with same hash-key */
    59     60     Fts5HashEntry *pScanNext;       /* Next entry in sorted order */
    60     61     
    61     62     int nAlloc;                     /* Total size of allocation */
    62     63     int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
    63     64     int nData;                      /* Total bytes of data (incl. structure) */
           65  +  int nKey;                       /* Length of zKey[] in bytes */
    64     66     u8 bDel;                        /* Set delete-flag @ iSzPoslist */
    65         -
    66         -  int iCol;                       /* Column of last value written */
           67  +  u8 bContent;                    /* Set content-flag (detail=none mode) */
           68  +  i16 iCol;                       /* Column of last value written */
    67     69     int iPos;                       /* Position of last value written */
    68     70     i64 iRowid;                     /* Rowid of last value written */
    69     71     char zKey[8];                   /* Nul-terminated entry key */
    70     72   };
    71     73   
    72     74   /*
    73     75   ** Size of Fts5HashEntry without the zKey[] array.
................................................................................
    75     77   #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
    76     78   
    77     79   
    78     80   
    79     81   /*
    80     82   ** Allocate a new hash table.
    81     83   */
    82         -int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
           84  +int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
    83     85     int rc = SQLITE_OK;
    84     86     Fts5Hash *pNew;
    85     87   
    86     88     *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash));
    87     89     if( pNew==0 ){
    88     90       rc = SQLITE_NOMEM;
    89     91     }else{
    90     92       int nByte;
    91     93       memset(pNew, 0, sizeof(Fts5Hash));
    92     94       pNew->pnByte = pnByte;
           95  +    pNew->eDetail = pConfig->eDetail;
    93     96   
    94     97       pNew->nSlot = 1024;
    95     98       nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
    96     99       pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
    97    100       if( pNew->aSlot==0 ){
    98    101         sqlite3_free(pNew);
    99    102         *ppNew = 0;
................................................................................
   178    181   
   179    182     sqlite3_free(apOld);
   180    183     pHash->nSlot = nNew;
   181    184     pHash->aSlot = apNew;
   182    185     return SQLITE_OK;
   183    186   }
   184    187   
   185         -static void fts5HashAddPoslistSize(Fts5HashEntry *p){
          188  +static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
   186    189     if( p->iSzPoslist ){
   187    190       u8 *pPtr = (u8*)p;
   188         -    int nSz = (p->nData - p->iSzPoslist - 1);         /* Size in bytes */
   189         -    int nPos = nSz*2 + p->bDel;                       /* Value of nPos field */
   190         -
   191         -    assert( p->bDel==0 || p->bDel==1 );
   192         -    if( nPos<=127 ){
   193         -      pPtr[p->iSzPoslist] = (u8)nPos;
          191  +    if( pHash->eDetail==FTS5_DETAIL_NONE ){
          192  +      assert( p->nData==p->iSzPoslist );
          193  +      if( p->bDel ){
          194  +        pPtr[p->nData++] = 0x00;
          195  +        if( p->bContent ){
          196  +          pPtr[p->nData++] = 0x00;
          197  +        }
          198  +      }
   194    199       }else{
   195         -      int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
   196         -      memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
   197         -      sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
   198         -      p->nData += (nByte-1);
          200  +      int nSz = (p->nData - p->iSzPoslist - 1);       /* Size in bytes */
          201  +      int nPos = nSz*2 + p->bDel;                     /* Value of nPos field */
          202  +
          203  +      assert( p->bDel==0 || p->bDel==1 );
          204  +      if( nPos<=127 ){
          205  +        pPtr[p->iSzPoslist] = (u8)nPos;
          206  +      }else{
          207  +        int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
          208  +        memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
          209  +        sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
          210  +        p->nData += (nByte-1);
          211  +      }
   199    212       }
   200         -    p->bDel = 0;
          213  +
   201    214       p->iSzPoslist = 0;
          215  +    p->bDel = 0;
          216  +    p->bContent = 0;
   202    217     }
   203    218   }
   204    219   
          220  +/*
          221  +** Add an entry to the in-memory hash table. The key is the concatenation
          222  +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
          223  +**
          224  +**     (bByte || pToken) -> (iRowid,iCol,iPos)
          225  +**
          226  +** Or, if iCol is negative, then the value is a delete marker.
          227  +*/
   205    228   int sqlite3Fts5HashWrite(
   206    229     Fts5Hash *pHash,
   207    230     i64 iRowid,                     /* Rowid for this entry */
   208    231     int iCol,                       /* Column token appears in (-ve -> delete) */
   209    232     int iPos,                       /* Position of token within column */
   210    233     char bByte,                     /* First byte of token */
   211    234     const char *pToken, int nToken  /* Token to add or remove to or from index */
   212    235   ){
   213    236     unsigned int iHash;
   214    237     Fts5HashEntry *p;
   215    238     u8 *pPtr;
   216    239     int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
          240  +  int bNew;                       /* If non-delete entry should be written */
          241  +  
          242  +  bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
   217    243   
   218    244     /* Attempt to locate an existing hash entry */
   219    245     iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   220    246     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   221    247       if( p->zKey[0]==bByte 
          248  +     && p->nKey==nToken
   222    249        && memcmp(&p->zKey[1], pToken, nToken)==0 
   223         -     && p->zKey[nToken+1]==0 
   224    250       ){
   225    251         break;
   226    252       }
   227    253     }
   228    254   
   229    255     /* If an existing hash entry cannot be found, create a new one. */
   230    256     if( p==0 ){
          257  +    /* Figure out how much space to allocate */
   231    258       int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
   232    259       if( nByte<128 ) nByte = 128;
   233    260   
          261  +    /* Grow the Fts5Hash.aSlot[] array if necessary. */
   234    262       if( (pHash->nEntry*2)>=pHash->nSlot ){
   235    263         int rc = fts5HashResize(pHash);
   236    264         if( rc!=SQLITE_OK ) return rc;
   237    265         iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   238    266       }
   239    267   
          268  +    /* Allocate new Fts5HashEntry and add it to the hash table. */
   240    269       p = (Fts5HashEntry*)sqlite3_malloc(nByte);
   241    270       if( !p ) return SQLITE_NOMEM;
   242    271       memset(p, 0, FTS5_HASHENTRYSIZE);
   243    272       p->nAlloc = nByte;
   244    273       p->zKey[0] = bByte;
   245    274       memcpy(&p->zKey[1], pToken, nToken);
   246    275       assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
          276  +    p->nKey = nToken;
   247    277       p->zKey[nToken+1] = '\0';
   248    278       p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
   249         -    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
   250         -    p->iSzPoslist = p->nData;
   251         -    p->nData += 1;
   252         -    p->iRowid = iRowid;
   253    279       p->pHashNext = pHash->aSlot[iHash];
   254    280       pHash->aSlot[iHash] = p;
   255    281       pHash->nEntry++;
          282  +
          283  +    /* Add the first rowid field to the hash-entry */
          284  +    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
          285  +    p->iRowid = iRowid;
          286  +
          287  +    p->iSzPoslist = p->nData;
          288  +    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
          289  +      p->nData += 1;
          290  +      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
          291  +    }
          292  +
   256    293       nIncr += p->nData;
   257         -  }
   258         -
   259         -  /* Check there is enough space to append a new entry. Worst case scenario
   260         -  ** is:
   261         -  **
   262         -  **     + 9 bytes for a new rowid,
   263         -  **     + 4 byte reserved for the "poslist size" varint.
   264         -  **     + 1 byte for a "new column" byte,
   265         -  **     + 3 bytes for a new column number (16-bit max) as a varint,
   266         -  **     + 5 bytes for the new position offset (32-bit max).
   267         -  */
   268         -  if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
   269         -    int nNew = p->nAlloc * 2;
   270         -    Fts5HashEntry *pNew;
   271         -    Fts5HashEntry **pp;
   272         -    pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
   273         -    if( pNew==0 ) return SQLITE_NOMEM;
   274         -    pNew->nAlloc = nNew;
   275         -    for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
   276         -    *pp = pNew;
   277         -    p = pNew;
          294  +  }else{
          295  +
          296  +    /* Appending to an existing hash-entry. Check that there is enough 
          297  +    ** space to append the largest possible new entry. Worst case scenario 
          298  +    ** is:
          299  +    **
          300  +    **     + 9 bytes for a new rowid,
          301  +    **     + 4 byte reserved for the "poslist size" varint.
          302  +    **     + 1 byte for a "new column" byte,
          303  +    **     + 3 bytes for a new column number (16-bit max) as a varint,
          304  +    **     + 5 bytes for the new position offset (32-bit max).
          305  +    */
          306  +    if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
          307  +      int nNew = p->nAlloc * 2;
          308  +      Fts5HashEntry *pNew;
          309  +      Fts5HashEntry **pp;
          310  +      pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
          311  +      if( pNew==0 ) return SQLITE_NOMEM;
          312  +      pNew->nAlloc = nNew;
          313  +      for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
          314  +      *pp = pNew;
          315  +      p = pNew;
          316  +    }
          317  +    nIncr -= p->nData;
   278    318     }
          319  +  assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
          320  +
   279    321     pPtr = (u8*)p;
   280         -  nIncr -= p->nData;
   281    322   
   282    323     /* If this is a new rowid, append the 4-byte size field for the previous
   283    324     ** entry, and the new rowid for this entry.  */
   284    325     if( iRowid!=p->iRowid ){
   285         -    fts5HashAddPoslistSize(p);
          326  +    fts5HashAddPoslistSize(pHash, p);
   286    327       p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
          328  +    p->iRowid = iRowid;
          329  +    bNew = 1;
   287    330       p->iSzPoslist = p->nData;
   288         -    p->nData += 1;
   289         -    p->iCol = 0;
   290         -    p->iPos = 0;
   291         -    p->iRowid = iRowid;
          331  +    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
          332  +      p->nData += 1;
          333  +      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
          334  +      p->iPos = 0;
          335  +    }
   292    336     }
   293    337   
   294    338     if( iCol>=0 ){
   295         -    /* Append a new column value, if necessary */
   296         -    assert( iCol>=p->iCol );
   297         -    if( iCol!=p->iCol ){
   298         -      pPtr[p->nData++] = 0x01;
   299         -      p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
   300         -      p->iCol = iCol;
   301         -      p->iPos = 0;
          339  +    if( pHash->eDetail==FTS5_DETAIL_NONE ){
          340  +      p->bContent = 1;
          341  +    }else{
          342  +      /* Append a new column value, if necessary */
          343  +      assert( iCol>=p->iCol );
          344  +      if( iCol!=p->iCol ){
          345  +        if( pHash->eDetail==FTS5_DETAIL_FULL ){
          346  +          pPtr[p->nData++] = 0x01;
          347  +          p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
          348  +          p->iCol = iCol;
          349  +          p->iPos = 0;
          350  +        }else{
          351  +          bNew = 1;
          352  +          p->iCol = iPos = iCol;
          353  +        }
          354  +      }
          355  +
          356  +      /* Append the new position offset, if necessary */
          357  +      if( bNew ){
          358  +        p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
          359  +        p->iPos = iPos;
          360  +      }
   302    361       }
   303         -
   304         -    /* Append the new position offset */
   305         -    p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
   306         -    p->iPos = iPos;
   307    362     }else{
   308    363       /* This is a delete. Set the delete flag. */
   309    364       p->bDel = 1;
   310    365     }
          366  +
   311    367     nIncr += p->nData;
   312         -
   313    368     *pHash->pnByte += nIncr;
   314    369     return SQLITE_OK;
   315    370   }
   316    371   
   317    372   
   318    373   /*
   319    374   ** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
................................................................................
   419    474     Fts5HashEntry *p;
   420    475   
   421    476     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   422    477       if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
   423    478     }
   424    479   
   425    480     if( p ){
   426         -    fts5HashAddPoslistSize(p);
          481  +    fts5HashAddPoslistSize(pHash, p);
   427    482       *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   428    483       *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
   429    484     }else{
   430    485       *ppDoclist = 0;
   431    486       *pnDoclist = 0;
   432    487     }
   433    488   
................................................................................
   455    510     const char **pzTerm,            /* OUT: term (nul-terminated) */
   456    511     const u8 **ppDoclist,           /* OUT: pointer to doclist */
   457    512     int *pnDoclist                  /* OUT: size of doclist in bytes */
   458    513   ){
   459    514     Fts5HashEntry *p;
   460    515     if( (p = pHash->pScan) ){
   461    516       int nTerm = (int)strlen(p->zKey);
   462         -    fts5HashAddPoslistSize(p);
          517  +    fts5HashAddPoslistSize(pHash, p);
   463    518       *pzTerm = p->zKey;
   464    519       *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   465    520       *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
   466    521     }else{
   467    522       *pzTerm = 0;
   468    523       *ppDoclist = 0;
   469    524       *pnDoclist = 0;
   470    525     }
   471    526   }
   472    527   

Changes to ext/fts5/fts5_index.c.

   257    257   #define FTS5_DATA_ZERO_PADDING 8
   258    258   #define FTS5_DATA_PADDING 20
   259    259   
   260    260   typedef struct Fts5Data Fts5Data;
   261    261   typedef struct Fts5DlidxIter Fts5DlidxIter;
   262    262   typedef struct Fts5DlidxLvl Fts5DlidxLvl;
   263    263   typedef struct Fts5DlidxWriter Fts5DlidxWriter;
          264  +typedef struct Fts5Iter Fts5Iter;
   264    265   typedef struct Fts5PageWriter Fts5PageWriter;
   265    266   typedef struct Fts5SegIter Fts5SegIter;
   266    267   typedef struct Fts5DoclistIter Fts5DoclistIter;
   267    268   typedef struct Fts5SegWriter Fts5SegWriter;
   268    269   typedef struct Fts5Structure Fts5Structure;
   269    270   typedef struct Fts5StructureLevel Fts5StructureLevel;
   270    271   typedef struct Fts5StructureSegment Fts5StructureSegment;
................................................................................
   299    300     sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
   300    301     sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   301    302     sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   302    303     sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
   303    304     sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
   304    305     sqlite3_stmt *pIdxSelect;
   305    306     int nRead;                      /* Total number of blocks read */
          307  +
          308  +  sqlite3_stmt *pDataVersion;
          309  +  i64 iStructVersion;             /* data_version when pStruct read */
          310  +  Fts5Structure *pStruct;         /* Current db structure (or NULL) */
   306    311   };
   307    312   
   308    313   struct Fts5DoclistIter {
   309    314     u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
   310    315   
   311    316     /* Output variables. aPoslist==0 at EOF */
   312    317     i64 iRowid;
................................................................................
   429    434     Fts5StructureSegment *pSeg;     /* Segment to iterate through */
   430    435     int flags;                      /* Mask of configuration flags */
   431    436     int iLeafPgno;                  /* Current leaf page number */
   432    437     Fts5Data *pLeaf;                /* Current leaf data */
   433    438     Fts5Data *pNextLeaf;            /* Leaf page (iLeafPgno+1) */
   434    439     int iLeafOffset;                /* Byte offset within current leaf */
   435    440   
          441  +  /* Next method */
          442  +  void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
          443  +
   436    444     /* The page and offset from which the current term was read. The offset 
   437    445     ** is the offset of the first rowid in the current doclist.  */
   438    446     int iTermLeafPgno;
   439    447     int iTermLeafOffset;
   440    448   
   441    449     int iPgidxOff;                  /* Next offset in pgidx */
   442    450     int iEndofDoclist;
................................................................................
   448    456   
   449    457     Fts5DlidxIter *pDlidx;          /* If there is a doclist-index */
   450    458   
   451    459     /* Variables populated based on current entry. */
   452    460     Fts5Buffer term;                /* Current term */
   453    461     i64 iRowid;                     /* Current rowid */
   454    462     int nPos;                       /* Number of bytes in current position list */
   455         -  int bDel;                       /* True if the delete flag is set */
          463  +  u8 bDel;                        /* True if the delete flag is set */
   456    464   };
   457    465   
   458    466   /*
   459    467   ** Argument is a pointer to an Fts5Data structure that contains a 
   460    468   ** leaf page.
   461    469   */
   462    470   #define ASSERT_SZLEAF_OK(x) assert( \
   463    471       (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \
   464    472   )
   465    473   
   466    474   #define FTS5_SEGITER_ONETERM 0x01
   467    475   #define FTS5_SEGITER_REVERSE 0x02
   468         -
   469    476   
   470    477   /* 
   471    478   ** Argument is a pointer to an Fts5Data structure that contains a leaf
   472    479   ** page. This macro evaluates to true if the leaf contains no terms, or
   473    480   ** false if it contains at least one term.
   474    481   */
   475    482   #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn)
................................................................................
   497    504   ** aFirst[1] contains the index in aSeg[] of the iterator that points to
   498    505   ** the smallest key overall. aFirst[0] is unused. 
   499    506   **
   500    507   ** poslist:
   501    508   **   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
   502    509   **   There is no way to tell if this is populated or not.
   503    510   */
   504         -struct Fts5IndexIter {
          511  +struct Fts5Iter {
          512  +  Fts5IndexIter base;             /* Base class containing output vars */
          513  +
   505    514     Fts5Index *pIndex;              /* Index that owns this iterator */
   506    515     Fts5Structure *pStruct;         /* Database structure for this iterator */
   507    516     Fts5Buffer poslist;             /* Buffer containing current poslist */
          517  +  Fts5Colset *pColset;            /* Restrict matches to these columns */
          518  +
          519  +  /* Invoked to set output variables. */
          520  +  void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
   508    521   
   509    522     int nSeg;                       /* Size of aSeg[] array */
   510    523     int bRev;                       /* True to iterate in reverse order */
   511    524     u8 bSkipEmpty;                  /* True to skip deleted entries */
   512         -  u8 bEof;                        /* True at EOF */
   513         -  u8 bFiltered;                   /* True if column-filter already applied */
   514    525   
   515    526     i64 iSwitchRowid;               /* Firstest rowid of other than aFirst[1] */
   516    527     Fts5CResult *aFirst;            /* Current merge state (see above) */
   517    528     Fts5SegIter aSeg[1];            /* Array of segment iterators */
   518    529   };
   519    530   
   520    531   
................................................................................
   596    607   */
   597    608   static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){
   598    609     int nCmp = MIN(pLeft->n, pRight->n);
   599    610     int res = memcmp(pLeft->p, pRight->p, nCmp);
   600    611     return (res==0 ? (pLeft->n - pRight->n) : res);
   601    612   }
   602    613   
   603         -#ifdef SQLITE_DEBUG
   604         -static int fts5BlobCompare(
   605         -  const u8 *pLeft, int nLeft, 
   606         -  const u8 *pRight, int nRight
   607         -){
   608         -  int nCmp = MIN(nLeft, nRight);
   609         -  int res = memcmp(pLeft, pRight, nCmp);
   610         -  return (res==0 ? (nLeft - nRight) : res);
   611         -}
   612         -#endif
   613         -
   614    614   static int fts5LeafFirstTermOff(Fts5Data *pLeaf){
   615    615     int ret;
   616    616     fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret);
   617    617     return ret;
   618    618   }
   619    619   
   620    620   /*
................................................................................
   697    697       p->rc = rc;
   698    698       p->nRead++;
   699    699     }
   700    700   
   701    701     assert( (pRet==0)==(p->rc!=SQLITE_OK) );
   702    702     return pRet;
   703    703   }
          704  +
   704    705   
   705    706   /*
   706    707   ** Release a reference to data record returned by an earlier call to
   707    708   ** fts5DataRead().
   708    709   */
   709    710   static void fts5DataRelease(Fts5Data *pData){
   710    711     sqlite3_free(pData);
................................................................................
   865    866       pRet->nRef = 1;
   866    867       pRet->nLevel = nLevel;
   867    868       pRet->nSegment = nSegment;
   868    869       i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter);
   869    870   
   870    871       for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){
   871    872         Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl];
   872         -      int nTotal;
          873  +      int nTotal = 0;
   873    874         int iSeg;
   874    875   
   875         -      i += fts5GetVarint32(&pData[i], pLvl->nMerge);
   876         -      i += fts5GetVarint32(&pData[i], nTotal);
   877         -      assert( nTotal>=pLvl->nMerge );
   878         -      pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, 
   879         -          nTotal * sizeof(Fts5StructureSegment)
   880         -      );
          876  +      if( i>=nData ){
          877  +        rc = FTS5_CORRUPT;
          878  +      }else{
          879  +        i += fts5GetVarint32(&pData[i], pLvl->nMerge);
          880  +        i += fts5GetVarint32(&pData[i], nTotal);
          881  +        assert( nTotal>=pLvl->nMerge );
          882  +        pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, 
          883  +            nTotal * sizeof(Fts5StructureSegment)
          884  +        );
          885  +      }
   881    886   
   882    887         if( rc==SQLITE_OK ){
   883    888           pLvl->nSeg = nTotal;
   884    889           for(iSeg=0; iSeg<nTotal; iSeg++){
          890  +          if( i>=nData ){
          891  +            rc = FTS5_CORRUPT;
          892  +            break;
          893  +          }
   885    894             i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
   886    895             i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
   887    896             i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
   888    897           }
   889         -      }else{
   890         -        fts5StructureRelease(pRet);
   891         -        pRet = 0;
   892    898         }
          899  +    }
          900  +    if( rc!=SQLITE_OK ){
          901  +      fts5StructureRelease(pRet);
          902  +      pRet = 0;
   893    903       }
   894    904     }
   895    905   
   896    906     *ppOut = pRet;
   897    907     return rc;
   898    908   }
   899    909   
................................................................................
   948    958         }
   949    959         pLvl->aSeg = aNew;
   950    960       }else{
   951    961         *pRc = SQLITE_NOMEM;
   952    962       }
   953    963     }
   954    964   }
          965  +
          966  +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
          967  +  Fts5Structure *pRet = 0;
          968  +  Fts5Config *pConfig = p->pConfig;
          969  +  int iCookie;                    /* Configuration cookie */
          970  +  Fts5Data *pData;
          971  +
          972  +  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
          973  +  if( p->rc==SQLITE_OK ){
          974  +    /* TODO: Do we need this if the leaf-index is appended? Probably... */
          975  +    memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
          976  +    p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
          977  +    if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
          978  +      p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
          979  +    }
          980  +    fts5DataRelease(pData);
          981  +    if( p->rc!=SQLITE_OK ){
          982  +      fts5StructureRelease(pRet);
          983  +      pRet = 0;
          984  +    }
          985  +  }
          986  +
          987  +  return pRet;
          988  +}
          989  +
          990  +static i64 fts5IndexDataVersion(Fts5Index *p){
          991  +  i64 iVersion = 0;
          992  +
          993  +  if( p->rc==SQLITE_OK ){
          994  +    if( p->pDataVersion==0 ){
          995  +      p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, 
          996  +          sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
          997  +          );
          998  +      if( p->rc ) return 0;
          999  +    }
         1000  +
         1001  +    if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
         1002  +      iVersion = sqlite3_column_int64(p->pDataVersion, 0);
         1003  +    }
         1004  +    p->rc = sqlite3_reset(p->pDataVersion);
         1005  +  }
         1006  +
         1007  +  return iVersion;
         1008  +}
   955   1009   
   956   1010   /*
   957   1011   ** Read, deserialize and return the structure record.
   958   1012   **
   959   1013   ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
   960   1014   ** are over-allocated as described for function fts5StructureDecode() 
   961   1015   ** above.
   962   1016   **
   963   1017   ** If an error occurs, NULL is returned and an error code left in the
   964   1018   ** Fts5Index handle. If an error has already occurred when this function
   965   1019   ** is called, it is a no-op.
   966   1020   */
   967   1021   static Fts5Structure *fts5StructureRead(Fts5Index *p){
   968         -  Fts5Config *pConfig = p->pConfig;
   969         -  Fts5Structure *pRet = 0;        /* Object to return */
   970         -  int iCookie;                    /* Configuration cookie */
   971         -  Fts5Data *pData;
   972         -
   973         -  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
   974         -  if( p->rc ) return 0;
   975         -  /* TODO: Do we need this if the leaf-index is appended? Probably... */
   976         -  memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
   977         -  p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
   978         -  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
   979         -    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
   980         -  }
   981         -
   982         -  fts5DataRelease(pData);
   983         -  if( p->rc!=SQLITE_OK ){
   984         -    fts5StructureRelease(pRet);
   985         -    pRet = 0;
   986         -  }
   987         -  return pRet;
         1022  +
         1023  +  if( p->pStruct==0 ){
         1024  +    p->iStructVersion = fts5IndexDataVersion(p);
         1025  +    if( p->rc==SQLITE_OK ){
         1026  +      p->pStruct = fts5StructureReadUncached(p);
         1027  +    }
         1028  +  }
         1029  +
         1030  +#if 0
         1031  +  else{
         1032  +    Fts5Structure *pTest = fts5StructureReadUncached(p);
         1033  +    if( pTest ){
         1034  +      int i, j;
         1035  +      assert_nc( p->pStruct->nSegment==pTest->nSegment );
         1036  +      assert_nc( p->pStruct->nLevel==pTest->nLevel );
         1037  +      for(i=0; i<pTest->nLevel; i++){
         1038  +        assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
         1039  +        assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
         1040  +        for(j=0; j<pTest->aLevel[i].nSeg; j++){
         1041  +          Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
         1042  +          Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j];
         1043  +          assert_nc( p1->iSegid==p2->iSegid );
         1044  +          assert_nc( p1->pgnoFirst==p2->pgnoFirst );
         1045  +          assert_nc( p1->pgnoLast==p2->pgnoLast );
         1046  +        }
         1047  +      }
         1048  +      fts5StructureRelease(pTest);
         1049  +    }
         1050  +  }
         1051  +#endif
         1052  +
         1053  +  if( p->rc!=SQLITE_OK ) return 0;
         1054  +  assert( p->iStructVersion!=0 );
         1055  +  assert( p->pStruct!=0 );
         1056  +  fts5StructureRef(p->pStruct);
         1057  +  return p->pStruct;
         1058  +}
         1059  +
         1060  +static void fts5StructureInvalidate(Fts5Index *p){
         1061  +  if( p->pStruct ){
         1062  +    fts5StructureRelease(p->pStruct);
         1063  +    p->pStruct = 0;
         1064  +  }
   988   1065   }
   989   1066   
   990   1067   /*
   991   1068   ** Return the total number of segments in index structure pStruct. This
   992   1069   ** function is only ever used as part of assert() conditions.
   993   1070   */
   994   1071   #ifdef SQLITE_DEBUG
................................................................................
  1488   1565   **
  1489   1566   ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the 
  1490   1567   ** position list content (if any).
  1491   1568   */
  1492   1569   static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
  1493   1570     if( p->rc==SQLITE_OK ){
  1494   1571       int iOff = pIter->iLeafOffset;  /* Offset to read at */
  1495         -    int nSz;
  1496   1572       ASSERT_SZLEAF_OK(pIter->pLeaf);
  1497         -    fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
  1498         -    pIter->bDel = (nSz & 0x0001);
  1499         -    pIter->nPos = nSz>>1;
         1573  +    if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1574  +      int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf);
         1575  +      pIter->bDel = 0;
         1576  +      pIter->nPos = 1;
         1577  +      if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
         1578  +        pIter->bDel = 1;
         1579  +        iOff++;
         1580  +        if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
         1581  +          pIter->nPos = 1;
         1582  +          iOff++;
         1583  +        }else{
         1584  +          pIter->nPos = 0;
         1585  +        }
         1586  +      }
         1587  +    }else{
         1588  +      int nSz;
         1589  +      fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
         1590  +      pIter->bDel = (nSz & 0x0001);
         1591  +      pIter->nPos = nSz>>1;
         1592  +      assert_nc( pIter->nPos>=0 );
         1593  +    }
  1500   1594       pIter->iLeafOffset = iOff;
  1501         -    assert_nc( pIter->nPos>=0 );
  1502   1595     }
  1503   1596   }
  1504   1597   
  1505   1598   static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
  1506   1599     u8 *a = pIter->pLeaf->p;        /* Buffer to read data from */
  1507   1600     int iOff = pIter->iLeafOffset;
  1508   1601   
................................................................................
  1537   1630   */
  1538   1631   static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
  1539   1632     u8 *a = pIter->pLeaf->p;        /* Buffer to read data from */
  1540   1633     int iOff = pIter->iLeafOffset;  /* Offset to read at */
  1541   1634     int nNew;                       /* Bytes of new data */
  1542   1635   
  1543   1636     iOff += fts5GetVarint32(&a[iOff], nNew);
         1637  +  if( iOff+nNew>pIter->pLeaf->nn ){
         1638  +    p->rc = FTS5_CORRUPT;
         1639  +    return;
         1640  +  }
  1544   1641     pIter->term.n = nKeep;
  1545   1642     fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
  1546   1643     iOff += nNew;
  1547   1644     pIter->iTermLeafOffset = iOff;
  1548   1645     pIter->iTermLeafPgno = pIter->iLeafPgno;
  1549   1646     pIter->iLeafOffset = iOff;
  1550   1647   
................................................................................
  1554   1651       int nExtra;
  1555   1652       pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra);
  1556   1653       pIter->iEndofDoclist += nExtra;
  1557   1654     }
  1558   1655   
  1559   1656     fts5SegIterLoadRowid(p, pIter);
  1560   1657   }
         1658  +
         1659  +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*);
         1660  +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*);
         1661  +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*);
         1662  +
         1663  +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
         1664  +  if( pIter->flags & FTS5_SEGITER_REVERSE ){
         1665  +    pIter->xNext = fts5SegIterNext_Reverse;
         1666  +  }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1667  +    pIter->xNext = fts5SegIterNext_None;
         1668  +  }else{
         1669  +    pIter->xNext = fts5SegIterNext;
         1670  +  }
         1671  +}
  1561   1672   
  1562   1673   /*
  1563   1674   ** Initialize the iterator object pIter to iterate through the entries in
  1564   1675   ** segment pSeg. The iterator is left pointing to the first entry when 
  1565   1676   ** this function returns.
  1566   1677   **
  1567   1678   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
................................................................................
  1580   1691       ** at EOF already. */
  1581   1692       assert( pIter->pLeaf==0 );
  1582   1693       return;
  1583   1694     }
  1584   1695   
  1585   1696     if( p->rc==SQLITE_OK ){
  1586   1697       memset(pIter, 0, sizeof(*pIter));
         1698  +    fts5SegIterSetNext(p, pIter);
  1587   1699       pIter->pSeg = pSeg;
  1588   1700       pIter->iLeafPgno = pSeg->pgnoFirst-1;
  1589   1701       fts5SegIterNextPage(p, pIter);
  1590   1702     }
  1591   1703   
  1592   1704     if( p->rc==SQLITE_OK ){
  1593   1705       pIter->iLeafOffset = 4;
................................................................................
  1611   1723   ** This function advances the iterator so that it points to the last 
  1612   1724   ** relevant rowid on the page and, if necessary, initializes the 
  1613   1725   ** aRowidOffset[] and iRowidOffset variables. At this point the iterator
  1614   1726   ** is in its regular state - Fts5SegIter.iLeafOffset points to the first
  1615   1727   ** byte of the position list content associated with said rowid.
  1616   1728   */
  1617   1729   static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
         1730  +  int eDetail = p->pConfig->eDetail;
  1618   1731     int n = pIter->pLeaf->szLeaf;
  1619   1732     int i = pIter->iLeafOffset;
  1620   1733     u8 *a = pIter->pLeaf->p;
  1621   1734     int iRowidOffset = 0;
  1622   1735   
  1623   1736     if( n>pIter->iEndofDoclist ){
  1624   1737       n = pIter->iEndofDoclist;
  1625   1738     }
  1626   1739   
  1627   1740     ASSERT_SZLEAF_OK(pIter->pLeaf);
  1628   1741     while( 1 ){
  1629   1742       i64 iDelta = 0;
  1630         -    int nPos;
  1631         -    int bDummy;
  1632   1743   
  1633         -    i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
  1634         -    i += nPos;
         1744  +    if( eDetail==FTS5_DETAIL_NONE ){
         1745  +      /* todo */
         1746  +      if( i<n && a[i]==0 ){
         1747  +        i++;
         1748  +        if( i<n && a[i]==0 ) i++;
         1749  +      }
         1750  +    }else{
         1751  +      int nPos;
         1752  +      int bDummy;
         1753  +      i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
         1754  +      i += nPos;
         1755  +    }
  1635   1756       if( i>=n ) break;
  1636   1757       i += fts5GetVarint(&a[i], (u64*)&iDelta);
  1637   1758       pIter->iRowid += iDelta;
  1638   1759   
         1760  +    /* If necessary, grow the pIter->aRowidOffset[] array. */
  1639   1761       if( iRowidOffset>=pIter->nRowidOffset ){
  1640   1762         int nNew = pIter->nRowidOffset + 8;
  1641   1763         int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int));
  1642   1764         if( aNew==0 ){
  1643   1765           p->rc = SQLITE_NOMEM;
  1644   1766           break;
  1645   1767         }
................................................................................
  1705   1827   }
  1706   1828   
  1707   1829   /*
  1708   1830   ** Return true if the iterator passed as the second argument currently
  1709   1831   ** points to a delete marker. A delete marker is an entry with a 0 byte
  1710   1832   ** position-list.
  1711   1833   */
  1712         -static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){
         1834  +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){
  1713   1835     Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
  1714   1836     return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
  1715   1837   }
         1838  +
         1839  +/*
         1840  +** Advance iterator pIter to the next entry.
         1841  +**
         1842  +** This version of fts5SegIterNext() is only used by reverse iterators.
         1843  +*/
         1844  +static void fts5SegIterNext_Reverse(
         1845  +  Fts5Index *p,                   /* FTS5 backend object */
         1846  +  Fts5SegIter *pIter,             /* Iterator to advance */
         1847  +  int *pbUnused                   /* Unused */
         1848  +){
         1849  +  assert( pIter->flags & FTS5_SEGITER_REVERSE );
         1850  +  assert( pIter->pNextLeaf==0 );
         1851  +  UNUSED_PARAM(pbUnused);
         1852  +
         1853  +  if( pIter->iRowidOffset>0 ){
         1854  +    u8 *a = pIter->pLeaf->p;
         1855  +    int iOff;
         1856  +    i64 iDelta;
         1857  +
         1858  +    pIter->iRowidOffset--;
         1859  +    pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset];
         1860  +    fts5SegIterLoadNPos(p, pIter);
         1861  +    iOff = pIter->iLeafOffset;
         1862  +    if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){
         1863  +      iOff += pIter->nPos;
         1864  +    }
         1865  +    fts5GetVarint(&a[iOff], (u64*)&iDelta);
         1866  +    pIter->iRowid -= iDelta;
         1867  +  }else{
         1868  +    fts5SegIterReverseNewPage(p, pIter);
         1869  +  }
         1870  +}
         1871  +
         1872  +/*
         1873  +** Advance iterator pIter to the next entry.
         1874  +**
         1875  +** This version of fts5SegIterNext() is only used if detail=none and the
         1876  +** iterator is not a reverse direction iterator.
         1877  +*/
         1878  +static void fts5SegIterNext_None(
         1879  +  Fts5Index *p,                   /* FTS5 backend object */
         1880  +  Fts5SegIter *pIter,             /* Iterator to advance */
         1881  +  int *pbNewTerm                  /* OUT: Set for new term */
         1882  +){
         1883  +  int iOff;
         1884  +
         1885  +  assert( p->rc==SQLITE_OK );
         1886  +  assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 );
         1887  +  assert( p->pConfig->eDetail==FTS5_DETAIL_NONE );
         1888  +
         1889  +  ASSERT_SZLEAF_OK(pIter->pLeaf);
         1890  +  iOff = pIter->iLeafOffset;
         1891  +
         1892  +  /* Next entry is on the next page */
         1893  +  if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
         1894  +    fts5SegIterNextPage(p, pIter);
         1895  +    if( p->rc || pIter->pLeaf==0 ) return;
         1896  +    pIter->iRowid = 0;
         1897  +    iOff = 4;
         1898  +  }
         1899  +
         1900  +  if( iOff<pIter->iEndofDoclist ){
         1901  +    /* Next entry is on the current page */
         1902  +    i64 iDelta;
         1903  +    iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta);
         1904  +    pIter->iLeafOffset = iOff;
         1905  +    pIter->iRowid += iDelta;
         1906  +  }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){
         1907  +    if( pIter->pSeg ){
         1908  +      int nKeep = 0;
         1909  +      if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){
         1910  +        iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep);
         1911  +      }
         1912  +      pIter->iLeafOffset = iOff;
         1913  +      fts5SegIterLoadTerm(p, pIter, nKeep);
         1914  +    }else{
         1915  +      const u8 *pList = 0;
         1916  +      const char *zTerm = 0;
         1917  +      int nList;
         1918  +      sqlite3Fts5HashScanNext(p->pHash);
         1919  +      sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
         1920  +      if( pList==0 ) goto next_none_eof;
         1921  +      pIter->pLeaf->p = (u8*)pList;
         1922  +      pIter->pLeaf->nn = nList;
         1923  +      pIter->pLeaf->szLeaf = nList;
         1924  +      pIter->iEndofDoclist = nList;
         1925  +      sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm);
         1926  +      pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
         1927  +    }
         1928  +
         1929  +    if( pbNewTerm ) *pbNewTerm = 1;
         1930  +  }else{
         1931  +    goto next_none_eof;
         1932  +  }
         1933  +
         1934  +  fts5SegIterLoadNPos(p, pIter);
         1935  +
         1936  +  return;
         1937  + next_none_eof:
         1938  +  fts5DataRelease(pIter->pLeaf);
         1939  +  pIter->pLeaf = 0;
         1940  +}
         1941  +
  1716   1942   
  1717   1943   /*
  1718   1944   ** Advance iterator pIter to the next entry. 
  1719   1945   **
  1720   1946   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It 
  1721   1947   ** is not considered an error if the iterator reaches EOF. If an error has 
  1722   1948   ** already occurred when this function is called, it is a no-op.
  1723   1949   */
  1724   1950   static void fts5SegIterNext(
  1725   1951     Fts5Index *p,                   /* FTS5 backend object */
  1726   1952     Fts5SegIter *pIter,             /* Iterator to advance */
  1727   1953     int *pbNewTerm                  /* OUT: Set for new term */
  1728   1954   ){
         1955  +  Fts5Data *pLeaf = pIter->pLeaf;
         1956  +  int iOff;
         1957  +  int bNewTerm = 0;
         1958  +  int nKeep = 0;
         1959  +  u8 *a;
         1960  +  int n;
         1961  +
  1729   1962     assert( pbNewTerm==0 || *pbNewTerm==0 );
  1730         -  if( p->rc==SQLITE_OK ){
  1731         -    if( pIter->flags & FTS5_SEGITER_REVERSE ){
  1732         -      assert( pIter->pNextLeaf==0 );
  1733         -      if( pIter->iRowidOffset>0 ){
  1734         -        u8 *a = pIter->pLeaf->p;
  1735         -        int iOff;
  1736         -        int nPos;
  1737         -        int bDummy;
  1738         -        i64 iDelta;
  1739         -
  1740         -        pIter->iRowidOffset--;
  1741         -        pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
  1742         -        iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
  1743         -        iOff += nPos;
  1744         -        fts5GetVarint(&a[iOff], (u64*)&iDelta);
  1745         -        pIter->iRowid -= iDelta;
  1746         -        fts5SegIterLoadNPos(p, pIter);
  1747         -      }else{
  1748         -        fts5SegIterReverseNewPage(p, pIter);
  1749         -      }
  1750         -    }else{
  1751         -      Fts5Data *pLeaf = pIter->pLeaf;
  1752         -      int iOff;
  1753         -      int bNewTerm = 0;
  1754         -      int nKeep = 0;
  1755         -
  1756         -      /* Search for the end of the position list within the current page. */
  1757         -      u8 *a = pLeaf->p;
  1758         -      int n = pLeaf->szLeaf;
  1759         -
         1963  +  assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE );
         1964  +
         1965  +  /* Search for the end of the position list within the current page. */
         1966  +  a = pLeaf->p;
         1967  +  n = pLeaf->szLeaf;
         1968  +
         1969  +  ASSERT_SZLEAF_OK(pLeaf);
         1970  +  iOff = pIter->iLeafOffset + pIter->nPos;
         1971  +
         1972  +  if( iOff<n ){
         1973  +    /* The next entry is on the current page. */
         1974  +    assert_nc( iOff<=pIter->iEndofDoclist );
         1975  +    if( iOff>=pIter->iEndofDoclist ){
         1976  +      bNewTerm = 1;
         1977  +      if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
         1978  +        iOff += fts5GetVarint32(&a[iOff], nKeep);
         1979  +      }
         1980  +    }else{
         1981  +      u64 iDelta;
         1982  +      iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
         1983  +      pIter->iRowid += iDelta;
         1984  +      assert_nc( iDelta>0 );
         1985  +    }
         1986  +    pIter->iLeafOffset = iOff;
         1987  +
         1988  +  }else if( pIter->pSeg==0 ){
         1989  +    const u8 *pList = 0;
         1990  +    const char *zTerm = 0;
         1991  +    int nList = 0;
         1992  +    assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm );
         1993  +    if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){
         1994  +      sqlite3Fts5HashScanNext(p->pHash);
         1995  +      sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
         1996  +    }
         1997  +    if( pList==0 ){
         1998  +      fts5DataRelease(pIter->pLeaf);
         1999  +      pIter->pLeaf = 0;
         2000  +    }else{
         2001  +      pIter->pLeaf->p = (u8*)pList;
         2002  +      pIter->pLeaf->nn = nList;
         2003  +      pIter->pLeaf->szLeaf = nList;
         2004  +      pIter->iEndofDoclist = nList+1;
         2005  +      sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm),
         2006  +          (u8*)zTerm);
         2007  +      pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
         2008  +      *pbNewTerm = 1;
         2009  +    }
         2010  +  }else{
         2011  +    iOff = 0;
         2012  +    /* Next entry is not on the current page */
         2013  +    while( iOff==0 ){
         2014  +      fts5SegIterNextPage(p, pIter);
         2015  +      pLeaf = pIter->pLeaf;
         2016  +      if( pLeaf==0 ) break;
  1760   2017         ASSERT_SZLEAF_OK(pLeaf);
  1761         -      iOff = pIter->iLeafOffset + pIter->nPos;
  1762         -
  1763         -      if( iOff<n ){
  1764         -        /* The next entry is on the current page. */
  1765         -        assert_nc( iOff<=pIter->iEndofDoclist );
  1766         -        if( iOff>=pIter->iEndofDoclist ){
  1767         -          bNewTerm = 1;
  1768         -          if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
  1769         -            iOff += fts5GetVarint32(&a[iOff], nKeep);
  1770         -          }
  1771         -        }else{
  1772         -          u64 iDelta;
  1773         -          iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
  1774         -          pIter->iRowid += iDelta;
  1775         -          assert_nc( iDelta>0 );
  1776         -        }
         2018  +      if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
         2019  +        iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
  1777   2020           pIter->iLeafOffset = iOff;
  1778   2021   
  1779         -      }else if( pIter->pSeg==0 ){
  1780         -        const u8 *pList = 0;
  1781         -        const char *zTerm = 0;
  1782         -        int nList = 0;
  1783         -        assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm );
  1784         -        if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){
  1785         -          sqlite3Fts5HashScanNext(p->pHash);
  1786         -          sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
  1787         -        }
  1788         -        if( pList==0 ){
  1789         -          fts5DataRelease(pIter->pLeaf);
  1790         -          pIter->pLeaf = 0;
  1791         -        }else{
  1792         -          pIter->pLeaf->p = (u8*)pList;
  1793         -          pIter->pLeaf->nn = nList;
  1794         -          pIter->pLeaf->szLeaf = nList;
  1795         -          pIter->iEndofDoclist = nList+1;
  1796         -          sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm),
  1797         -              (u8*)zTerm);
  1798         -          pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
  1799         -          *pbNewTerm = 1;
  1800         -        }
  1801         -      }else{
  1802         -        iOff = 0;
  1803         -        /* Next entry is not on the current page */
  1804         -        while( iOff==0 ){
  1805         -          fts5SegIterNextPage(p, pIter);
  1806         -          pLeaf = pIter->pLeaf;
  1807         -          if( pLeaf==0 ) break;
  1808         -          ASSERT_SZLEAF_OK(pLeaf);
  1809         -          if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
  1810         -            iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
  1811         -            pIter->iLeafOffset = iOff;
  1812         -
  1813         -            if( pLeaf->nn>pLeaf->szLeaf ){
  1814         -              pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  1815         -                  &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
         2022  +        if( pLeaf->nn>pLeaf->szLeaf ){
         2023  +          pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
         2024  +              &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
  1816   2025                 );
  1817         -            }
         2026  +        }
  1818   2027   
  1819         -          }
  1820         -          else if( pLeaf->nn>pLeaf->szLeaf ){
  1821         -            pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  1822         -                &pLeaf->p[pLeaf->szLeaf], iOff
         2028  +      }
         2029  +      else if( pLeaf->nn>pLeaf->szLeaf ){
         2030  +        pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
         2031  +            &pLeaf->p[pLeaf->szLeaf], iOff
  1823   2032               );
  1824         -            pIter->iLeafOffset = iOff;
  1825         -            pIter->iEndofDoclist = iOff;
  1826         -            bNewTerm = 1;
  1827         -          }
  1828         -          if( iOff>=pLeaf->szLeaf ){
  1829         -            p->rc = FTS5_CORRUPT;
  1830         -            return;
  1831         -          }
  1832         -        }
  1833         -      }
  1834         -
  1835         -      /* Check if the iterator is now at EOF. If so, return early. */
  1836         -      if( pIter->pLeaf ){
  1837         -        if( bNewTerm ){
  1838         -          if( pIter->flags & FTS5_SEGITER_ONETERM ){
  1839         -            fts5DataRelease(pIter->pLeaf);
  1840         -            pIter->pLeaf = 0;
  1841         -          }else{
  1842         -            fts5SegIterLoadTerm(p, pIter, nKeep);
  1843         -            fts5SegIterLoadNPos(p, pIter);
  1844         -            if( pbNewTerm ) *pbNewTerm = 1;
  1845         -          }
  1846         -        }else{
  1847         -          /* The following could be done by calling fts5SegIterLoadNPos(). But
  1848         -          ** this block is particularly performance critical, so equivalent
  1849         -          ** code is inlined. */
  1850         -          int nSz;
  1851         -          assert( p->rc==SQLITE_OK );
  1852         -          fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
  1853         -          pIter->bDel = (nSz & 0x0001);
  1854         -          pIter->nPos = nSz>>1;
  1855         -          assert_nc( pIter->nPos>=0 );
  1856         -        }
  1857         -      }
         2033  +        pIter->iLeafOffset = iOff;
         2034  +        pIter->iEndofDoclist = iOff;
         2035  +        bNewTerm = 1;
         2036  +      }
         2037  +      assert_nc( iOff<pLeaf->szLeaf );
         2038  +      if( iOff>pLeaf->szLeaf ){
         2039  +        p->rc = FTS5_CORRUPT;
         2040  +        return;
         2041  +      }
         2042  +    }
         2043  +  }
         2044  +
         2045  +  /* Check if the iterator is now at EOF. If so, return early. */
         2046  +  if( pIter->pLeaf ){
         2047  +    if( bNewTerm ){
         2048  +      if( pIter->flags & FTS5_SEGITER_ONETERM ){
         2049  +        fts5DataRelease(pIter->pLeaf);
         2050  +        pIter->pLeaf = 0;
         2051  +      }else{
         2052  +        fts5SegIterLoadTerm(p, pIter, nKeep);
         2053  +        fts5SegIterLoadNPos(p, pIter);
         2054  +        if( pbNewTerm ) *pbNewTerm = 1;
         2055  +      }
         2056  +    }else{
         2057  +      /* The following could be done by calling fts5SegIterLoadNPos(). But
         2058  +      ** this block is particularly performance critical, so equivalent
         2059  +      ** code is inlined. 
         2060  +      **
         2061  +      ** Later: Switched back to fts5SegIterLoadNPos() because it supports
         2062  +      ** detail=none mode. Not ideal.
         2063  +      */
         2064  +      int nSz;
         2065  +      assert( p->rc==SQLITE_OK );
         2066  +      fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
         2067  +      pIter->bDel = (nSz & 0x0001);
         2068  +      pIter->nPos = nSz>>1;
         2069  +      assert_nc( pIter->nPos>=0 );
  1858   2070       }
  1859   2071     }
  1860   2072   }
  1861   2073   
  1862   2074   #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; }
         2075  +
         2076  +#define fts5IndexSkipVarint(a, iOff) {            \
         2077  +  int iEnd = iOff+9;                              \
         2078  +  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
         2079  +}
  1863   2080   
  1864   2081   /*
  1865   2082   ** Iterator pIter currently points to the first rowid in a doclist. This
  1866   2083   ** function sets the iterator up so that iterates in reverse order through
  1867   2084   ** the doclist.
  1868   2085   */
  1869   2086   static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
................................................................................
  1877   2094       pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast));
  1878   2095     }else{
  1879   2096       Fts5Data *pLeaf = pIter->pLeaf;         /* Current leaf data */
  1880   2097   
  1881   2098       /* Currently, Fts5SegIter.iLeafOffset points to the first byte of
  1882   2099       ** position-list content for the current rowid. Back it up so that it
  1883   2100       ** points to the start of the position-list size field. */
  1884         -    pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel);
         2101  +    int iPoslist;
         2102  +    if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
         2103  +      iPoslist = pIter->iTermLeafOffset;
         2104  +    }else{
         2105  +      iPoslist = 4;
         2106  +    }
         2107  +    fts5IndexSkipVarint(pLeaf->p, iPoslist);
         2108  +    pIter->iLeafOffset = iPoslist;
  1885   2109   
  1886   2110       /* If this condition is true then the largest rowid for the current
  1887   2111       ** term may not be stored on the current page. So search forward to
  1888   2112       ** see where said rowid really is.  */
  1889   2113       if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
  1890   2114         int pgno;
  1891   2115         Fts5StructureSegment *pSeg = pIter->pSeg;
................................................................................
  1961   2185     ){
  1962   2186       return;
  1963   2187     }
  1964   2188   
  1965   2189     pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
  1966   2190   }
  1967   2191   
  1968         -#define fts5IndexSkipVarint(a, iOff) {            \
  1969         -  int iEnd = iOff+9;                              \
  1970         -  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
  1971         -}
  1972         -
  1973   2192   /*
  1974   2193   ** The iterator object passed as the second argument currently contains
  1975   2194   ** no valid values except for the Fts5SegIter.pLeaf member variable. This
  1976   2195   ** function searches the leaf page for a term matching (pTerm/nTerm).
  1977   2196   **
  1978   2197   ** If the specified term is found on the page, then the iterator is left
  1979   2198   ** pointing to it. If argument bGe is zero and the term is not found,
................................................................................
  2003   2222     int bEndOfPage = 0;
  2004   2223   
  2005   2224     assert( p->rc==SQLITE_OK );
  2006   2225   
  2007   2226     iPgidx = szLeaf;
  2008   2227     iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
  2009   2228     iOff = iTermOff;
         2229  +  if( iOff>n ){
         2230  +    p->rc = FTS5_CORRUPT;
         2231  +    return;
         2232  +  }
  2010   2233   
  2011   2234     while( 1 ){
  2012   2235   
  2013   2236       /* Figure out how many new bytes are in this term */
  2014   2237       fts5FastGetVarint32(a, iOff, nNew);
  2015   2238       if( nKeep<nMatch ){
  2016   2239         goto search_failed;
................................................................................
  2093   2316       pIter->iEndofDoclist = iTermOff + nExtra;
  2094   2317     }
  2095   2318     pIter->iPgidxOff = iPgidx;
  2096   2319   
  2097   2320     fts5SegIterLoadRowid(p, pIter);
  2098   2321     fts5SegIterLoadNPos(p, pIter);
  2099   2322   }
         2323  +
         2324  +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){
         2325  +  if( p->pIdxSelect==0 ){
         2326  +    Fts5Config *pConfig = p->pConfig;
         2327  +    fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
         2328  +          "SELECT pgno FROM '%q'.'%q_idx' WHERE "
         2329  +          "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
         2330  +          pConfig->zDb, pConfig->zName
         2331  +    ));
         2332  +  }
         2333  +  return p->pIdxSelect;
         2334  +}
  2100   2335   
  2101   2336   /*
  2102   2337   ** Initialize the object pIter to point to term pTerm/nTerm within segment
  2103   2338   ** pSeg. If there is no such term in the index, the iterator is set to EOF.
  2104   2339   **
  2105   2340   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
  2106   2341   ** an error has already occurred when this function is called, it is a no-op.
  2107   2342   */
  2108   2343   static void fts5SegIterSeekInit(
  2109   2344     Fts5Index *p,                   /* FTS5 backend */
  2110         -  Fts5Buffer *pBuf,               /* Buffer to use for loading pages */
  2111   2345     const u8 *pTerm, int nTerm,     /* Term to seek to */
  2112   2346     int flags,                      /* Mask of FTS5INDEX_XXX flags */
  2113   2347     Fts5StructureSegment *pSeg,     /* Description of segment */
  2114   2348     Fts5SegIter *pIter              /* Object to populate */
  2115   2349   ){
  2116   2350     int iPg = 1;
  2117   2351     int bGe = (flags & FTS5INDEX_QUERY_SCAN);
  2118   2352     int bDlidx = 0;                 /* True if there is a doclist-index */
  2119         -
  2120         -  static int nCall = 0;
  2121         -  nCall++;
         2353  +  sqlite3_stmt *pIdxSelect = 0;
  2122   2354   
  2123   2355     assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
  2124   2356     assert( pTerm && nTerm );
  2125   2357     memset(pIter, 0, sizeof(*pIter));
  2126   2358     pIter->pSeg = pSeg;
  2127   2359   
  2128   2360     /* This block sets stack variable iPg to the leaf page number that may
  2129   2361     ** contain term (pTerm/nTerm), if it is present in the segment. */
  2130         -  if( p->pIdxSelect==0 ){
  2131         -    Fts5Config *pConfig = p->pConfig;
  2132         -    fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
  2133         -          "SELECT pgno FROM '%q'.'%q_idx' WHERE "
  2134         -          "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
  2135         -          pConfig->zDb, pConfig->zName
  2136         -    ));
  2137         -  }
         2362  +  pIdxSelect = fts5IdxSelectStmt(p);
  2138   2363     if( p->rc ) return;
  2139         -  sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid);
  2140         -  sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
  2141         -  if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){
  2142         -    i64 val = sqlite3_column_int(p->pIdxSelect, 0);
         2364  +  sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid);
         2365  +  sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
         2366  +  if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){
         2367  +    i64 val = sqlite3_column_int(pIdxSelect, 0);
  2143   2368       iPg = (int)(val>>1);
  2144   2369       bDlidx = (val & 0x0001);
  2145   2370     }
  2146         -  p->rc = sqlite3_reset(p->pIdxSelect);
         2371  +  p->rc = sqlite3_reset(pIdxSelect);
  2147   2372   
  2148   2373     if( iPg<pSeg->pgnoFirst ){
  2149   2374       iPg = pSeg->pgnoFirst;
  2150   2375       bDlidx = 0;
  2151   2376     }
  2152   2377   
  2153   2378     pIter->iLeafPgno = iPg - 1;
................................................................................
  2167   2392           fts5SegIterLoadDlidx(p, pIter);
  2168   2393         }
  2169   2394         if( flags & FTS5INDEX_QUERY_DESC ){
  2170   2395           fts5SegIterReverse(p, pIter);
  2171   2396         }
  2172   2397       }
  2173   2398     }
         2399  +
         2400  +  fts5SegIterSetNext(p, pIter);
  2174   2401   
  2175   2402     /* Either:
  2176   2403     **
  2177   2404     **   1) an error has occurred, or
  2178   2405     **   2) the iterator points to EOF, or
  2179   2406     **   3) the iterator points to an entry with term (pTerm/nTerm), or
  2180   2407     **   4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
................................................................................
  2225   2452       sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
  2226   2453       pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
  2227   2454       if( pLeaf==0 ) return;
  2228   2455       pLeaf->p = (u8*)pList;
  2229   2456       pLeaf->nn = pLeaf->szLeaf = nList;
  2230   2457       pIter->pLeaf = pLeaf;
  2231   2458       pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
  2232         -    pIter->iEndofDoclist = pLeaf->nn+1;
         2459  +    pIter->iEndofDoclist = pLeaf->nn;
  2233   2460   
  2234   2461       if( flags & FTS5INDEX_QUERY_DESC ){
  2235   2462         pIter->flags |= FTS5_SEGITER_REVERSE;
  2236   2463         fts5SegIterReverseInitPage(p, pIter);
  2237   2464       }else{
  2238   2465         fts5SegIterLoadNPos(p, pIter);
  2239   2466       }
  2240   2467     }
         2468  +
         2469  +  fts5SegIterSetNext(p, pIter);
  2241   2470   }
  2242   2471   
  2243   2472   /*
  2244   2473   ** Zero the iterator passed as the only argument.
  2245   2474   */
  2246   2475   static void fts5SegIterClear(Fts5SegIter *pIter){
  2247   2476     fts5BufferFree(&pIter->term);
................................................................................
  2257   2486   /*
  2258   2487   ** This function is used as part of the big assert() procedure implemented by
  2259   2488   ** fts5AssertMultiIterSetup(). It ensures that the result currently stored
  2260   2489   ** in *pRes is the correct result of comparing the current positions of the
  2261   2490   ** two iterators.
  2262   2491   */
  2263   2492   static void fts5AssertComparisonResult(
  2264         -  Fts5IndexIter *pIter, 
         2493  +  Fts5Iter *pIter, 
  2265   2494     Fts5SegIter *p1,
  2266   2495     Fts5SegIter *p2,
  2267   2496     Fts5CResult *pRes
  2268   2497   ){
  2269   2498     int i1 = p1 - pIter->aSeg;
  2270   2499     int i2 = p2 - pIter->aSeg;
  2271   2500   
................................................................................
  2298   2527   
  2299   2528   /*
  2300   2529   ** This function is a no-op unless SQLITE_DEBUG is defined when this module
  2301   2530   ** is compiled. In that case, this function is essentially an assert() 
  2302   2531   ** statement used to verify that the contents of the pIter->aFirst[] array
  2303   2532   ** are correct.
  2304   2533   */
  2305         -static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
         2534  +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){
  2306   2535     if( p->rc==SQLITE_OK ){
  2307   2536       Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  2308   2537       int i;
  2309   2538   
  2310         -    assert( (pFirst->pLeaf==0)==pIter->bEof );
         2539  +    assert( (pFirst->pLeaf==0)==pIter->base.bEof );
  2311   2540   
  2312   2541       /* Check that pIter->iSwitchRowid is set correctly. */
  2313   2542       for(i=0; i<pIter->nSeg; i++){
  2314   2543         Fts5SegIter *p1 = &pIter->aSeg[i];
  2315   2544         assert( p1==pFirst 
  2316   2545              || p1->pLeaf==0 
  2317   2546              || fts5BufferCompare(&pFirst->term, &p1->term) 
................................................................................
  2343   2572   ** Do the comparison necessary to populate pIter->aFirst[iOut].
  2344   2573   **
  2345   2574   ** If the returned value is non-zero, then it is the index of an entry
  2346   2575   ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing
  2347   2576   ** to a key that is a duplicate of another, higher priority, 
  2348   2577   ** segment-iterator in the pSeg->aSeg[] array.
  2349   2578   */
  2350         -static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){
         2579  +static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
  2351   2580     int i1;                         /* Index of left-hand Fts5SegIter */
  2352   2581     int i2;                         /* Index of right-hand Fts5SegIter */
  2353   2582     int iRes;
  2354   2583     Fts5SegIter *p1;                /* Left-hand Fts5SegIter */
  2355   2584     Fts5SegIter *p2;                /* Right-hand Fts5SegIter */
  2356   2585     Fts5CResult *pRes = &pIter->aFirst[iOut];
  2357   2586   
................................................................................
  2477   2706         pIter->iLeafPgno = iLeafPgno+1;
  2478   2707         fts5SegIterReverseNewPage(p, pIter);
  2479   2708         bMove = 0;
  2480   2709       }
  2481   2710     }
  2482   2711   
  2483   2712     do{
  2484         -    if( bMove ) fts5SegIterNext(p, pIter, 0);
         2713  +    if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0);
  2485   2714       if( pIter->pLeaf==0 ) break;
  2486   2715       if( bRev==0 && pIter->iRowid>=iMatch ) break;
  2487   2716       if( bRev!=0 && pIter->iRowid<=iMatch ) break;
  2488   2717       bMove = 1;
  2489   2718     }while( p->rc==SQLITE_OK );
  2490   2719   }
  2491   2720   
  2492   2721   
  2493   2722   /*
  2494   2723   ** Free the iterator object passed as the second argument.
  2495   2724   */
  2496         -static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){
         2725  +static void fts5MultiIterFree(Fts5Iter *pIter){
  2497   2726     if( pIter ){
  2498   2727       int i;
  2499   2728       for(i=0; i<pIter->nSeg; i++){
  2500   2729         fts5SegIterClear(&pIter->aSeg[i]);
  2501   2730       }
  2502   2731       fts5StructureRelease(pIter->pStruct);
  2503   2732       fts5BufferFree(&pIter->poslist);
  2504   2733       sqlite3_free(pIter);
  2505   2734     }
  2506   2735   }
  2507   2736   
  2508   2737   static void fts5MultiIterAdvanced(
  2509   2738     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2510         -  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
         2739  +  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
  2511   2740     int iChanged,                   /* Index of sub-iterator just advanced */
  2512   2741     int iMinset                     /* Minimum entry in aFirst[] to set */
  2513   2742   ){
  2514   2743     int i;
  2515   2744     for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){
  2516   2745       int iEq;
  2517   2746       if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
  2518         -      fts5SegIterNext(p, &pIter->aSeg[iEq], 0);
         2747  +      Fts5SegIter *pSeg = &pIter->aSeg[iEq];
         2748  +      assert( p->rc==SQLITE_OK );
         2749  +      pSeg->xNext(p, pSeg, 0);
  2519   2750         i = pIter->nSeg + iEq;
  2520   2751       }
  2521   2752     }
  2522   2753   }
  2523   2754   
  2524   2755   /*
  2525   2756   ** Sub-iterator iChanged of iterator pIter has just been advanced. It still
................................................................................
  2528   2759   ** If it does so successfully, 0 is returned. Otherwise 1.
  2529   2760   **
  2530   2761   ** If non-zero is returned, the caller should call fts5MultiIterAdvanced()
  2531   2762   ** on the iterator instead. That function does the same as this one, except
  2532   2763   ** that it deals with more complicated cases as well.
  2533   2764   */ 
  2534   2765   static int fts5MultiIterAdvanceRowid(
  2535         -  Fts5Index *p,                   /* FTS5 backend to iterate within */
  2536         -  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
  2537         -  int iChanged                    /* Index of sub-iterator just advanced */
         2766  +  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
         2767  +  int iChanged,                   /* Index of sub-iterator just advanced */
         2768  +  Fts5SegIter **ppFirst
  2538   2769   ){
  2539   2770     Fts5SegIter *pNew = &pIter->aSeg[iChanged];
  2540   2771   
  2541   2772     if( pNew->iRowid==pIter->iSwitchRowid
  2542   2773      || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev
  2543   2774     ){
  2544   2775       int i;
................................................................................
  2563   2794         pRes->iFirst = (u16)(pNew - pIter->aSeg);
  2564   2795         if( i==1 ) break;
  2565   2796   
  2566   2797         pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
  2567   2798       }
  2568   2799     }
  2569   2800   
         2801  +  *ppFirst = pNew;
  2570   2802     return 0;
  2571   2803   }
  2572   2804   
  2573   2805   /*
  2574   2806   ** Set the pIter->bEof variable based on the state of the sub-iterators.
  2575   2807   */
  2576         -static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
         2808  +static void fts5MultiIterSetEof(Fts5Iter *pIter){
  2577   2809     Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  2578         -  pIter->bEof = pSeg->pLeaf==0;
         2810  +  pIter->base.bEof = pSeg->pLeaf==0;
  2579   2811     pIter->iSwitchRowid = pSeg->iRowid;
  2580   2812   }
  2581   2813   
  2582   2814   /*
  2583   2815   ** Move the iterator to the next entry. 
  2584   2816   **
  2585   2817   ** If an error occurs, an error code is left in Fts5Index.rc. It is not 
  2586   2818   ** considered an error if the iterator reaches EOF, or if it is already at 
  2587   2819   ** EOF when this function is called.
  2588   2820   */
  2589   2821   static void fts5MultiIterNext(
  2590   2822     Fts5Index *p, 
  2591         -  Fts5IndexIter *pIter,
         2823  +  Fts5Iter *pIter,
  2592   2824     int bFrom,                      /* True if argument iFrom is valid */
  2593   2825     i64 iFrom                       /* Advance at least as far as this */
  2594   2826   ){
  2595         -  if( p->rc==SQLITE_OK ){
  2596         -    int bUseFrom = bFrom;
  2597         -    do {
  2598         -      int iFirst = pIter->aFirst[1].iFirst;
  2599         -      int bNewTerm = 0;
  2600         -      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2601         -      assert( p->rc==SQLITE_OK );
  2602         -      if( bUseFrom && pSeg->pDlidx ){
  2603         -        fts5SegIterNextFrom(p, pSeg, iFrom);
  2604         -      }else{
  2605         -        fts5SegIterNext(p, pSeg, &bNewTerm);
  2606         -      }
  2607         -
  2608         -      if( pSeg->pLeaf==0 || bNewTerm 
  2609         -       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
  2610         -      ){
  2611         -        fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2612         -        fts5MultiIterSetEof(pIter);
  2613         -      }
  2614         -      fts5AssertMultiIterSetup(p, pIter);
  2615         -
  2616         -      bUseFrom = 0;
  2617         -    }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) );
         2827  +  int bUseFrom = bFrom;
         2828  +  while( p->rc==SQLITE_OK ){
         2829  +    int iFirst = pIter->aFirst[1].iFirst;
         2830  +    int bNewTerm = 0;
         2831  +    Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
         2832  +    assert( p->rc==SQLITE_OK );
         2833  +    if( bUseFrom && pSeg->pDlidx ){
         2834  +      fts5SegIterNextFrom(p, pSeg, iFrom);
         2835  +    }else{
         2836  +      pSeg->xNext(p, pSeg, &bNewTerm);
         2837  +    }
         2838  +
         2839  +    if( pSeg->pLeaf==0 || bNewTerm 
         2840  +     || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
         2841  +    ){
         2842  +      fts5MultiIterAdvanced(p, pIter, iFirst, 1);
         2843  +      fts5MultiIterSetEof(pIter);
         2844  +      pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
         2845  +      if( pSeg->pLeaf==0 ) return;
         2846  +    }
         2847  +
         2848  +    fts5AssertMultiIterSetup(p, pIter);
         2849  +    assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf );
         2850  +    if( pIter->bSkipEmpty==0 || pSeg->nPos ){
         2851  +      pIter->xSetOutputs(pIter, pSeg);
         2852  +      return;
         2853  +    }
         2854  +    bUseFrom = 0;
  2618   2855     }
  2619   2856   }
  2620   2857   
  2621   2858   static void fts5MultiIterNext2(
  2622   2859     Fts5Index *p, 
  2623         -  Fts5IndexIter *pIter,
         2860  +  Fts5Iter *pIter,
  2624   2861     int *pbNewTerm                  /* OUT: True if *might* be new term */
  2625   2862   ){
  2626   2863     assert( pIter->bSkipEmpty );
  2627   2864     if( p->rc==SQLITE_OK ){
  2628   2865       do {
  2629   2866         int iFirst = pIter->aFirst[1].iFirst;
  2630   2867         Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2631   2868         int bNewTerm = 0;
  2632   2869   
  2633         -      fts5SegIterNext(p, pSeg, &bNewTerm);
         2870  +      assert( p->rc==SQLITE_OK );
         2871  +      pSeg->xNext(p, pSeg, &bNewTerm);
  2634   2872         if( pSeg->pLeaf==0 || bNewTerm 
  2635         -       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
         2873  +       || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
  2636   2874         ){
  2637   2875           fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2638   2876           fts5MultiIterSetEof(pIter);
  2639   2877           *pbNewTerm = 1;
  2640   2878         }else{
  2641   2879           *pbNewTerm = 0;
  2642   2880         }
  2643   2881         fts5AssertMultiIterSetup(p, pIter);
  2644   2882   
  2645   2883       }while( fts5MultiIterIsEmpty(p, pIter) );
  2646   2884     }
  2647   2885   }
  2648   2886   
         2887  +static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){
         2888  +  UNUSED_PARAM2(pUnused1, pUnused2);
         2889  +}
  2649   2890   
  2650         -static Fts5IndexIter *fts5MultiIterAlloc(
         2891  +static Fts5Iter *fts5MultiIterAlloc(
  2651   2892     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2652   2893     int nSeg
  2653   2894   ){
  2654         -  Fts5IndexIter *pNew;
         2895  +  Fts5Iter *pNew;
  2655   2896     int nSlot;                      /* Power of two >= nSeg */
  2656   2897   
  2657   2898     for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
  2658   2899     pNew = fts5IdxMalloc(p, 
  2659         -      sizeof(Fts5IndexIter) +             /* pNew */
         2900  +      sizeof(Fts5Iter) +                  /* pNew */
  2660   2901         sizeof(Fts5SegIter) * (nSlot-1) +   /* pNew->aSeg[] */
  2661   2902         sizeof(Fts5CResult) * nSlot         /* pNew->aFirst[] */
  2662   2903     );
  2663   2904     if( pNew ){
  2664   2905       pNew->nSeg = nSlot;
  2665   2906       pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
  2666   2907       pNew->pIndex = p;
         2908  +    pNew->xSetOutputs = fts5IterSetOutputs_Noop;
  2667   2909     }
  2668   2910     return pNew;
  2669   2911   }
  2670   2912   
  2671         -/*
  2672         -** Allocate a new Fts5IndexIter object.
  2673         -**
  2674         -** The new object will be used to iterate through data in structure pStruct.
  2675         -** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
  2676         -** is zero or greater, data from the first nSegment segments on level iLevel
  2677         -** is merged.
  2678         -**
  2679         -** The iterator initially points to the first term/rowid entry in the 
  2680         -** iterated data.
  2681         -*/
  2682         -static void fts5MultiIterNew(
  2683         -  Fts5Index *p,                   /* FTS5 backend to iterate within */
  2684         -  Fts5Structure *pStruct,         /* Structure of specific index */
  2685         -  int bSkipEmpty,                 /* True to ignore delete-keys */
  2686         -  int flags,                      /* FTS5INDEX_QUERY_XXX flags */
  2687         -  const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
  2688         -  int iLevel,                     /* Level to iterate (-1 for all) */
  2689         -  int nSegment,                   /* Number of segments to merge (iLevel>=0) */
  2690         -  Fts5IndexIter **ppOut           /* New object */
  2691         -){
  2692         -  int nSeg = 0;                   /* Number of segment-iters in use */
  2693         -  int iIter = 0;                  /* */
  2694         -  int iSeg;                       /* Used to iterate through segments */
  2695         -  Fts5Buffer buf = {0,0,0};       /* Buffer used by fts5SegIterSeekInit() */
  2696         -  Fts5StructureLevel *pLvl;
  2697         -  Fts5IndexIter *pNew;
  2698         -
  2699         -  assert( (pTerm==0 && nTerm==0) || iLevel<0 );
  2700         -
  2701         -  /* Allocate space for the new multi-seg-iterator. */
  2702         -  if( p->rc==SQLITE_OK ){
  2703         -    if( iLevel<0 ){
  2704         -      assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  2705         -      nSeg = pStruct->nSegment;
  2706         -      nSeg += (p->pHash ? 1 : 0);
  2707         -    }else{
  2708         -      nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
  2709         -    }
  2710         -  }
  2711         -  *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
  2712         -  if( pNew==0 ) return;
  2713         -  pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
  2714         -  pNew->bSkipEmpty = (u8)bSkipEmpty;
  2715         -  pNew->pStruct = pStruct;
  2716         -  fts5StructureRef(pStruct);
  2717         -
  2718         -  /* Initialize each of the component segment iterators. */
  2719         -  if( iLevel<0 ){
  2720         -    Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
  2721         -    if( p->pHash ){
  2722         -      /* Add a segment iterator for the current contents of the hash table. */
  2723         -      Fts5SegIter *pIter = &pNew->aSeg[iIter++];
  2724         -      fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
  2725         -    }
  2726         -    for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
  2727         -      for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
  2728         -        Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
  2729         -        Fts5SegIter *pIter = &pNew->aSeg[iIter++];
  2730         -        if( pTerm==0 ){
  2731         -          fts5SegIterInit(p, pSeg, pIter);
         2913  +static void fts5PoslistCallback(
         2914  +  Fts5Index *pUnused, 
         2915  +  void *pContext, 
         2916  +  const u8 *pChunk, int nChunk
         2917  +){
         2918  +  UNUSED_PARAM(pUnused);
         2919  +  assert_nc( nChunk>=0 );
         2920  +  if( nChunk>0 ){
         2921  +    fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk);
         2922  +  }
         2923  +}
         2924  +
         2925  +typedef struct PoslistCallbackCtx PoslistCallbackCtx;
         2926  +struct PoslistCallbackCtx {
         2927  +  Fts5Buffer *pBuf;               /* Append to this buffer */
         2928  +  Fts5Colset *pColset;            /* Restrict matches to this column */
         2929  +  int eState;                     /* See above */
         2930  +};
         2931  +
         2932  +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx;
         2933  +struct PoslistOffsetsCtx {
         2934  +  Fts5Buffer *pBuf;               /* Append to this buffer */
         2935  +  Fts5Colset *pColset;            /* Restrict matches to this column */
         2936  +  int iRead;
         2937  +  int iWrite;
         2938  +};
         2939  +
         2940  +/*
         2941  +** TODO: Make this more efficient!
         2942  +*/
         2943  +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
         2944  +  int i;
         2945  +  for(i=0; i<pColset->nCol; i++){
         2946  +    if( pColset->aiCol[i]==iCol ) return 1;
         2947  +  }
         2948  +  return 0;
         2949  +}
         2950  +
         2951  +static void fts5PoslistOffsetsCallback(
         2952  +  Fts5Index *pUnused, 
         2953  +  void *pContext, 
         2954  +  const u8 *pChunk, int nChunk
         2955  +){
         2956  +  PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext;
         2957  +  UNUSED_PARAM(pUnused);
         2958  +  assert_nc( nChunk>=0 );
         2959  +  if( nChunk>0 ){
         2960  +    int i = 0;
         2961  +    while( i<nChunk ){
         2962  +      int iVal;
         2963  +      i += fts5GetVarint32(&pChunk[i], iVal);
         2964  +      iVal += pCtx->iRead - 2;
         2965  +      pCtx->iRead = iVal;
         2966  +      if( fts5IndexColsetTest(pCtx->pColset, iVal) ){
         2967  +        fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite);
         2968  +        pCtx->iWrite = iVal;
         2969  +      }
         2970  +    }
         2971  +  }
         2972  +}
         2973  +
         2974  +static void fts5PoslistFilterCallback(
         2975  +  Fts5Index *pUnused,
         2976  +  void *pContext, 
         2977  +  const u8 *pChunk, int nChunk
         2978  +){
         2979  +  PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
         2980  +  UNUSED_PARAM(pUnused);
         2981  +  assert_nc( nChunk>=0 );
         2982  +  if( nChunk>0 ){
         2983  +    /* Search through to find the first varint with value 1. This is the
         2984  +    ** start of the next columns hits. */
         2985  +    int i = 0;
         2986  +    int iStart = 0;
         2987  +
         2988  +    if( pCtx->eState==2 ){
         2989  +      int iCol;
         2990  +      fts5FastGetVarint32(pChunk, i, iCol);
         2991  +      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
         2992  +        pCtx->eState = 1;
         2993  +        fts5BufferSafeAppendVarint(pCtx->pBuf, 1);
         2994  +      }else{
         2995  +        pCtx->eState = 0;
         2996  +      }
         2997  +    }
         2998  +
         2999  +    do {
         3000  +      while( i<nChunk && pChunk[i]!=0x01 ){
         3001  +        while( pChunk[i] & 0x80 ) i++;
         3002  +        i++;
         3003  +      }
         3004  +      if( pCtx->eState ){
         3005  +        fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
         3006  +      }
         3007  +      if( i<nChunk ){
         3008  +        int iCol;
         3009  +        iStart = i;
         3010  +        i++;
         3011  +        if( i>=nChunk ){
         3012  +          pCtx->eState = 2;
  2732   3013           }else{
  2733         -          fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter);
  2734         -        }
  2735         -      }
  2736         -    }
  2737         -  }else{
  2738         -    pLvl = &pStruct->aLevel[iLevel];
  2739         -    for(iSeg=nSeg-1; iSeg>=0; iSeg--){
  2740         -      fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
  2741         -    }
  2742         -  }
  2743         -  assert( iIter==nSeg );
  2744         -
  2745         -  /* If the above was successful, each component iterators now points 
  2746         -  ** to the first entry in its segment. In this case initialize the 
  2747         -  ** aFirst[] array. Or, if an error has occurred, free the iterator
  2748         -  ** object and set the output variable to NULL.  */
  2749         -  if( p->rc==SQLITE_OK ){
  2750         -    for(iIter=pNew->nSeg-1; iIter>0; iIter--){
  2751         -      int iEq;
  2752         -      if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){
  2753         -        fts5SegIterNext(p, &pNew->aSeg[iEq], 0);
  2754         -        fts5MultiIterAdvanced(p, pNew, iEq, iIter);
  2755         -      }
  2756         -    }
  2757         -    fts5MultiIterSetEof(pNew);
  2758         -    fts5AssertMultiIterSetup(p, pNew);
  2759         -
  2760         -    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
  2761         -      fts5MultiIterNext(p, pNew, 0, 0);
  2762         -    }
  2763         -  }else{
  2764         -    fts5MultiIterFree(p, pNew);
  2765         -    *ppOut = 0;
  2766         -  }
  2767         -  fts5BufferFree(&buf);
  2768         -}
  2769         -
  2770         -/*
  2771         -** Create an Fts5IndexIter that iterates through the doclist provided
  2772         -** as the second argument.
  2773         -*/
  2774         -static void fts5MultiIterNew2(
  2775         -  Fts5Index *p,                   /* FTS5 backend to iterate within */
  2776         -  Fts5Data *pData,                /* Doclist to iterate through */
  2777         -  int bDesc,                      /* True for descending rowid order */
  2778         -  Fts5IndexIter **ppOut           /* New object */
  2779         -){
  2780         -  Fts5IndexIter *pNew;
  2781         -  pNew = fts5MultiIterAlloc(p, 2);
  2782         -  if( pNew ){
  2783         -    Fts5SegIter *pIter = &pNew->aSeg[1];
  2784         -
  2785         -    pNew->bFiltered = 1;
  2786         -    pIter->flags = FTS5_SEGITER_ONETERM;
  2787         -    if( pData->szLeaf>0 ){
  2788         -      pIter->pLeaf = pData;
  2789         -      pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
  2790         -      pIter->iEndofDoclist = pData->nn;
  2791         -      pNew->aFirst[1].iFirst = 1;
  2792         -      if( bDesc ){
  2793         -        pNew->bRev = 1;
  2794         -        pIter->flags |= FTS5_SEGITER_REVERSE;
  2795         -        fts5SegIterReverseInitPage(p, pIter);
  2796         -      }else{
  2797         -        fts5SegIterLoadNPos(p, pIter);
  2798         -      }
  2799         -      pData = 0;
  2800         -    }else{
  2801         -      pNew->bEof = 1;
  2802         -    }
  2803         -
  2804         -    *ppOut = pNew;
  2805         -  }
  2806         -
  2807         -  fts5DataRelease(pData);
  2808         -}
  2809         -
  2810         -/*
  2811         -** Return true if the iterator is at EOF or if an error has occurred. 
  2812         -** False otherwise.
  2813         -*/
  2814         -static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){
  2815         -  assert( p->rc 
  2816         -      || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof 
  2817         -  );
  2818         -  return (p->rc || pIter->bEof);
  2819         -}
  2820         -
  2821         -/*
  2822         -** Return the rowid of the entry that the iterator currently points
  2823         -** to. If the iterator points to EOF when this function is called the
  2824         -** results are undefined.
  2825         -*/
  2826         -static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){
  2827         -  assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
  2828         -  return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
  2829         -}
  2830         -
  2831         -/*
  2832         -** Move the iterator to the next entry at or following iMatch.
  2833         -*/
  2834         -static void fts5MultiIterNextFrom(
  2835         -  Fts5Index *p, 
  2836         -  Fts5IndexIter *pIter, 
  2837         -  i64 iMatch
  2838         -){
  2839         -  while( 1 ){
  2840         -    i64 iRowid;
  2841         -    fts5MultiIterNext(p, pIter, 1, iMatch);
  2842         -    if( fts5MultiIterEof(p, pIter) ) break;
  2843         -    iRowid = fts5MultiIterRowid(pIter);
  2844         -    if( pIter->bRev==0 && iRowid>=iMatch ) break;
  2845         -    if( pIter->bRev!=0 && iRowid<=iMatch ) break;
  2846         -  }
  2847         -}
  2848         -
  2849         -/*
  2850         -** Return a pointer to a buffer containing the term associated with the 
  2851         -** entry that the iterator currently points to.
  2852         -*/
  2853         -static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){
  2854         -  Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  2855         -  *pn = p->term.n;
  2856         -  return p->term.p;
         3014  +          fts5FastGetVarint32(pChunk, i, iCol);
         3015  +          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
         3016  +          if( pCtx->eState ){
         3017  +            fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
         3018  +            iStart = i;
         3019  +          }
         3020  +        }
         3021  +      }
         3022  +    }while( i<nChunk );
         3023  +  }
  2857   3024   }
  2858   3025   
  2859   3026   static void fts5ChunkIterate(
  2860   3027     Fts5Index *p,                   /* Index object */
  2861   3028     Fts5SegIter *pSeg,              /* Poslist of this iterator */
  2862   3029     void *pCtx,                     /* Context pointer for xChunk callback */
  2863   3030     void (*xChunk)(Fts5Index*, void*, const u8*, int)
................................................................................
  2864   3031   ){
  2865   3032     int nRem = pSeg->nPos;          /* Number of bytes still to come */
  2866   3033     Fts5Data *pData = 0;
  2867   3034     u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  2868   3035     int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset);
  2869   3036     int pgno = pSeg->iLeafPgno;
  2870   3037     int pgnoSave = 0;
         3038  +
         3039  +  /* This function does notmwork with detail=none databases. */
         3040  +  assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE );
  2871   3041   
  2872   3042     if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){
  2873   3043       pgnoSave = pgno+1;
  2874   3044     }
  2875   3045   
  2876   3046     while( 1 ){
  2877   3047       xChunk(p, pCtx, pChunk, nChunk);
................................................................................
  2890   3060           pSeg->pNextLeaf = pData;
  2891   3061           pData = 0;
  2892   3062         }
  2893   3063       }
  2894   3064     }
  2895   3065   }
  2896   3066   
  2897         -
         3067  +/*
         3068  +** Iterator pIter currently points to a valid entry (not EOF). This
         3069  +** function appends the position list data for the current entry to
         3070  +** buffer pBuf. It does not make a copy of the position-list size
         3071  +** field.
         3072  +*/
         3073  +static void fts5SegiterPoslist(
         3074  +  Fts5Index *p,
         3075  +  Fts5SegIter *pSeg,
         3076  +  Fts5Colset *pColset,
         3077  +  Fts5Buffer *pBuf
         3078  +){
         3079  +  if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
         3080  +    if( pColset==0 ){
         3081  +      fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
         3082  +    }else{
         3083  +      if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){
         3084  +        PoslistCallbackCtx sCtx;
         3085  +        sCtx.pBuf = pBuf;
         3086  +        sCtx.pColset = pColset;
         3087  +        sCtx.eState = fts5IndexColsetTest(pColset, 0);
         3088  +        assert( sCtx.eState==0 || sCtx.eState==1 );
         3089  +        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
         3090  +      }else{
         3091  +        PoslistOffsetsCtx sCtx;
         3092  +        memset(&sCtx, 0, sizeof(sCtx));
         3093  +        sCtx.pBuf = pBuf;
         3094  +        sCtx.pColset = pColset;
         3095  +        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
         3096  +      }
         3097  +    }
         3098  +  }
         3099  +}
         3100  +
         3101  +/*
         3102  +** IN/OUT parameter (*pa) points to a position list n bytes in size. If
         3103  +** the position list contains entries for column iCol, then (*pa) is set
         3104  +** to point to the sub-position-list for that column and the number of
         3105  +** bytes in it returned. Or, if the argument position list does not
         3106  +** contain any entries for column iCol, return 0.
         3107  +*/
         3108  +static int fts5IndexExtractCol(
         3109  +  const u8 **pa,                  /* IN/OUT: Pointer to poslist */
         3110  +  int n,                          /* IN: Size of poslist in bytes */
         3111  +  int iCol                        /* Column to extract from poslist */
         3112  +){
         3113  +  int iCurrent = 0;               /* Anything before the first 0x01 is col 0 */
         3114  +  const u8 *p = *pa;
         3115  +  const u8 *pEnd = &p[n];         /* One byte past end of position list */
         3116  +
         3117  +  while( iCol>iCurrent ){
         3118  +    /* Advance pointer p until it points to pEnd or an 0x01 byte that is
         3119  +    ** not part of a varint. Note that it is not possible for a negative
         3120  +    ** or extremely large varint to occur within an uncorrupted position 
         3121  +    ** list. So the last byte of each varint may be assumed to have a clear
         3122  +    ** 0x80 bit.  */
         3123  +    while( *p!=0x01 ){
         3124  +      while( *p++ & 0x80 );
         3125  +      if( p>=pEnd ) return 0;
         3126  +    }
         3127  +    *pa = p++;
         3128  +    iCurrent = *p++;
         3129  +    if( iCurrent & 0x80 ){
         3130  +      p--;
         3131  +      p += fts5GetVarint32(p, iCurrent);
         3132  +    }
         3133  +  }
         3134  +  if( iCol!=iCurrent ) return 0;
         3135  +
         3136  +  /* Advance pointer p until it points to pEnd or an 0x01 byte that is
         3137  +  ** not part of a varint */
         3138  +  while( p<pEnd && *p!=0x01 ){
         3139  +    while( *p++ & 0x80 );
         3140  +  }
         3141  +
         3142  +  return p - (*pa);
         3143  +}
         3144  +
         3145  +static int fts5IndexExtractColset (
         3146  +  Fts5Colset *pColset,            /* Colset to filter on */
         3147  +  const u8 *pPos, int nPos,       /* Position list */
         3148  +  Fts5Buffer *pBuf                /* Output buffer */
         3149  +){
         3150  +  int rc = SQLITE_OK;
         3151  +  int i;
         3152  +
         3153  +  fts5BufferZero(pBuf);
         3154  +  for(i=0; i<pColset->nCol; i++){
         3155  +    const u8 *pSub = pPos;
         3156  +    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
         3157  +    if( nSub ){
         3158  +      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
         3159  +    }
         3160  +  }
         3161  +  return rc;
         3162  +}
         3163  +
         3164  +/*
         3165  +** xSetOutputs callback used by detail=none tables.
         3166  +*/
         3167  +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3168  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
         3169  +  pIter->base.iRowid = pSeg->iRowid;
         3170  +  pIter->base.nData = pSeg->nPos;
         3171  +}
         3172  +
         3173  +/*
         3174  +** xSetOutputs callback used by detail=full and detail=col tables when no
         3175  +** column filters are specified.
         3176  +*/
         3177  +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3178  +  pIter->base.iRowid = pSeg->iRowid;
         3179  +  pIter->base.nData = pSeg->nPos;
         3180  +
         3181  +  assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE );
         3182  +  assert( pIter->pColset==0 );
         3183  +
         3184  +  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
         3185  +    /* All data is stored on the current page. Populate the output 
         3186  +    ** variables to point into the body of the page object. */
         3187  +    pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
         3188  +  }else{
         3189  +    /* The data is distributed over two or more pages. Copy it into the
         3190  +    ** Fts5Iter.poslist buffer and then set the output pointer to point
         3191  +    ** to this buffer.  */
         3192  +    fts5BufferZero(&pIter->poslist);
         3193  +    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
         3194  +    pIter->base.pData = pIter->poslist.p;
         3195  +  }
         3196  +}
         3197  +
         3198  +/*
         3199  +** xSetOutputs callback used by detail=col when there is a column filter
         3200  +** and there are 100 or more columns. Also called as a fallback from
         3201  +** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
         3202  +*/
         3203  +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3204  +  fts5BufferZero(&pIter->poslist);
         3205  +  fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);
         3206  +  pIter->base.iRowid = pSeg->iRowid;
         3207  +  pIter->base.pData = pIter->poslist.p;
         3208  +  pIter->base.nData = pIter->poslist.n;
         3209  +}
         3210  +
         3211  +/*
         3212  +** xSetOutputs callback used when: 
         3213  +**
         3214  +**   * detail=col,
         3215  +**   * there is a column filter, and
         3216  +**   * the table contains 100 or fewer columns. 
         3217  +**
         3218  +** The last point is to ensure all column numbers are stored as 
         3219  +** single-byte varints.
         3220  +*/
         3221  +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3222  +
         3223  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         3224  +  assert( pIter->pColset );
         3225  +
         3226  +  if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){
         3227  +    fts5IterSetOutputs_Col(pIter, pSeg);
         3228  +  }else{
         3229  +    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
         3230  +    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
         3231  +    int iPrev = 0;
         3232  +    int *aiCol = pIter->pColset->aiCol;
         3233  +    int *aiColEnd = &aiCol[pIter->pColset->nCol];
         3234  +
         3235  +    u8 *aOut = pIter->poslist.p;
         3236  +    int iPrevOut = 0;
         3237  +
         3238  +    pIter->base.iRowid = pSeg->iRowid;
         3239  +
         3240  +    while( a<pEnd ){
         3241  +      iPrev += (int)a++[0] - 2;
         3242  +      while( *aiCol<iPrev ){
         3243  +        aiCol++;
         3244  +        if( aiCol==aiColEnd ) goto setoutputs_col_out;
         3245  +      }
         3246  +      if( *aiCol==iPrev ){
         3247  +        *aOut++ = (iPrev - iPrevOut) + 2;
         3248  +        iPrevOut = iPrev;
         3249  +      }
         3250  +    }
         3251  +
         3252  +setoutputs_col_out:
         3253  +    pIter->base.pData = pIter->poslist.p;
         3254  +    pIter->base.nData = aOut - pIter->poslist.p;
         3255  +  }
         3256  +}
         3257  +
         3258  +/*
         3259  +** xSetOutputs callback used by detail=full when there is a column filter.
         3260  +*/
         3261  +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
         3262  +  Fts5Colset *pColset = pIter->pColset;
         3263  +  pIter->base.iRowid = pSeg->iRowid;
         3264  +
         3265  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL );
         3266  +  assert( pColset );
         3267  +
         3268  +  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
         3269  +    /* All data is stored on the current page. Populate the output 
         3270  +    ** variables to point into the body of the page object. */
         3271  +    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
         3272  +    if( pColset->nCol==1 ){
         3273  +      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
         3274  +      pIter->base.pData = a;
         3275  +    }else{
         3276  +      fts5BufferZero(&pIter->poslist);
         3277  +      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
         3278  +      pIter->base.pData = pIter->poslist.p;
         3279  +      pIter->base.nData = pIter->poslist.n;
         3280  +    }
         3281  +  }else{
         3282  +    /* The data is distributed over two or more pages. Copy it into the
         3283  +    ** Fts5Iter.poslist buffer and then set the output pointer to point
         3284  +    ** to this buffer.  */
         3285  +    fts5BufferZero(&pIter->poslist);
         3286  +    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
         3287  +    pIter->base.pData = pIter->poslist.p;
         3288  +    pIter->base.nData = pIter->poslist.n;
         3289  +  }
         3290  +}
         3291  +
         3292  +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
         3293  +  if( *pRc==SQLITE_OK ){
         3294  +    Fts5Config *pConfig = pIter->pIndex->pConfig;
         3295  +    if( pConfig->eDetail==FTS5_DETAIL_NONE ){
         3296  +      pIter->xSetOutputs = fts5IterSetOutputs_None;
         3297  +    }
         3298  +
         3299  +    else if( pIter->pColset==0 ){
         3300  +      pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
         3301  +    }
         3302  +
         3303  +    else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
         3304  +      pIter->xSetOutputs = fts5IterSetOutputs_Full;
         3305  +    }
         3306  +
         3307  +    else{
         3308  +      assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         3309  +      if( pConfig->nCol<=100 ){
         3310  +        pIter->xSetOutputs = fts5IterSetOutputs_Col100;
         3311  +        sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
         3312  +      }else{
         3313  +        pIter->xSetOutputs = fts5IterSetOutputs_Col;
         3314  +      }
         3315  +    }
         3316  +  }
         3317  +}
         3318  +
         3319  +
         3320  +/*
         3321  +** Allocate a new Fts5Iter object.
         3322  +**
         3323  +** The new object will be used to iterate through data in structure pStruct.
         3324  +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
         3325  +** is zero or greater, data from the first nSegment segments on level iLevel
         3326  +** is merged.
         3327  +**
         3328  +** The iterator initially points to the first term/rowid entry in the 
         3329  +** iterated data.
         3330  +*/
         3331  +static void fts5MultiIterNew(
         3332  +  Fts5Index *p,                   /* FTS5 backend to iterate within */
         3333  +  Fts5Structure *pStruct,         /* Structure of specific index */
         3334  +  int flags,                      /* FTS5INDEX_QUERY_XXX flags */
         3335  +  Fts5Colset *pColset,            /* Colset to filter on (or NULL) */
         3336  +  const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
         3337  +  int iLevel,                     /* Level to iterate (-1 for all) */
         3338  +  int nSegment,                   /* Number of segments to merge (iLevel>=0) */
         3339  +  Fts5Iter **ppOut                /* New object */
         3340  +){
         3341  +  int nSeg = 0;                   /* Number of segment-iters in use */
         3342  +  int iIter = 0;                  /* */
         3343  +  int iSeg;                       /* Used to iterate through segments */
         3344  +  Fts5StructureLevel *pLvl;
         3345  +  Fts5Iter *pNew;
         3346  +
         3347  +  assert( (pTerm==0 && nTerm==0) || iLevel<0 );
         3348  +
         3349  +  /* Allocate space for the new multi-seg-iterator. */
         3350  +  if( p->rc==SQLITE_OK ){
         3351  +    if( iLevel<0 ){
         3352  +      assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
         3353  +      nSeg = pStruct->nSegment;
         3354  +      nSeg += (p->pHash ? 1 : 0);
         3355  +    }else{
         3356  +      nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
         3357  +    }
         3358  +  }
         3359  +  *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
         3360  +  if( pNew==0 ) return;
         3361  +  pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
         3362  +  pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY));
         3363  +  pNew->pStruct = pStruct;
         3364  +  pNew->pColset = pColset;
         3365  +  fts5StructureRef(pStruct);
         3366  +  if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){
         3367  +    fts5IterSetOutputCb(&p->rc, pNew);
         3368  +  }
         3369  +
         3370  +  /* Initialize each of the component segment iterators. */
         3371  +  if( p->rc==SQLITE_OK ){
         3372  +    if( iLevel<0 ){
         3373  +      Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
         3374  +      if( p->pHash ){
         3375  +        /* Add a segment iterator for the current contents of the hash table. */
         3376  +        Fts5SegIter *pIter = &pNew->aSeg[iIter++];
         3377  +        fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
         3378  +      }
         3379  +      for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
         3380  +        for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
         3381  +          Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
         3382  +          Fts5SegIter *pIter = &pNew->aSeg[iIter++];
         3383  +          if( pTerm==0 ){
         3384  +            fts5SegIterInit(p, pSeg, pIter);
         3385  +          }else{
         3386  +            fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
         3387  +          }
         3388  +        }
         3389  +      }
         3390  +    }else{
         3391  +      pLvl = &pStruct->aLevel[iLevel];
         3392  +      for(iSeg=nSeg-1; iSeg>=0; iSeg--){
         3393  +        fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
         3394  +      }
         3395  +    }
         3396  +    assert( iIter==nSeg );
         3397  +  }
         3398  +
         3399  +  /* If the above was successful, each component iterators now points 
         3400  +  ** to the first entry in its segment. In this case initialize the 
         3401  +  ** aFirst[] array. Or, if an error has occurred, free the iterator
         3402  +  ** object and set the output variable to NULL.  */
         3403  +  if( p->rc==SQLITE_OK ){
         3404  +    for(iIter=pNew->nSeg-1; iIter>0; iIter--){
         3405  +      int iEq;
         3406  +      if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){
         3407  +        Fts5SegIter *pSeg = &pNew->aSeg[iEq];
         3408  +        if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0);
         3409  +        fts5MultiIterAdvanced(p, pNew, iEq, iIter);
         3410  +      }
         3411  +    }
         3412  +    fts5MultiIterSetEof(pNew);
         3413  +    fts5AssertMultiIterSetup(p, pNew);
         3414  +
         3415  +    if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
         3416  +      fts5MultiIterNext(p, pNew, 0, 0);
         3417  +    }else if( pNew->base.bEof==0 ){
         3418  +      Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst];
         3419  +      pNew->xSetOutputs(pNew, pSeg);
         3420  +    }
         3421  +
         3422  +  }else{
         3423  +    fts5MultiIterFree(pNew);
         3424  +    *ppOut = 0;
         3425  +  }
         3426  +}
         3427  +
         3428  +/*
         3429  +** Create an Fts5Iter that iterates through the doclist provided
         3430  +** as the second argument.
         3431  +*/
         3432  +static void fts5MultiIterNew2(
         3433  +  Fts5Index *p,                   /* FTS5 backend to iterate within */
         3434  +  Fts5Data *pData,                /* Doclist to iterate through */
         3435  +  int bDesc,                      /* True for descending rowid order */
         3436  +  Fts5Iter **ppOut                /* New object */
         3437  +){
         3438  +  Fts5Iter *pNew;
         3439  +  pNew = fts5MultiIterAlloc(p, 2);
         3440  +  if( pNew ){
         3441  +    Fts5SegIter *pIter = &pNew->aSeg[1];
         3442  +
         3443  +    pIter->flags = FTS5_SEGITER_ONETERM;
         3444  +    if( pData->szLeaf>0 ){
         3445  +      pIter->pLeaf = pData;
         3446  +      pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
         3447  +      pIter->iEndofDoclist = pData->nn;
         3448  +      pNew->aFirst[1].iFirst = 1;
         3449  +      if( bDesc ){
         3450  +        pNew->bRev = 1;
         3451  +        pIter->flags |= FTS5_SEGITER_REVERSE;
         3452  +        fts5SegIterReverseInitPage(p, pIter);
         3453  +      }else{
         3454  +        fts5SegIterLoadNPos(p, pIter);
         3455  +      }
         3456  +      pData = 0;
         3457  +    }else{
         3458  +      pNew->base.bEof = 1;
         3459  +    }
         3460  +    fts5SegIterSetNext(p, pIter);
         3461  +
         3462  +    *ppOut = pNew;
         3463  +  }
         3464  +
         3465  +  fts5DataRelease(pData);
         3466  +}
         3467  +
         3468  +/*
         3469  +** Return true if the iterator is at EOF or if an error has occurred. 
         3470  +** False otherwise.
         3471  +*/
         3472  +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){
         3473  +  assert( p->rc 
         3474  +      || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof 
         3475  +  );
         3476  +  return (p->rc || pIter->base.bEof);
         3477  +}
         3478  +
         3479  +/*
         3480  +** Return the rowid of the entry that the iterator currently points
         3481  +** to. If the iterator points to EOF when this function is called the
         3482  +** results are undefined.
         3483  +*/
         3484  +static i64 fts5MultiIterRowid(Fts5Iter *pIter){
         3485  +  assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
         3486  +  return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
         3487  +}
         3488  +
         3489  +/*
         3490  +** Move the iterator to the next entry at or following iMatch.
         3491  +*/
         3492  +static void fts5MultiIterNextFrom(
         3493  +  Fts5Index *p, 
         3494  +  Fts5Iter *pIter, 
         3495  +  i64 iMatch
         3496  +){
         3497  +  while( 1 ){
         3498  +    i64 iRowid;
         3499  +    fts5MultiIterNext(p, pIter, 1, iMatch);
         3500  +    if( fts5MultiIterEof(p, pIter) ) break;
         3501  +    iRowid = fts5MultiIterRowid(pIter);
         3502  +    if( pIter->bRev==0 && iRowid>=iMatch ) break;
         3503  +    if( pIter->bRev!=0 && iRowid<=iMatch ) break;
         3504  +  }
         3505  +}
         3506  +
         3507  +/*
         3508  +** Return a pointer to a buffer containing the term associated with the 
         3509  +** entry that the iterator currently points to.
         3510  +*/
         3511  +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){
         3512  +  Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
         3513  +  *pn = p->term.n;
         3514  +  return p->term.p;
         3515  +}
  2898   3516   
  2899   3517   /*
  2900   3518   ** Allocate a new segment-id for the structure pStruct. The new segment
  2901   3519   ** id must be between 1 and 65335 inclusive, and must not be used by 
  2902   3520   ** any currently existing segment. If a free segment id cannot be found,
  2903   3521   ** SQLITE_FULL is returned.
  2904   3522   **
................................................................................
  2908   3526   static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
  2909   3527     int iSegid = 0;
  2910   3528   
  2911   3529     if( p->rc==SQLITE_OK ){
  2912   3530       if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
  2913   3531         p->rc = SQLITE_FULL;
  2914   3532       }else{
  2915         -      while( iSegid==0 ){
  2916         -        int iLvl, iSeg;
  2917         -        sqlite3_randomness(sizeof(u32), (void*)&iSegid);
  2918         -        iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
  2919         -        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
  2920         -          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  2921         -            if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
  2922         -              iSegid = 0;
  2923         -            }
         3533  +      /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following
         3534  +      ** array is 63 elements, or 252 bytes, in size.  */
         3535  +      u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32];
         3536  +      int iLvl, iSeg;
         3537  +      int i;
         3538  +      u32 mask;
         3539  +      memset(aUsed, 0, sizeof(aUsed));
         3540  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3541  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3542  +          int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
         3543  +          if( iId<=FTS5_MAX_SEGMENT ){
         3544  +            aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
  2924   3545             }
  2925   3546           }
  2926   3547         }
         3548  +
         3549  +      for(i=0; aUsed[i]==0xFFFFFFFF; i++);
         3550  +      mask = aUsed[i];
         3551  +      for(iSegid=0; mask & (1 << iSegid); iSegid++);
         3552  +      iSegid += 1 + i*32;
         3553  +
         3554  +#ifdef SQLITE_DEBUG
         3555  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3556  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3557  +          assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid );
         3558  +        }
         3559  +      }
         3560  +      assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT );
         3561  +
         3562  +      {
         3563  +        sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
         3564  +        if( p->rc==SQLITE_OK ){
         3565  +          u8 aBlob[2] = {0xff, 0xff};
         3566  +          sqlite3_bind_int(pIdxSelect, 1, iSegid);
         3567  +          sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
         3568  +          assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
         3569  +          p->rc = sqlite3_reset(pIdxSelect);
         3570  +        }
         3571  +      }
         3572  +#endif
  2927   3573       }
  2928   3574     }
  2929   3575   
  2930   3576     return iSegid;
  2931   3577   }
  2932   3578   
  2933   3579   /*
................................................................................
  2938   3584     if( p->pHash ){
  2939   3585       sqlite3Fts5HashClear(p->pHash);
  2940   3586       p->nPendingData = 0;
  2941   3587     }
  2942   3588   }
  2943   3589   
  2944   3590   /*
  2945         -** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
  2946         -** with buffer (nOld/pOld).
         3591  +** Return the size of the prefix, in bytes, that buffer 
         3592  +** (pNew/<length-unknown>) shares with buffer (pOld/nOld).
         3593  +**
         3594  +** Buffer (pNew/<length-unknown>) is guaranteed to be greater 
         3595  +** than buffer (pOld/nOld).
  2947   3596   */
  2948         -static int fts5PrefixCompress(
  2949         -  int nOld, const u8 *pOld,
  2950         -  int nNew, const u8 *pNew
  2951         -){
         3597  +static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){
  2952   3598     int i;
  2953         -  assert( fts5BlobCompare(pOld, nOld, pNew, nNew)<0 );
  2954   3599     for(i=0; i<nOld; i++){
  2955   3600       if( pOld[i]!=pNew[i] ) break;
  2956   3601     }
  2957   3602     return i;
  2958   3603   }
  2959   3604   
  2960   3605   static void fts5WriteDlidxClear(
................................................................................
  3165   3810     }
  3166   3811   }
  3167   3812   
  3168   3813   static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
  3169   3814     static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
  3170   3815     Fts5PageWriter *pPage = &pWriter->writer;
  3171   3816     i64 iRowid;
         3817  +
         3818  +static int nCall = 0;
         3819  +nCall++;
  3172   3820   
  3173   3821     assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
  3174   3822   
  3175   3823     /* Set the szLeaf header field. */
  3176   3824     assert( 0==fts5GetU16(&pPage->buf.p[2]) );
  3177   3825     fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
  3178   3826   
................................................................................
  3256   3904         ** Usually, the previous term is available in pPage->term. The exception
  3257   3905         ** is if this is the first term written in an incremental-merge step.
  3258   3906         ** In this case the previous term is not available, so just write a
  3259   3907         ** copy of (pTerm/nTerm) into the parent node. This is slightly
  3260   3908         ** inefficient, but still correct.  */
  3261   3909         int n = nTerm;
  3262   3910         if( pPage->term.n ){
  3263         -        n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
         3911  +        n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
  3264   3912         }
  3265   3913         fts5WriteBtreeTerm(p, pWriter, n, pTerm);
  3266   3914         pPage = &pWriter->writer;
  3267   3915       }
  3268   3916     }else{
  3269         -    nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
         3917  +    nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
  3270   3918       fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix);
  3271   3919     }
  3272   3920   
  3273   3921     /* Append the number of bytes of new data, then the term data itself
  3274   3922     ** to the page. */
  3275   3923     fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix);
  3276   3924     fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]);
................................................................................
  3288   3936   
  3289   3937   /*
  3290   3938   ** Append a rowid and position-list size field to the writers output. 
  3291   3939   */
  3292   3940   static void fts5WriteAppendRowid(
  3293   3941     Fts5Index *p, 
  3294   3942     Fts5SegWriter *pWriter,
  3295         -  i64 iRowid,
  3296         -  int nPos
         3943  +  i64 iRowid
  3297   3944   ){
  3298   3945     if( p->rc==SQLITE_OK ){
  3299   3946       Fts5PageWriter *pPage = &pWriter->writer;
  3300   3947   
  3301   3948       if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){
  3302   3949         fts5WriteFlushLeaf(p, pWriter);
  3303   3950       }
................................................................................
  3316   3963       }else{
  3317   3964         assert( p->rc || iRowid>pWriter->iPrevRowid );
  3318   3965         fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid);
  3319   3966       }
  3320   3967       pWriter->iPrevRowid = iRowid;
  3321   3968       pWriter->bFirstRowidInDoclist = 0;
  3322   3969       pWriter->bFirstRowidInPage = 0;
  3323         -
  3324         -    fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos);
  3325   3970     }
  3326   3971   }
  3327   3972   
  3328   3973   static void fts5WriteAppendPoslistData(
  3329   3974     Fts5Index *p, 
  3330   3975     Fts5SegWriter *pWriter, 
  3331   3976     const u8 *aData, 
................................................................................
  3368   4013     Fts5PageWriter *pLeaf = &pWriter->writer;
  3369   4014     if( p->rc==SQLITE_OK ){
  3370   4015       assert( pLeaf->pgno>=1 );
  3371   4016       if( pLeaf->buf.n>4 ){
  3372   4017         fts5WriteFlushLeaf(p, pWriter);
  3373   4018       }
  3374   4019       *pnLeaf = pLeaf->pgno-1;
  3375         -    fts5WriteFlushBtree(p, pWriter);
         4020  +    if( pLeaf->pgno>1 ){
         4021  +      fts5WriteFlushBtree(p, pWriter);
         4022  +    }
  3376   4023     }
  3377   4024     fts5BufferFree(&pLeaf->term);
  3378   4025     fts5BufferFree(&pLeaf->buf);
  3379   4026     fts5BufferFree(&pLeaf->pgidx);
  3380   4027     fts5BufferFree(&pWriter->btterm);
  3381   4028   
  3382   4029     for(i=0; i<pWriter->nDlidx; i++){
................................................................................
  3428   4075   }
  3429   4076   
  3430   4077   /*
  3431   4078   ** Iterator pIter was used to iterate through the input segments of on an
  3432   4079   ** incremental merge operation. This function is called if the incremental
  3433   4080   ** merge step has finished but the input has not been completely exhausted.
  3434   4081   */
  3435         -static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
         4082  +static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
  3436   4083     int i;
  3437   4084     Fts5Buffer buf;
  3438   4085     memset(&buf, 0, sizeof(Fts5Buffer));
  3439   4086     for(i=0; i<pIter->nSeg; i++){
  3440   4087       Fts5SegIter *pSeg = &pIter->aSeg[i];
  3441   4088       if( pSeg->pSeg==0 ){
  3442   4089         /* no-op */
................................................................................
  3506   4153     Fts5Structure **ppStruct,       /* IN/OUT: Stucture of index */
  3507   4154     int iLvl,                       /* Level to read input from */
  3508   4155     int *pnRem                      /* Write up to this many output leaves */
  3509   4156   ){
  3510   4157     Fts5Structure *pStruct = *ppStruct;
  3511   4158     Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
  3512   4159     Fts5StructureLevel *pLvlOut;
  3513         -  Fts5IndexIter *pIter = 0;       /* Iterator to read input data */
         4160  +  Fts5Iter *pIter = 0;       /* Iterator to read input data */
  3514   4161     int nRem = pnRem ? *pnRem : 0;  /* Output leaf pages left to write */
  3515   4162     int nInput;                     /* Number of input segments */
  3516   4163     Fts5SegWriter writer;           /* Writer object */
  3517   4164     Fts5StructureSegment *pSeg;     /* Output segment */
  3518   4165     Fts5Buffer term;
  3519   4166     int bOldest;                    /* True if the output segment is the oldest */
         4167  +  int eDetail = p->pConfig->eDetail;
         4168  +  const int flags = FTS5INDEX_QUERY_NOOUTPUT;
  3520   4169   
  3521   4170     assert( iLvl<pStruct->nLevel );
  3522   4171     assert( pLvl->nMerge<=pLvl->nSeg );
  3523   4172   
  3524   4173     memset(&writer, 0, sizeof(Fts5SegWriter));
  3525   4174     memset(&term, 0, sizeof(Fts5Buffer));
  3526   4175     if( pLvl->nMerge ){
................................................................................
  3557   4206   
  3558   4207       /* Read input from all segments in the input level */
  3559   4208       nInput = pLvl->nSeg;
  3560   4209     }
  3561   4210     bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
  3562   4211   
  3563   4212     assert( iLvl>=0 );
  3564         -  for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter);
         4213  +  for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter);
  3565   4214         fts5MultiIterEof(p, pIter)==0;
  3566   4215         fts5MultiIterNext(p, pIter, 0, 0)
  3567   4216     ){
  3568   4217       Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  3569   4218       int nPos;                     /* position-list size field value */
  3570   4219       int nTerm;
  3571   4220       const u8 *pTerm;
................................................................................
  3582   4231         /* This is a new term. Append a term to the output segment. */
  3583   4232         fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
  3584   4233         fts5BufferSet(&p->rc, &term, nTerm, pTerm);
  3585   4234       }
  3586   4235   
  3587   4236       /* Append the rowid to the output */
  3588   4237       /* WRITEPOSLISTSIZE */
  3589         -    nPos = pSegIter->nPos*2 + pSegIter->bDel;
  3590         -    fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos);
         4238  +    fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));
  3591   4239   
  3592         -    /* Append the position-list data to the output */
  3593         -    fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
         4240  +    if( eDetail==FTS5_DETAIL_NONE ){
         4241  +      if( pSegIter->bDel ){
         4242  +        fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0);
         4243  +        if( pSegIter->nPos>0 ){
         4244  +          fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0);
         4245  +        }
         4246  +      }
         4247  +    }else{
         4248  +      /* Append the position-list data to the output */
         4249  +      nPos = pSegIter->nPos*2 + pSegIter->bDel;
         4250  +      fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos);
         4251  +      fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
         4252  +    }
  3594   4253     }
  3595   4254   
  3596   4255     /* Flush the last leaf page to disk. Set the output segment b-tree height
  3597   4256     ** and last leaf page number at the same time.  */
  3598   4257     fts5WriteFinish(p, &writer, &pSeg->pgnoLast);
  3599   4258   
  3600   4259     if( fts5MultiIterEof(p, pIter) ){
................................................................................
  3619   4278       }
  3620   4279     }else{
  3621   4280       assert( pSeg->pgnoLast>0 );
  3622   4281       fts5TrimSegments(p, pIter);
  3623   4282       pLvl->nMerge = nInput;
  3624   4283     }
  3625   4284   
  3626         -  fts5MultiIterFree(p, pIter);
         4285  +  fts5MultiIterFree(pIter);
  3627   4286     fts5BufferFree(&term);
  3628   4287     if( pnRem ) *pnRem -= writer.nLeafWritten;
  3629   4288   }
  3630   4289   
  3631   4290   /*
  3632   4291   ** Do up to nPg pages of automerge work on the index.
         4292  +**
         4293  +** Return true if any changes were actually made, or false otherwise.
  3633   4294   */
  3634         -static void fts5IndexMerge(
         4295  +static int fts5IndexMerge(
  3635   4296     Fts5Index *p,                   /* FTS5 backend object */
  3636   4297     Fts5Structure **ppStruct,       /* IN/OUT: Current structure of index */
  3637         -  int nPg                         /* Pages of work to do */
         4298  +  int nPg,                        /* Pages of work to do */
         4299  +  int nMin                        /* Minimum number of segments to merge */
  3638   4300   ){
  3639   4301     int nRem = nPg;
         4302  +  int bRet = 0;
  3640   4303     Fts5Structure *pStruct = *ppStruct;
  3641   4304     while( nRem>0 && p->rc==SQLITE_OK ){
  3642   4305       int iLvl;                   /* To iterate through levels */
  3643   4306       int iBestLvl = 0;           /* Level offering the most input segments */
  3644   4307       int nBest = 0;              /* Number of input segments on best level */
  3645   4308   
  3646   4309       /* Set iBestLvl to the level to read input segments from. */
................................................................................
  3663   4326       /* If nBest is still 0, then the index must be empty. */
  3664   4327   #ifdef SQLITE_DEBUG
  3665   4328       for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
  3666   4329         assert( pStruct->aLevel[iLvl].nSeg==0 );
  3667   4330       }
  3668   4331   #endif
  3669   4332   
  3670         -    if( nBest<p->pConfig->nAutomerge 
  3671         -        && pStruct->aLevel[iBestLvl].nMerge==0 
  3672         -      ){
         4333  +    if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
  3673   4334         break;
  3674   4335       }
         4336  +    bRet = 1;
  3675   4337       fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
  3676   4338       if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
  3677   4339         fts5StructurePromote(p, iBestLvl+1, pStruct);
  3678   4340       }
  3679   4341     }
  3680   4342     *ppStruct = pStruct;
         4343  +  return bRet;
  3681   4344   }
  3682   4345   
  3683   4346   /*
  3684   4347   ** A total of nLeaf leaf pages of data has just been flushed to a level-0
  3685   4348   ** segment. This function updates the write-counter accordingly and, if
  3686   4349   ** necessary, performs incremental merge work.
  3687   4350   **
................................................................................
  3701   4364   
  3702   4365       /* Update the write-counter. While doing so, set nWork. */
  3703   4366       nWrite = pStruct->nWriteCounter;
  3704   4367       nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit));
  3705   4368       pStruct->nWriteCounter += nLeaf;
  3706   4369       nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);
  3707   4370   
  3708         -    fts5IndexMerge(p, ppStruct, nRem);
         4371  +    fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge);
  3709   4372     }
  3710   4373   }
  3711   4374   
  3712   4375   static void fts5IndexCrisismerge(
  3713   4376     Fts5Index *p,                   /* FTS5 backend object */
  3714   4377     Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
  3715   4378   ){
................................................................................
  3771   4434     int iSegid;
  3772   4435     int pgnoLast = 0;                 /* Last leaf page number in segment */
  3773   4436   
  3774   4437     /* Obtain a reference to the index structure and allocate a new segment-id
  3775   4438     ** for the new level-0 segment.  */
  3776   4439     pStruct = fts5StructureRead(p);
  3777   4440     iSegid = fts5AllocateSegid(p, pStruct);
         4441  +  fts5StructureInvalidate(p);
  3778   4442   
  3779   4443     if( iSegid ){
  3780   4444       const int pgsz = p->pConfig->pgsz;
  3781         -
         4445  +    int eDetail = p->pConfig->eDetail;
  3782   4446       Fts5StructureSegment *pSeg;   /* New segment within pStruct */
  3783   4447       Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
  3784   4448       Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
  3785   4449   
  3786   4450       Fts5SegWriter writer;
  3787   4451       fts5WriteInit(p, &writer, iSegid);
  3788   4452   
................................................................................
  3817   4481           i64 iDelta = 0;
  3818   4482           int iOff = 0;
  3819   4483   
  3820   4484           /* The entire doclist will not fit on this leaf. The following 
  3821   4485           ** loop iterates through the poslists that make up the current 
  3822   4486           ** doclist.  */
  3823   4487           while( p->rc==SQLITE_OK && iOff<nDoclist ){
  3824         -          int nPos;
  3825         -          int nCopy;
  3826         -          int bDummy;
  3827   4488             iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta);
  3828         -          nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
  3829         -          nCopy += nPos;
  3830   4489             iRowid += iDelta;
  3831   4490             
  3832   4491             if( writer.bFirstRowidInPage ){
  3833   4492               fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
  3834   4493               pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
  3835   4494               writer.bFirstRowidInPage = 0;
  3836   4495               fts5WriteDlidxAppend(p, &writer, iRowid);
  3837   4496             }else{
  3838   4497               pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
  3839   4498             }
  3840   4499             assert( pBuf->n<=pBuf->nSpace );
  3841   4500   
  3842         -          if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
  3843         -            /* The entire poslist will fit on the current leaf. So copy
  3844         -            ** it in one go. */
  3845         -            fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
         4501  +          if( eDetail==FTS5_DETAIL_NONE ){
         4502  +            if( iOff<nDoclist && pDoclist[iOff]==0 ){
         4503  +              pBuf->p[pBuf->n++] = 0;
         4504  +              iOff++;
         4505  +              if( iOff<nDoclist && pDoclist[iOff]==0 ){
         4506  +                pBuf->p[pBuf->n++] = 0;
         4507  +                iOff++;
         4508  +              }
         4509  +            }
         4510  +            if( (pBuf->n + pPgidx->n)>=pgsz ){
         4511  +              fts5WriteFlushLeaf(p, &writer);
         4512  +            }
  3846   4513             }else{
  3847         -            /* The entire poslist will not fit on this leaf. So it needs
  3848         -            ** to be broken into sections. The only qualification being
  3849         -            ** that each varint must be stored contiguously.  */
  3850         -            const u8 *pPoslist = &pDoclist[iOff];
  3851         -            int iPos = 0;
  3852         -            while( p->rc==SQLITE_OK ){
  3853         -              int nSpace = pgsz - pBuf->n - pPgidx->n;
  3854         -              int n = 0;
  3855         -              if( (nCopy - iPos)<=nSpace ){
  3856         -                n = nCopy - iPos;
  3857         -              }else{
  3858         -                n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
         4514  +            int bDummy;
         4515  +            int nPos;
         4516  +            int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
         4517  +            nCopy += nPos;
         4518  +            if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
         4519  +              /* The entire poslist will fit on the current leaf. So copy
         4520  +              ** it in one go. */
         4521  +              fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
         4522  +            }else{
         4523  +              /* The entire poslist will not fit on this leaf. So it needs
         4524  +              ** to be broken into sections. The only qualification being
         4525  +              ** that each varint must be stored contiguously.  */
         4526  +              const u8 *pPoslist = &pDoclist[iOff];
         4527  +              int iPos = 0;
         4528  +              while( p->rc==SQLITE_OK ){
         4529  +                int nSpace = pgsz - pBuf->n - pPgidx->n;
         4530  +                int n = 0;
         4531  +                if( (nCopy - iPos)<=nSpace ){
         4532  +                  n = nCopy - iPos;
         4533  +                }else{
         4534  +                  n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
         4535  +                }
         4536  +                assert( n>0 );
         4537  +                fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
         4538  +                iPos += n;
         4539  +                if( (pBuf->n + pPgidx->n)>=pgsz ){
         4540  +                  fts5WriteFlushLeaf(p, &writer);
         4541  +                }
         4542  +                if( iPos>=nCopy ) break;
  3859   4543                 }
  3860         -              assert( n>0 );
  3861         -              fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
  3862         -              iPos += n;
  3863         -              if( (pBuf->n + pPgidx->n)>=pgsz ){
  3864         -                fts5WriteFlushLeaf(p, &writer);
  3865         -              }
  3866         -              if( iPos>=nCopy ) break;
  3867   4544               }
         4545  +            iOff += nCopy;
  3868   4546             }
  3869         -          iOff += nCopy;
  3870   4547           }
  3871   4548         }
  3872   4549   
  3873   4550         /* TODO2: Doclist terminator written here. */
  3874   4551         /* pBuf->p[pBuf->n++] = '\0'; */
  3875   4552         assert( pBuf->n<=pBuf->nSpace );
  3876   4553         sqlite3Fts5HashScanNext(pHash);
................................................................................
  3908   4585     if( p->nPendingData ){
  3909   4586       assert( p->pHash );
  3910   4587       p->nPendingData = 0;
  3911   4588       fts5FlushOneHash(p);
  3912   4589     }
  3913   4590   }
  3914   4591   
  3915         -
  3916         -int sqlite3Fts5IndexOptimize(Fts5Index *p){
  3917         -  Fts5Structure *pStruct;
         4592  +static Fts5Structure *fts5IndexOptimizeStruct(
         4593  +  Fts5Index *p, 
         4594  +  Fts5Structure *pStruct
         4595  +){
  3918   4596     Fts5Structure *pNew = 0;
  3919         -  int nSeg = 0;
  3920         -
  3921         -  assert( p->rc==SQLITE_OK );
  3922         -  fts5IndexFlush(p);
  3923         -  pStruct = fts5StructureRead(p);
  3924         -
  3925         -  if( pStruct ){
  3926         -    assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  3927         -    nSeg = pStruct->nSegment;
  3928         -    if( nSeg>1 ){
  3929         -      int nByte = sizeof(Fts5Structure);
  3930         -      nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
  3931         -      pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
  3932         -    }
  3933         -  }
         4597  +  int nByte = sizeof(Fts5Structure);
         4598  +  int nSeg = pStruct->nSegment;
         4599  +  int i;
         4600  +
         4601  +  /* Figure out if this structure requires optimization. A structure does
         4602  +  ** not require optimization if either:
         4603  +  **
         4604  +  **  + it consists of fewer than two segments, or 
         4605  +  **  + all segments are on the same level, or
         4606  +  **  + all segments except one are currently inputs to a merge operation.
         4607  +  **
         4608  +  ** In the first case, return NULL. In the second, increment the ref-count
         4609  +  ** on *pStruct and return a copy of the pointer to it.
         4610  +  */
         4611  +  if( nSeg<2 ) return 0;
         4612  +  for(i=0; i<pStruct->nLevel; i++){
         4613  +    int nThis = pStruct->aLevel[i].nSeg;
         4614  +    if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){
         4615  +      fts5StructureRef(pStruct);
         4616  +      return pStruct;
         4617  +    }
         4618  +    assert( pStruct->aLevel[i].nMerge<=nThis );
         4619  +  }
         4620  +
         4621  +  nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
         4622  +  pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
         4623  +
  3934   4624     if( pNew ){
  3935   4625       Fts5StructureLevel *pLvl;
  3936         -    int nByte = nSeg * sizeof(Fts5StructureSegment);
         4626  +    nByte = nSeg * sizeof(Fts5StructureSegment);
  3937   4627       pNew->nLevel = pStruct->nLevel+1;
  3938   4628       pNew->nRef = 1;
  3939   4629       pNew->nWriteCounter = pStruct->nWriteCounter;
  3940   4630       pLvl = &pNew->aLevel[pStruct->nLevel];
  3941   4631       pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
  3942   4632       if( pLvl->aSeg ){
  3943   4633         int iLvl, iSeg;
  3944   4634         int iSegOut = 0;
  3945         -      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         4635  +      /* Iterate through all segments, from oldest to newest. Add them to
         4636  +      ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest
         4637  +      ** segment in the data structure.  */
         4638  +      for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
  3946   4639           for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  3947   4640             pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
  3948   4641             iSegOut++;
  3949   4642           }
  3950   4643         }
  3951   4644         pNew->nSegment = pLvl->nSeg = nSeg;
  3952   4645       }else{
  3953   4646         sqlite3_free(pNew);
  3954   4647         pNew = 0;
  3955   4648       }
  3956   4649     }
  3957   4650   
         4651  +  return pNew;
         4652  +}
         4653  +
         4654  +int sqlite3Fts5IndexOptimize(Fts5Index *p){
         4655  +  Fts5Structure *pStruct;
         4656  +  Fts5Structure *pNew = 0;
         4657  +
         4658  +  assert( p->rc==SQLITE_OK );
         4659  +  fts5IndexFlush(p);
         4660  +  pStruct = fts5StructureRead(p);
         4661  +  fts5StructureInvalidate(p);
         4662  +
         4663  +  if( pStruct ){
         4664  +    pNew = fts5IndexOptimizeStruct(p, pStruct);
         4665  +  }
         4666  +  fts5StructureRelease(pStruct);
         4667  +
         4668  +  assert( pNew==0 || pNew->nSegment>0 );
  3958   4669     if( pNew ){
  3959         -    int iLvl = pNew->nLevel-1;
         4670  +    int iLvl;
         4671  +    for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){}
  3960   4672       while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
  3961   4673         int nRem = FTS5_OPT_WORK_UNIT;
  3962   4674         fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
  3963   4675       }
  3964   4676   
  3965   4677       fts5StructureWrite(p, pNew);
  3966   4678       fts5StructureRelease(pNew);
  3967   4679     }
  3968   4680   
  3969         -  fts5StructureRelease(pStruct);
  3970   4681     return fts5IndexReturn(p); 
  3971   4682   }
  3972   4683   
         4684  +/*
         4685  +** This is called to implement the special "VALUES('merge', $nMerge)"
         4686  +** INSERT command.
         4687  +*/
  3973   4688   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  3974         -  Fts5Structure *pStruct;
  3975         -
  3976         -  pStruct = fts5StructureRead(p);
  3977         -  if( pStruct && pStruct->nLevel ){
  3978         -    fts5IndexMerge(p, &pStruct, nMerge);
  3979         -    fts5StructureWrite(p, pStruct);
         4689  +  Fts5Structure *pStruct = fts5StructureRead(p);
         4690  +  if( pStruct ){
         4691  +    int nMin = p->pConfig->nUsermerge;
         4692  +    fts5StructureInvalidate(p);
         4693  +    if( nMerge<0 ){
         4694  +      Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
         4695  +      fts5StructureRelease(pStruct);
         4696  +      pStruct = pNew;
         4697  +      nMin = 2;
         4698  +      nMerge = nMerge*-1;
         4699  +    }
         4700  +    if( pStruct && pStruct->nLevel ){
         4701  +      if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){
         4702  +        fts5StructureWrite(p, pStruct);
         4703  +      }
         4704  +    }
         4705  +    fts5StructureRelease(pStruct);
  3980   4706     }
  3981         -  fts5StructureRelease(pStruct);
  3982         -
  3983   4707     return fts5IndexReturn(p);
  3984   4708   }
  3985   4709   
  3986         -static void fts5PoslistCallback(
  3987         -  Fts5Index *p, 
  3988         -  void *pContext, 
  3989         -  const u8 *pChunk, int nChunk
  3990         -){
  3991         -  assert_nc( nChunk>=0 );
  3992         -  if( nChunk>0 ){
  3993         -    fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk);
  3994         -  }
  3995         -}
  3996         -
  3997         -typedef struct PoslistCallbackCtx PoslistCallbackCtx;
  3998         -struct PoslistCallbackCtx {
  3999         -  Fts5Buffer *pBuf;               /* Append to this buffer */
  4000         -  Fts5Colset *pColset;            /* Restrict matches to this column */
  4001         -  int eState;                     /* See above */
  4002         -};
  4003         -
  4004         -/*
  4005         -** TODO: Make this more efficient!
  4006         -*/
  4007         -static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
  4008         -  int i;
  4009         -  for(i=0; i<pColset->nCol; i++){
  4010         -    if( pColset->aiCol[i]==iCol ) return 1;
  4011         -  }
  4012         -  return 0;
  4013         -}
  4014         -
  4015         -static void fts5PoslistFilterCallback(
  4016         -  Fts5Index *p, 
  4017         -  void *pContext, 
  4018         -  const u8 *pChunk, int nChunk
  4019         -){
  4020         -  PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
  4021         -  assert_nc( nChunk>=0 );
  4022         -  if( nChunk>0 ){
  4023         -    /* Search through to find the first varint with value 1. This is the
  4024         -    ** start of the next columns hits. */
  4025         -    int i = 0;
  4026         -    int iStart = 0;
  4027         -
  4028         -    if( pCtx->eState==2 ){
  4029         -      int iCol;
  4030         -      fts5FastGetVarint32(pChunk, i, iCol);
  4031         -      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
  4032         -        pCtx->eState = 1;
  4033         -        fts5BufferSafeAppendVarint(pCtx->pBuf, 1);
  4034         -      }else{
  4035         -        pCtx->eState = 0;
  4036         -      }
  4037         -    }
  4038         -
  4039         -    do {
  4040         -      while( i<nChunk && pChunk[i]!=0x01 ){
  4041         -        while( pChunk[i] & 0x80 ) i++;
  4042         -        i++;
  4043         -      }
  4044         -      if( pCtx->eState ){
  4045         -        fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
  4046         -      }
  4047         -      if( i<nChunk ){
  4048         -        int iCol;
  4049         -        iStart = i;
  4050         -        i++;
  4051         -        if( i>=nChunk ){
  4052         -          pCtx->eState = 2;
  4053         -        }else{
  4054         -          fts5FastGetVarint32(pChunk, i, iCol);
  4055         -          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
  4056         -          if( pCtx->eState ){
  4057         -            fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart);
  4058         -            iStart = i;
  4059         -          }
  4060         -        }
  4061         -      }
  4062         -    }while( i<nChunk );
  4063         -  }
  4064         -}
  4065         -
  4066         -/*
  4067         -** Iterator pIter currently points to a valid entry (not EOF). This
  4068         -** function appends the position list data for the current entry to
  4069         -** buffer pBuf. It does not make a copy of the position-list size
  4070         -** field.
  4071         -*/
  4072         -static void fts5SegiterPoslist(
         4710  +static void fts5AppendRowid(
         4711  +  Fts5Index *p,
         4712  +  i64 iDelta,
         4713  +  Fts5Iter *pUnused,
         4714  +  Fts5Buffer *pBuf
         4715  +){
         4716  +  UNUSED_PARAM(pUnused);
         4717  +  fts5BufferAppendVarint(&p->rc, pBuf, iDelta);
         4718  +}
         4719  +
         4720  +static void fts5AppendPoslist(
  4073   4721     Fts5Index *p,
  4074         -  Fts5SegIter *pSeg,
  4075         -  Fts5Colset *pColset,
         4722  +  i64 iDelta,
         4723  +  Fts5Iter *pMulti,
  4076   4724     Fts5Buffer *pBuf
  4077   4725   ){
  4078         -  if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
  4079         -    if( pColset==0 ){
  4080         -      fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
  4081         -    }else{
  4082         -      PoslistCallbackCtx sCtx;
  4083         -      sCtx.pBuf = pBuf;
  4084         -      sCtx.pColset = pColset;
  4085         -      sCtx.eState = fts5IndexColsetTest(pColset, 0);
  4086         -      assert( sCtx.eState==0 || sCtx.eState==1 );
  4087         -      fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
  4088         -    }
  4089         -  }
  4090         -}
  4091         -
  4092         -/*
  4093         -** IN/OUT parameter (*pa) points to a position list n bytes in size. If
  4094         -** the position list contains entries for column iCol, then (*pa) is set
  4095         -** to point to the sub-position-list for that column and the number of
  4096         -** bytes in it returned. Or, if the argument position list does not
  4097         -** contain any entries for column iCol, return 0.
  4098         -*/
  4099         -static int fts5IndexExtractCol(
  4100         -  const u8 **pa,                  /* IN/OUT: Pointer to poslist */
  4101         -  int n,                          /* IN: Size of poslist in bytes */
  4102         -  int iCol                        /* Column to extract from poslist */
  4103         -){
  4104         -  int iCurrent = 0;               /* Anything before the first 0x01 is col 0 */
  4105         -  const u8 *p = *pa;
  4106         -  const u8 *pEnd = &p[n];         /* One byte past end of position list */
  4107         -  u8 prev = 0;
  4108         -
  4109         -  while( iCol>iCurrent ){
  4110         -    /* Advance pointer p until it points to pEnd or an 0x01 byte that is
  4111         -    ** not part of a varint */
  4112         -    while( (prev & 0x80) || *p!=0x01 ){
  4113         -      prev = *p++;
  4114         -      if( p==pEnd ) return 0;
  4115         -    }
  4116         -    *pa = p++;
  4117         -    p += fts5GetVarint32(p, iCurrent);
  4118         -  }
  4119         -  if( iCol!=iCurrent ) return 0;
  4120         -
  4121         -  /* Advance pointer p until it points to pEnd or an 0x01 byte that is
  4122         -  ** not part of a varint */
  4123         -  assert( (prev & 0x80)==0 );
  4124         -  while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
  4125         -    prev = *p++;
  4126         -  }
  4127         -  return p - (*pa);
  4128         -}
  4129         -
  4130         -
  4131         -/*
  4132         -** Iterator pMulti currently points to a valid entry (not EOF). This
  4133         -** function appends the following to buffer pBuf:
  4134         -**
  4135         -**   * The varint iDelta, and
  4136         -**   * the position list that currently points to, including the size field.
  4137         -**
  4138         -** If argument pColset is NULL, then the position list is filtered according
  4139         -** to pColset before being appended to the buffer. If this means there are
  4140         -** no entries in the position list, nothing is appended to the buffer (not
  4141         -** even iDelta).
  4142         -**
  4143         -** If an error occurs, an error code is left in p->rc. 
  4144         -*/
  4145         -static int fts5AppendPoslist(
  4146         -  Fts5Index *p,
  4147         -  i64 iDelta,
  4148         -  Fts5IndexIter *pMulti,
  4149         -  Fts5Colset *pColset,
  4150         -  Fts5Buffer *pBuf
  4151         -){
  4152         -  if( p->rc==SQLITE_OK ){
  4153         -    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  4154         -    assert( fts5MultiIterEof(p, pMulti)==0 );
  4155         -    assert( pSeg->nPos>0 );
  4156         -    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
  4157         -
  4158         -      if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  4159         -       && (pColset==0 || pColset->nCol==1)
  4160         -      ){
  4161         -        const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4162         -        int nPos;
  4163         -        if( pColset ){
  4164         -          nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
  4165         -          if( nPos==0 ) return 1;
  4166         -        }else{
  4167         -          nPos = pSeg->nPos;
  4168         -        }
  4169         -        assert( nPos>0 );
  4170         -        fts5BufferSafeAppendVarint(pBuf, iDelta);
  4171         -        fts5BufferSafeAppendVarint(pBuf, nPos*2);
  4172         -        fts5BufferSafeAppendBlob(pBuf, pPos, nPos);
  4173         -      }else{
  4174         -        int iSv1;
  4175         -        int iSv2;
  4176         -        int iData;
  4177         -
  4178         -        /* Append iDelta */
  4179         -        iSv1 = pBuf->n;
  4180         -        fts5BufferSafeAppendVarint(pBuf, iDelta);
  4181         -
  4182         -        /* WRITEPOSLISTSIZE */
  4183         -        iSv2 = pBuf->n;
  4184         -        fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
  4185         -        iData = pBuf->n;
  4186         -
  4187         -        fts5SegiterPoslist(p, pSeg, pColset, pBuf);
  4188         -
  4189         -        if( pColset ){
  4190         -          int nActual = pBuf->n - iData;
  4191         -          if( nActual!=pSeg->nPos ){
  4192         -            if( nActual==0 ){
  4193         -              pBuf->n = iSv1;
  4194         -              return 1;
  4195         -            }else{
  4196         -              int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
  4197         -              while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
  4198         -              sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
  4199         -            }
  4200         -          }
  4201         -        }
  4202         -      }
  4203         -
  4204         -    }
  4205         -  }
  4206         -
  4207         -  return 0;
  4208         -}
         4726  +  int nData = pMulti->base.nData;
         4727  +  assert( nData>0 );
         4728  +  if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){
         4729  +    fts5BufferSafeAppendVarint(pBuf, iDelta);
         4730  +    fts5BufferSafeAppendVarint(pBuf, nData*2);
         4731  +    fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData);
         4732  +  }
         4733  +}
         4734  +
  4209   4735   
  4210   4736   static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  4211   4737     u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
  4212   4738   
  4213   4739     assert( pIter->aPoslist );
  4214   4740     if( p>=pIter->aEof ){
  4215   4741       pIter->aPoslist = 0;
................................................................................
  4262   4788   #endif
  4263   4789   
  4264   4790   #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) {       \
  4265   4791     assert( (pBuf)->n!=0 || (iLastRowid)==0 );                   \
  4266   4792     fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
  4267   4793     (iLastRowid) = (iRowid);                                     \
  4268   4794   }
         4795  +
         4796  +/*
         4797  +** Swap the contents of buffer *p1 with that of *p2.
         4798  +*/
         4799  +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
         4800  +  Fts5Buffer tmp = *p1;
         4801  +  *p1 = *p2;
         4802  +  *p2 = tmp;
         4803  +}
         4804  +
         4805  +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){
         4806  +  int i = *piOff;
         4807  +  if( i>=pBuf->n ){
         4808  +    *piOff = -1;
         4809  +  }else{
         4810  +    u64 iVal;
         4811  +    *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal);
         4812  +    *piRowid += iVal;
         4813  +  }
         4814  +}
         4815  +
         4816  +/*
         4817  +** This is the equivalent of fts5MergePrefixLists() for detail=none mode.
         4818  +** In this case the buffers consist of a delta-encoded list of rowids only.
         4819  +*/
         4820  +static void fts5MergeRowidLists(
         4821  +  Fts5Index *p,                   /* FTS5 backend object */
         4822  +  Fts5Buffer *p1,                 /* First list to merge */