/ Check-in [4c69e827]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest trunk changes into the apple-osx branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1:4c69e827f4d65e597e125e0d6691b0c8fb5588d7
User & Date: drh 2011-06-20 11:57:58
Context
2011-06-21
01:30
Merge the latest trunk changes into the apple-osx branch. check-in: 76005fdc user: drh tags: apple-osx
2011-06-20
11:57
Merge the latest trunk changes into the apple-osx branch. check-in: 4c69e827 user: drh tags: apple-osx
11:15
Fix a problem where FTS test code was not being included in the testfixture build when SQLITE_ENABLE_FTS4 was defined. check-in: e539d08a user: dan tags: trunk
2011-06-02
13:07
Merge the latest trunk changes, including the read-only shared memory enhancement, into the apple-osx branch. check-in: ce5f95de user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   378    378     $(TOP)/src/test_syscall.c \
   379    379     $(TOP)/src/test_stat.c \
   380    380     $(TOP)/src/test_tclvar.c \
   381    381     $(TOP)/src/test_thread.c \
   382    382     $(TOP)/src/test_vfs.c \
   383    383     $(TOP)/src/test_wholenumber.c \
   384    384     $(TOP)/src/test_wsd.c       \
   385         -  $(TOP)/ext/fts3/fts3_term.c 
          385  +  $(TOP)/ext/fts3/fts3_term.c \
          386  +  $(TOP)/ext/fts3/fts3_test.c 
   386    387   
   387    388   # Source code to the library files needed by the test fixture
   388    389   #
   389    390   TESTSRC2 = \
   390    391     $(TOP)/src/attach.c \
   391    392     $(TOP)/src/backup.c \
   392    393     $(TOP)/src/bitvec.c \

Added Makefile.msc.

            1  +#
            2  +# nmake Makefile for SQLite
            3  +#
            4  +
            5  +# The toplevel directory of the source tree.  This is the directory
            6  +# that contains this "Makefile.msc".
            7  +#
            8  +TOP = .
            9  +
           10  +# Set this non-0 to create and use the SQLite amalgamation file.
           11  +#
           12  +USE_AMALGAMATION = 1
           13  +
           14  +# Version numbers and release number for the SQLite being compiled.
           15  +#
           16  +VERSION = 3.7
           17  +VERSION_NUMBER = 3007007
           18  +RELEASE = 3.7.7
           19  +
           20  +# C Compiler and options for use in building executables that
           21  +# will run on the platform that is doing the build.
           22  +#
           23  +BCC = cl.exe -O2
           24  +
           25  +# C Compile and options for use in building executables that
           26  +# will run on the target platform.  (BCC and TCC are usually the
           27  +# same unless your are cross-compiling.)
           28  +#
           29  +TCC = cl.exe -W3 -O2 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src
           30  +
           31  +# The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in 
           32  +# any extension header files by default.  For non-amalgamation
           33  +# builds, we need to make sure the compiler can find these.
           34  +#
           35  +!IF $(USE_AMALGAMATION)==0
           36  +TCC = $(TCC) -I$(TOP)\ext\fts3
           37  +TCC = $(TCC) -I$(TOP)\ext\rtree
           38  +!ENDIF
           39  +
           40  +# Define -DNDEBUG to compile without debugging (i.e., for production usage)
           41  +# Omitting the define will cause extra debugging code to be inserted and
           42  +# includes extra comments when "EXPLAIN stmt" is used.
           43  +#
           44  +TCC = $(TCC) -DNDEBUG
           45  +
           46  +# The library that programs using TCL must link against.
           47  +#
           48  +LIBTCL = tcl85.lib
           49  +TCLINCDIR = c:\tcl\include
           50  +TCLLIBDIR = c:\tcl\lib
           51  +
           52  +# This is the command to use for tclsh - normally just "tclsh", but we may
           53  +# know the specific version we want to use
           54  +#
           55  +TCLSH_CMD = tclsh85
           56  +
           57  +# Compiler options needed for programs that use the readline() library.
           58  +#
           59  +READLINE_FLAGS = -DHAVE_READLINE=0
           60  +
           61  +# The library that programs using readline() must link against.
           62  +#
           63  +LIBREADLINE =
           64  +
           65  +# Should the database engine be compiled threadsafe
           66  +#
           67  +TCC = $(TCC) -DSQLITE_THREADSAFE=1
           68  +
           69  +# Do threads override each others locks by default (1), or do we test (-1)
           70  +#
           71  +TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
           72  +
           73  +# Any target libraries which libsqlite must be linked against
           74  +#
           75  +TLIBS =
           76  +
           77  +# Flags controlling use of the in memory btree implementation
           78  +#
           79  +# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to
           80  +# default to file, 2 to default to memory, and 3 to force temporary
           81  +# tables to always be in memory.
           82  +#
           83  +TCC = $(TCC) -DSQLITE_TEMP_STORE=1
           84  +
           85  +# Enable/disable loadable extensions, and other optional features
           86  +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
           87  +# The same set of OMIT and ENABLE flags should be passed to the
           88  +# LEMON parser generator and the mkkeywordhash tool as well.
           89  +
           90  +# BEGIN standard options
           91  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
           92  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
           93  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
           94  +# END standard options
           95  +
           96  +# BEGIN required Windows option
           97  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100
           98  +# END required Windows option
           99  +
          100  +TCC = $(TCC) $(OPT_FEATURE_FLAGS)
          101  +
          102  +# Add in any optional parameters specified on the make commane line
          103  +# ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
          104  +TCC = $(TCC) $(OPTS)
          105  +
          106  +# libtool compile/link
          107  +LTCOMPILE = $(TCC) -Fo$@
          108  +LTLINK = $(TCC) -Fe$@
          109  +LTLIB = lib.exe
          110  +
          111  +# nawk compatible awk.
          112  +NAWK = .\gawk.exe
          113  +
          114  +# You should not have to change anything below this line
          115  +###############################################################################
          116  +
          117  +# Object files for the SQLite library (non-amalgamation).
          118  +#
          119  +LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          120  +         backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
          121  +         callback.lo complete.lo ctime.lo date.lo delete.lo \
          122  +         expr.lo fault.lo fkey.lo \
          123  +         fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo fts3_porter.lo \
          124  +         fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo fts3_write.lo \
          125  +         func.lo global.lo hash.lo \
          126  +         icu.lo insert.lo journal.lo legacy.lo loadext.lo \
          127  +         main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
          128  +         memjournal.lo \
          129  +         mutex.lo mutex_noop.lo mutex_os2.lo mutex_unix.lo mutex_w32.lo \
          130  +         notify.lo opcodes.lo os.lo os_os2.lo os_unix.lo os_win.lo \
          131  +         pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
          132  +         random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
          133  +         table.lo tokenize.lo trigger.lo \
          134  +         update.lo util.lo vacuum.lo \
          135  +         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbetrace.lo \
          136  +         wal.lo walker.lo where.lo utf.lo vtab.lo
          137  +
          138  +# Object files for the amalgamation.
          139  +#
          140  +LIBOBJS1 = sqlite3.lo
          141  +
          142  +# Determine the real value of LIBOBJ based on the 'configure' script
          143  +#
          144  +!IF $(USE_AMALGAMATION)==0
          145  +LIBOBJ = $(LIBOBJS0)
          146  +!ELSE
          147  +LIBOBJ = $(LIBOBJS1)
          148  +!ENDIF
          149  +
          150  +# All of the source code files.
          151  +#
          152  +SRC = \
          153  +  $(TOP)\src\alter.c \
          154  +  $(TOP)\src\analyze.c \
          155  +  $(TOP)\src\attach.c \
          156  +  $(TOP)\src\auth.c \
          157  +  $(TOP)\src\backup.c \
          158  +  $(TOP)\src\bitvec.c \
          159  +  $(TOP)\src\btmutex.c \
          160  +  $(TOP)\src\btree.c \
          161  +  $(TOP)\src\btree.h \
          162  +  $(TOP)\src\btreeInt.h \
          163  +  $(TOP)\src\build.c \
          164  +  $(TOP)\src\callback.c \
          165  +  $(TOP)\src\complete.c \
          166  +  $(TOP)\src\ctime.c \
          167  +  $(TOP)\src\date.c \
          168  +  $(TOP)\src\delete.c \
          169  +  $(TOP)\src\expr.c \
          170  +  $(TOP)\src\fault.c \
          171  +  $(TOP)\src\fkey.c \
          172  +  $(TOP)\src\func.c \
          173  +  $(TOP)\src\global.c \
          174  +  $(TOP)\src\hash.c \
          175  +  $(TOP)\src\hash.h \
          176  +  $(TOP)\src\hwtime.h \
          177  +  $(TOP)\src\insert.c \
          178  +  $(TOP)\src\journal.c \
          179  +  $(TOP)\src\legacy.c \
          180  +  $(TOP)\src\loadext.c \
          181  +  $(TOP)\src\main.c \
          182  +  $(TOP)\src\malloc.c \
          183  +  $(TOP)\src\mem0.c \
          184  +  $(TOP)\src\mem1.c \
          185  +  $(TOP)\src\mem2.c \
          186  +  $(TOP)\src\mem3.c \
          187  +  $(TOP)\src\mem5.c \
          188  +  $(TOP)\src\memjournal.c \
          189  +  $(TOP)\src\mutex.c \
          190  +  $(TOP)\src\mutex.h \
          191  +  $(TOP)\src\mutex_noop.c \
          192  +  $(TOP)\src\mutex_os2.c \
          193  +  $(TOP)\src\mutex_unix.c \
          194  +  $(TOP)\src\mutex_w32.c \
          195  +  $(TOP)\src\notify.c \
          196  +  $(TOP)\src\os.c \
          197  +  $(TOP)\src\os.h \
          198  +  $(TOP)\src\os_common.h \
          199  +  $(TOP)\src\os_os2.c \
          200  +  $(TOP)\src\os_unix.c \
          201  +  $(TOP)\src\os_win.c \
          202  +  $(TOP)\src\pager.c \
          203  +  $(TOP)\src\pager.h \
          204  +  $(TOP)\src\parse.y \
          205  +  $(TOP)\src\pcache.c \
          206  +  $(TOP)\src\pcache.h \
          207  +  $(TOP)\src\pcache1.c \
          208  +  $(TOP)\src\pragma.c \
          209  +  $(TOP)\src\prepare.c \
          210  +  $(TOP)\src\printf.c \
          211  +  $(TOP)\src\random.c \
          212  +  $(TOP)\src\resolve.c \
          213  +  $(TOP)\src\rowset.c \
          214  +  $(TOP)\src\select.c \
          215  +  $(TOP)\src\status.c \
          216  +  $(TOP)\src\shell.c \
          217  +  $(TOP)\src\sqlite.h.in \
          218  +  $(TOP)\src\sqlite3ext.h \
          219  +  $(TOP)\src\sqliteInt.h \
          220  +  $(TOP)\src\sqliteLimit.h \
          221  +  $(TOP)\src\table.c \
          222  +  $(TOP)\src\tclsqlite.c \
          223  +  $(TOP)\src\tokenize.c \
          224  +  $(TOP)\src\trigger.c \
          225  +  $(TOP)\src\utf.c \
          226  +  $(TOP)\src\update.c \
          227  +  $(TOP)\src\util.c \
          228  +  $(TOP)\src\vacuum.c \
          229  +  $(TOP)\src\vdbe.c \
          230  +  $(TOP)\src\vdbe.h \
          231  +  $(TOP)\src\vdbeapi.c \
          232  +  $(TOP)\src\vdbeaux.c \
          233  +  $(TOP)\src\vdbeblob.c \
          234  +  $(TOP)\src\vdbemem.c \
          235  +  $(TOP)\src\vdbetrace.c \
          236  +  $(TOP)\src\vdbeInt.h \
          237  +  $(TOP)\src\vtab.c \
          238  +  $(TOP)\src\wal.c \
          239  +  $(TOP)\src\wal.h \
          240  +  $(TOP)\src\walker.c \
          241  +  $(TOP)\src\where.c
          242  +
          243  +# Source code for extensions
          244  +#
          245  +SRC = $(SRC) \
          246  +  $(TOP)\ext\fts1\fts1.c \
          247  +  $(TOP)\ext\fts1\fts1.h \
          248  +  $(TOP)\ext\fts1\fts1_hash.c \
          249  +  $(TOP)\ext\fts1\fts1_hash.h \
          250  +  $(TOP)\ext\fts1\fts1_porter.c \
          251  +  $(TOP)\ext\fts1\fts1_tokenizer.h \
          252  +  $(TOP)\ext\fts1\fts1_tokenizer1.c
          253  +SRC = $(SRC) \
          254  +  $(TOP)\ext\fts2\fts2.c \
          255  +  $(TOP)\ext\fts2\fts2.h \
          256  +  $(TOP)\ext\fts2\fts2_hash.c \
          257  +  $(TOP)\ext\fts2\fts2_hash.h \
          258  +  $(TOP)\ext\fts2\fts2_icu.c \
          259  +  $(TOP)\ext\fts2\fts2_porter.c \
          260  +  $(TOP)\ext\fts2\fts2_tokenizer.h \
          261  +  $(TOP)\ext\fts2\fts2_tokenizer.c \
          262  +  $(TOP)\ext\fts2\fts2_tokenizer1.c
          263  +SRC = $(SRC) \
          264  +  $(TOP)\ext\fts3\fts3.c \
          265  +  $(TOP)\ext\fts3\fts3.h \
          266  +  $(TOP)\ext\fts3\fts3Int.h \
          267  +  $(TOP)\ext\fts3\fts3_aux.c \
          268  +  $(TOP)\ext\fts3\fts3_expr.c \
          269  +  $(TOP)\ext\fts3\fts3_hash.c \
          270  +  $(TOP)\ext\fts3\fts3_hash.h \
          271  +  $(TOP)\ext\fts3\fts3_icu.c \
          272  +  $(TOP)\ext\fts3\fts3_porter.c \
          273  +  $(TOP)\ext\fts3\fts3_snippet.c \
          274  +  $(TOP)\ext\fts3\fts3_tokenizer.h \
          275  +  $(TOP)\ext\fts3\fts3_tokenizer.c \
          276  +  $(TOP)\ext\fts3\fts3_tokenizer1.c \
          277  +  $(TOP)\ext\fts3\fts3_write.c
          278  +SRC = $(SRC) \
          279  +  $(TOP)\ext\icu\sqliteicu.h \
          280  +  $(TOP)\ext\icu\icu.c
          281  +SRC = $(SRC) \
          282  +  $(TOP)\ext\rtree\rtree.h \
          283  +  $(TOP)\ext\rtree\rtree.c
          284  +
          285  +
          286  +# Generated source code files
          287  +#
          288  +SRC = $(SRC) \
          289  +  keywordhash.h \
          290  +  opcodes.c \
          291  +  opcodes.h \
          292  +  parse.c \
          293  +  parse.h \
          294  +  sqlite3.h
          295  +
          296  +# Source code to the test files.
          297  +#
          298  +TESTSRC = \
          299  +  $(TOP)\src\test1.c \
          300  +  $(TOP)\src\test2.c \
          301  +  $(TOP)\src\test3.c \
          302  +  $(TOP)\src\test4.c \
          303  +  $(TOP)\src\test5.c \
          304  +  $(TOP)\src\test6.c \
          305  +  $(TOP)\src\test7.c \
          306  +  $(TOP)\src\test8.c \
          307  +  $(TOP)\src\test9.c \
          308  +  $(TOP)\src\test_autoext.c \
          309  +  $(TOP)\src\test_async.c \
          310  +  $(TOP)\src\test_backup.c \
          311  +  $(TOP)\src\test_btree.c \
          312  +  $(TOP)\src\test_config.c \
          313  +  $(TOP)\src\test_demovfs.c \
          314  +  $(TOP)\src\test_devsym.c \
          315  +  $(TOP)\src\test_func.c \
          316  +  $(TOP)\src\test_fuzzer.c \
          317  +  $(TOP)\src\test_hexio.c \
          318  +  $(TOP)\src\test_init.c \
          319  +  $(TOP)\src\test_intarray.c \
          320  +  $(TOP)\src\test_journal.c \
          321  +  $(TOP)\src\test_malloc.c \
          322  +  $(TOP)\src\test_multiplex.c \
          323  +  $(TOP)\src\test_mutex.c \
          324  +  $(TOP)\src\test_onefile.c \
          325  +  $(TOP)\src\test_osinst.c \
          326  +  $(TOP)\src\test_pcache.c \
          327  +  $(TOP)\src\test_quota.c \
          328  +  $(TOP)\src\test_rtree.c \
          329  +  $(TOP)\src\test_schema.c \
          330  +  $(TOP)\src\test_server.c \
          331  +  $(TOP)\src\test_superlock.c \
          332  +  $(TOP)\src\test_syscall.c \
          333  +  $(TOP)\src\test_stat.c \
          334  +  $(TOP)\src\test_tclvar.c \
          335  +  $(TOP)\src\test_thread.c \
          336  +  $(TOP)\src\test_vfs.c \
          337  +  $(TOP)\src\test_wholenumber.c \
          338  +  $(TOP)\src\test_wsd.c \
          339  +  $(TOP)\ext\fts3\fts3_term.c \
          340  +  $(TOP)\ext\fts3\fts3_test.c
          341  +
          342  +# Source code to the library files needed by the test fixture
          343  +#
          344  +TESTSRC2 = \
          345  +  $(TOP)\src\attach.c \
          346  +  $(TOP)\src\backup.c \
          347  +  $(TOP)\src\bitvec.c \
          348  +  $(TOP)\src\btree.c \
          349  +  $(TOP)\src\build.c \
          350  +  $(TOP)\src\ctime.c \
          351  +  $(TOP)\src\date.c \
          352  +  $(TOP)\src\expr.c \
          353  +  $(TOP)\src\func.c \
          354  +  $(TOP)\src\insert.c \
          355  +  $(TOP)\src\wal.c \
          356  +  $(TOP)\src\mem5.c \
          357  +  $(TOP)\src\os.c \
          358  +  $(TOP)\src\os_os2.c \
          359  +  $(TOP)\src\os_unix.c \
          360  +  $(TOP)\src\os_win.c \
          361  +  $(TOP)\src\pager.c \
          362  +  $(TOP)\src\pragma.c \
          363  +  $(TOP)\src\prepare.c \
          364  +  $(TOP)\src\printf.c \
          365  +  $(TOP)\src\random.c \
          366  +  $(TOP)\src\pcache.c \
          367  +  $(TOP)\src\pcache1.c \
          368  +  $(TOP)\src\select.c \
          369  +  $(TOP)\src\tokenize.c \
          370  +  $(TOP)\src\utf.c \
          371  +  $(TOP)\src\util.c \
          372  +  $(TOP)\src\vdbeapi.c \
          373  +  $(TOP)\src\vdbeaux.c \
          374  +  $(TOP)\src\vdbe.c \
          375  +  $(TOP)\src\vdbemem.c \
          376  +  $(TOP)\src\vdbetrace.c \
          377  +  $(TOP)\src\where.c \
          378  +  parse.c \
          379  +  $(TOP)\ext\fts3\fts3.c \
          380  +  $(TOP)\ext\fts3\fts3_aux.c \
          381  +  $(TOP)\ext\fts3\fts3_expr.c \
          382  +  $(TOP)\ext\fts3\fts3_tokenizer.c \
          383  +  $(TOP)\ext\fts3\fts3_write.c \
          384  +  $(TOP)\ext\async\sqlite3async.c
          385  +
          386  +# Header files used by all library source files.
          387  +#
          388  +HDR = \
          389  +   $(TOP)\src\btree.h \
          390  +   $(TOP)\src\btreeInt.h \
          391  +   $(TOP)\src\hash.h \
          392  +   $(TOP)\src\hwtime.h \
          393  +   keywordhash.h \
          394  +   $(TOP)\src\mutex.h \
          395  +   opcodes.h \
          396  +   $(TOP)\src\os.h \
          397  +   $(TOP)\src\os_common.h \
          398  +   $(TOP)\src\pager.h \
          399  +   $(TOP)\src\pcache.h \
          400  +   parse.h \
          401  +   sqlite3.h \
          402  +   $(TOP)\src\sqlite3ext.h \
          403  +   $(TOP)\src\sqliteInt.h \
          404  +   $(TOP)\src\sqliteLimit.h \
          405  +   $(TOP)\src\vdbe.h \
          406  +   $(TOP)\src\vdbeInt.h
          407  +
          408  +# Header files used by extensions
          409  +#
          410  +EXTHDR = $(EXTHDR) \
          411  +  $(TOP)\ext\fts1\fts1.h \
          412  +  $(TOP)\ext\fts1\fts1_hash.h \
          413  +  $(TOP)\ext\fts1\fts1_tokenizer.h
          414  +EXTHDR = $(EXTHDR) \
          415  +  $(TOP)\ext\fts2\fts2.h \
          416  +  $(TOP)\ext\fts2\fts2_hash.h \
          417  +  $(TOP)\ext\fts2\fts2_tokenizer.h
          418  +EXTHDR = $(EXTHDR) \
          419  +  $(TOP)\ext\fts3\fts3.h \
          420  +  $(TOP)\ext\fts3\fts3Int.h \
          421  +  $(TOP)\ext\fts3\fts3_hash.h \
          422  +  $(TOP)\ext\fts3\fts3_tokenizer.h
          423  +EXTHDR = $(EXTHDR) \
          424  +  $(TOP)\ext\rtree\rtree.h
          425  +EXTHDR = $(EXTHDR) \
          426  +  $(TOP)\ext\icu\sqliteicu.h
          427  +EXTHDR = $(EXTHDR) \
          428  +  $(TOP)\ext\rtree\sqlite3rtree.h
          429  +
          430  +# This is the default Makefile target.  The objects listed here
          431  +# are what get build when you type just "make" with no arguments.
          432  +#
          433  +all:	libsqlite3.lib sqlite3.exe libtclsqlite3.lib
          434  +
          435  +libsqlite3.lib:	$(LIBOBJ)
          436  +	$(LTLIB) -OUT:$@ $(LIBOBJ) $(TLIBS)
          437  +
          438  +libtclsqlite3.lib:	tclsqlite.lo libsqlite3.lib
          439  +	$(LTLIB) /LIBPATH:$(TCLLIBDIR) -OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS)
          440  +
          441  +sqlite3.exe:	$(TOP)\src\shell.c libsqlite3.lib sqlite3.h
          442  +	$(LTLINK) $(READLINE_FLAGS) \
          443  +		$(TOP)\src\shell.c libsqlite3.lib \
          444  +		$(LIBREADLINE) $(TLIBS)
          445  +
          446  +# This target creates a directory named "tsrc" and fills it with
          447  +# copies of all of the C source code and header files needed to
          448  +# build on the target system.  Some of the C source code and header
          449  +# files are automatically generated.  This target takes care of
          450  +# all that automatic generation.
          451  +#
          452  +.target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl
          453  +	-rmdir /S/Q tsrc
          454  +	-mkdir tsrc
          455  +	for %i in ($(SRC)) do copy /Y %i tsrc
          456  +	del /Q tsrc\sqlite.h.in tsrc\parse.y
          457  +	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl <tsrc\vdbe.c >vdbe.new
          458  +	move vdbe.new tsrc\vdbe.c
          459  +	echo > .target_source
          460  +
          461  +sqlite3.c:	.target_source $(TOP)\tool\mksqlite3c.tcl
          462  +	$(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl
          463  +
          464  +# Rule to build the amalgamation
          465  +#
          466  +sqlite3.lo:	sqlite3.c
          467  +	$(LTCOMPILE) -c sqlite3.c
          468  +
          469  +# Rules to build the LEMON compiler generator
          470  +#
          471  +lempar.c:	$(TOP)\src\lempar.c
          472  +	copy $(TOP)\src\lempar.c .
          473  +
          474  +lemon.exe:	$(TOP)\tool\lemon.c lempar.c
          475  +	$(BCC) -Fe$@ $(TOP)\tool\lemon.c
          476  +
          477  +# Rules to build individual *.lo files from generated *.c files. This
          478  +# applies to:
          479  +#
          480  +#     parse.lo
          481  +#     opcodes.lo
          482  +#
          483  +parse.lo:	parse.c $(HDR)
          484  +	$(LTCOMPILE) -c parse.c
          485  +
          486  +opcodes.lo:	opcodes.c
          487  +	$(LTCOMPILE) -c opcodes.c
          488  +
          489  +# Rules to build individual *.lo files from files in the src directory.
          490  +#
          491  +alter.lo:	$(TOP)\src\alter.c $(HDR)
          492  +	$(LTCOMPILE) -c $(TOP)\src\alter.c
          493  +
          494  +analyze.lo:	$(TOP)\src\analyze.c $(HDR)
          495  +	$(LTCOMPILE) -c $(TOP)\src\analyze.c
          496  +
          497  +attach.lo:	$(TOP)\src\attach.c $(HDR)
          498  +	$(LTCOMPILE) -c $(TOP)\src\attach.c
          499  +
          500  +auth.lo:	$(TOP)\src\auth.c $(HDR)
          501  +	$(LTCOMPILE) -c $(TOP)\src\auth.c
          502  +
          503  +backup.lo:	$(TOP)\src\backup.c $(HDR)
          504  +	$(LTCOMPILE) -c $(TOP)\src\backup.c
          505  +
          506  +bitvec.lo:	$(TOP)\src\bitvec.c $(HDR)
          507  +	$(LTCOMPILE) -c $(TOP)\src\bitvec.c
          508  +
          509  +btmutex.lo:	$(TOP)\src\btmutex.c $(HDR)
          510  +	$(LTCOMPILE) -c $(TOP)\src\btmutex.c
          511  +
          512  +btree.lo:	$(TOP)\src\btree.c $(HDR) $(TOP)\src\pager.h
          513  +	$(LTCOMPILE) -c $(TOP)\src\btree.c
          514  +
          515  +build.lo:	$(TOP)\src\build.c $(HDR)
          516  +	$(LTCOMPILE) -c $(TOP)\src\build.c
          517  +
          518  +callback.lo:	$(TOP)\src\callback.c $(HDR)
          519  +	$(LTCOMPILE) -c $(TOP)\src\callback.c
          520  +
          521  +complete.lo:	$(TOP)\src\complete.c $(HDR)
          522  +	$(LTCOMPILE) -c $(TOP)\src\complete.c
          523  +
          524  +ctime.lo:	$(TOP)\src\ctime.c $(HDR)
          525  +	$(LTCOMPILE) -c $(TOP)\src\ctime.c
          526  +
          527  +date.lo:	$(TOP)\src\date.c $(HDR)
          528  +	$(LTCOMPILE) -c $(TOP)\src\date.c
          529  +
          530  +delete.lo:	$(TOP)\src\delete.c $(HDR)
          531  +	$(LTCOMPILE) -c $(TOP)\src\delete.c
          532  +
          533  +expr.lo:	$(TOP)\src\expr.c $(HDR)
          534  +	$(LTCOMPILE) -c $(TOP)\src\expr.c
          535  +
          536  +fault.lo:	$(TOP)\src\fault.c $(HDR)
          537  +	$(LTCOMPILE) -c $(TOP)\src\fault.c
          538  +
          539  +fkey.lo:	$(TOP)\src\fkey.c $(HDR)
          540  +	$(LTCOMPILE) -c $(TOP)\src\fkey.c
          541  +
          542  +func.lo:	$(TOP)\src\func.c $(HDR)
          543  +	$(LTCOMPILE) -c $(TOP)\src\func.c
          544  +
          545  +global.lo:	$(TOP)\src\global.c $(HDR)
          546  +	$(LTCOMPILE) -c $(TOP)\src\global.c
          547  +
          548  +hash.lo:	$(TOP)\src\hash.c $(HDR)
          549  +	$(LTCOMPILE) -c $(TOP)\src\hash.c
          550  +
          551  +insert.lo:	$(TOP)\src\insert.c $(HDR)
          552  +	$(LTCOMPILE) -c $(TOP)\src\insert.c
          553  +
          554  +journal.lo:	$(TOP)\src\journal.c $(HDR)
          555  +	$(LTCOMPILE) -c $(TOP)\src\journal.c
          556  +
          557  +legacy.lo:	$(TOP)\src\legacy.c $(HDR)
          558  +	$(LTCOMPILE) -c $(TOP)\src\legacy.c
          559  +
          560  +loadext.lo:	$(TOP)\src\loadext.c $(HDR)
          561  +	$(LTCOMPILE) -c $(TOP)\src\loadext.c
          562  +
          563  +main.lo:	$(TOP)\src\main.c $(HDR)
          564  +	$(LTCOMPILE) -c $(TOP)\src\main.c
          565  +
          566  +malloc.lo:	$(TOP)\src\malloc.c $(HDR)
          567  +	$(LTCOMPILE) -c $(TOP)\src\malloc.c
          568  +
          569  +mem0.lo:	$(TOP)\src\mem0.c $(HDR)
          570  +	$(LTCOMPILE) -c $(TOP)\src\mem0.c
          571  +
          572  +mem1.lo:	$(TOP)\src\mem1.c $(HDR)
          573  +	$(LTCOMPILE) -c $(TOP)\src\mem1.c
          574  +
          575  +mem2.lo:	$(TOP)\src\mem2.c $(HDR)
          576  +	$(LTCOMPILE) -c $(TOP)\src\mem2.c
          577  +
          578  +mem3.lo:	$(TOP)\src\mem3.c $(HDR)
          579  +	$(LTCOMPILE) -c $(TOP)\src\mem3.c
          580  +
          581  +mem5.lo:	$(TOP)\src\mem5.c $(HDR)
          582  +	$(LTCOMPILE) -c $(TOP)\src\mem5.c
          583  +
          584  +memjournal.lo:	$(TOP)\src\memjournal.c $(HDR)
          585  +	$(LTCOMPILE) -c $(TOP)\src\memjournal.c
          586  +
          587  +mutex.lo:	$(TOP)\src\mutex.c $(HDR)
          588  +	$(LTCOMPILE) -c $(TOP)\src\mutex.c
          589  +
          590  +mutex_noop.lo:	$(TOP)\src\mutex_noop.c $(HDR)
          591  +	$(LTCOMPILE) -c $(TOP)\src\mutex_noop.c
          592  +
          593  +mutex_os2.lo:	$(TOP)\src\mutex_os2.c $(HDR)
          594  +	$(LTCOMPILE) -c $(TOP)\src\mutex_os2.c
          595  +
          596  +mutex_unix.lo:	$(TOP)\src\mutex_unix.c $(HDR)
          597  +	$(LTCOMPILE) -c $(TOP)\src\mutex_unix.c
          598  +
          599  +mutex_w32.lo:	$(TOP)\src\mutex_w32.c $(HDR)
          600  +	$(LTCOMPILE) -c $(TOP)\src\mutex_w32.c
          601  +
          602  +notify.lo:	$(TOP)\src\notify.c $(HDR)
          603  +	$(LTCOMPILE) -c $(TOP)\src\notify.c
          604  +
          605  +pager.lo:	$(TOP)\src\pager.c $(HDR) $(TOP)\src\pager.h
          606  +	$(LTCOMPILE) -c $(TOP)\src\pager.c
          607  +
          608  +pcache.lo:	$(TOP)\src\pcache.c $(HDR) $(TOP)\src\pcache.h
          609  +	$(LTCOMPILE) -c $(TOP)\src\pcache.c
          610  +
          611  +pcache1.lo:	$(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h
          612  +	$(LTCOMPILE) -c $(TOP)\src\pcache1.c
          613  +
          614  +os.lo:	$(TOP)\src\os.c $(HDR)
          615  +	$(LTCOMPILE) -c $(TOP)\src\os.c
          616  +
          617  +os_unix.lo:	$(TOP)\src\os_unix.c $(HDR)
          618  +	$(LTCOMPILE) -c $(TOP)\src\os_unix.c
          619  +
          620  +os_win.lo:	$(TOP)\src\os_win.c $(HDR)
          621  +	$(LTCOMPILE) -c $(TOP)\src\os_win.c
          622  +
          623  +os_os2.lo:	$(TOP)\src\os_os2.c $(HDR)
          624  +	$(LTCOMPILE) -c $(TOP)\src\os_os2.c
          625  +
          626  +pragma.lo:	$(TOP)\src\pragma.c $(HDR)
          627  +	$(LTCOMPILE) -c $(TOP)\src\pragma.c
          628  +
          629  +prepare.lo:	$(TOP)\src\prepare.c $(HDR)
          630  +	$(LTCOMPILE) -c $(TOP)\src\prepare.c
          631  +
          632  +printf.lo:	$(TOP)\src\printf.c $(HDR)
          633  +	$(LTCOMPILE) -c $(TOP)\src\printf.c
          634  +
          635  +random.lo:	$(TOP)\src\random.c $(HDR)
          636  +	$(LTCOMPILE) -c $(TOP)\src\random.c
          637  +
          638  +resolve.lo:	$(TOP)\src\resolve.c $(HDR)
          639  +	$(LTCOMPILE) -c $(TOP)\src\resolve.c
          640  +
          641  +rowset.lo:	$(TOP)\src\rowset.c $(HDR)
          642  +	$(LTCOMPILE) -c $(TOP)\src\rowset.c
          643  +
          644  +select.lo:	$(TOP)\src\select.c $(HDR)
          645  +	$(LTCOMPILE) -c $(TOP)\src\select.c
          646  +
          647  +status.lo:	$(TOP)\src\status.c $(HDR)
          648  +	$(LTCOMPILE) -c $(TOP)\src\status.c
          649  +
          650  +table.lo:	$(TOP)\src\table.c $(HDR)
          651  +	$(LTCOMPILE) -c $(TOP)\src\table.c
          652  +
          653  +tokenize.lo:	$(TOP)\src\tokenize.c keywordhash.h $(HDR)
          654  +	$(LTCOMPILE) -c $(TOP)\src\tokenize.c
          655  +
          656  +trigger.lo:	$(TOP)\src\trigger.c $(HDR)
          657  +	$(LTCOMPILE) -c $(TOP)\src\trigger.c
          658  +
          659  +update.lo:	$(TOP)\src\update.c $(HDR)
          660  +	$(LTCOMPILE) -c $(TOP)\src\update.c
          661  +
          662  +utf.lo:	$(TOP)\src\utf.c $(HDR)
          663  +	$(LTCOMPILE) -c $(TOP)\src\utf.c
          664  +
          665  +util.lo:	$(TOP)\src\util.c $(HDR)
          666  +	$(LTCOMPILE) -c $(TOP)\src\util.c
          667  +
          668  +vacuum.lo:	$(TOP)\src\vacuum.c $(HDR)
          669  +	$(LTCOMPILE) -c $(TOP)\src\vacuum.c
          670  +
          671  +vdbe.lo:	$(TOP)\src\vdbe.c $(HDR)
          672  +	$(LTCOMPILE) -c $(TOP)\src\vdbe.c
          673  +
          674  +vdbeapi.lo:	$(TOP)\src\vdbeapi.c $(HDR)
          675  +	$(LTCOMPILE) -c $(TOP)\src\vdbeapi.c
          676  +
          677  +vdbeaux.lo:	$(TOP)\src\vdbeaux.c $(HDR)
          678  +	$(LTCOMPILE) -c $(TOP)\src\vdbeaux.c
          679  +
          680  +vdbeblob.lo:	$(TOP)\src\vdbeblob.c $(HDR)
          681  +	$(LTCOMPILE) -c $(TOP)\src\vdbeblob.c
          682  +
          683  +vdbemem.lo:	$(TOP)\src\vdbemem.c $(HDR)
          684  +	$(LTCOMPILE) -c $(TOP)\src\vdbemem.c
          685  +
          686  +vdbetrace.lo:	$(TOP)\src\vdbetrace.c $(HDR)
          687  +	$(LTCOMPILE) -c $(TOP)\src\vdbetrace.c
          688  +
          689  +vtab.lo:	$(TOP)\src\vtab.c $(HDR)
          690  +	$(LTCOMPILE) -c $(TOP)\src\vtab.c
          691  +
          692  +wal.lo:	$(TOP)\src\wal.c $(HDR)
          693  +	$(LTCOMPILE) -c $(TOP)\src\wal.c
          694  +
          695  +walker.lo:	$(TOP)\src\walker.c $(HDR)
          696  +	$(LTCOMPILE) -c $(TOP)\src\walker.c
          697  +
          698  +where.lo:	$(TOP)\src\where.c $(HDR)
          699  +	$(LTCOMPILE) -c $(TOP)\src\where.c
          700  +
          701  +tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR)
          702  +	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
          703  +
          704  +tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR)
          705  +	$(LTCOMPILE) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
          706  +
          707  +tclsqlite3.exe:	tclsqlite-shell.lo libsqlite3.lib
          708  +	$(LTLINK) tclsqlite-shell.lo \
          709  +		/link /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL)
          710  +
          711  +# Rules to build opcodes.c and opcodes.h
          712  +#
          713  +opcodes.c:	opcodes.h $(TOP)\mkopcodec.awk
          714  +	$(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c
          715  +
          716  +opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk
          717  +	type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h
          718  +
          719  +# Rules to build parse.c and parse.h - the outputs of lemon.
          720  +#
          721  +parse.h:	parse.c
          722  +
          723  +parse.c:	$(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk
          724  +	del /Q parse.y parse.h parse.h.temp
          725  +	copy $(TOP)\src\parse.y .
          726  +	.\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
          727  +	move parse.h parse.h.temp
          728  +	$(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h
          729  +
          730  +sqlite3.h:	$(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION
          731  +	$(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h
          732  +
          733  +mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
          734  +	$(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c
          735  +
          736  +keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
          737  +	.\mkkeywordhash.exe >keywordhash.h
          738  +
          739  +
          740  +
          741  +# Rules to build the extension objects.
          742  +#
          743  +icu.lo:	$(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
          744  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c
          745  +
          746  +fts2.lo:	$(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR)
          747  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2.c
          748  +
          749  +fts2_hash.lo:	$(TOP)\ext\fts2\fts2_hash.c $(HDR) $(EXTHDR)
          750  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_hash.c
          751  +
          752  +fts2_icu.lo:	$(TOP)\ext\fts2\fts2_icu.c $(HDR) $(EXTHDR)
          753  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_icu.c
          754  +
          755  +fts2_porter.lo:	$(TOP)\ext\fts2\fts2_porter.c $(HDR) $(EXTHDR)
          756  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_porter.c
          757  +
          758  +fts2_tokenizer.lo:	$(TOP)\ext\fts2\fts2_tokenizer.c $(HDR) $(EXTHDR)
          759  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer.c
          760  +
          761  +fts2_tokenizer1.lo:	$(TOP)\ext\fts2\fts2_tokenizer1.c $(HDR) $(EXTHDR)
          762  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer1.c
          763  +
          764  +fts3.lo:	$(TOP)\ext\fts3\fts3.c $(HDR) $(EXTHDR)
          765  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3.c
          766  +
          767  +fts3_aux.lo:	$(TOP)\ext\fts3\fts3_aux.c $(HDR) $(EXTHDR)
          768  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_aux.c
          769  +
          770  +fts3_expr.lo:	$(TOP)\ext\fts3\fts3_expr.c $(HDR) $(EXTHDR)
          771  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_expr.c
          772  +
          773  +fts3_hash.lo:	$(TOP)\ext\fts3\fts3_hash.c $(HDR) $(EXTHDR)
          774  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_hash.c
          775  +
          776  +fts3_icu.lo:	$(TOP)\ext\fts3\fts3_icu.c $(HDR) $(EXTHDR)
          777  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_icu.c
          778  +
          779  +fts3_snippet.lo:	$(TOP)\ext\fts3\fts3_snippet.c $(HDR) $(EXTHDR)
          780  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_snippet.c
          781  +
          782  +fts3_porter.lo:	$(TOP)\ext\fts3\fts3_porter.c $(HDR) $(EXTHDR)
          783  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_porter.c
          784  +
          785  +fts3_tokenizer.lo:	$(TOP)\ext\fts3\fts3_tokenizer.c $(HDR) $(EXTHDR)
          786  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer.c
          787  +
          788  +fts3_tokenizer1.lo:	$(TOP)\ext\fts3\fts3_tokenizer1.c $(HDR) $(EXTHDR)
          789  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer1.c
          790  +
          791  +fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
          792  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
          793  +
          794  +rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
          795  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
          796  +
          797  +
          798  +# Rules to build the 'testfixture' application.
          799  +#
          800  +# If using the amalgamation, use sqlite3.c directly to build the test
          801  +# fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
          802  +# necessary because the test fixture requires non-API symbols which are
          803  +# hidden when the library is built via the amalgamation).
          804  +#
          805  +TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
          806  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
          807  +
          808  +TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.lib
          809  +TESTFIXTURE_SRC1 = sqlite3.c
          810  +!IF $(USE_AMALGAMATION)==0
          811  +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
          812  +!ELSE
          813  +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1)
          814  +!ENDIF
          815  +
          816  +testfixture.exe:	$(TESTFIXTURE_SRC) $(HDR)
          817  +	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
          818  +		-DBUILD_sqlite -I$(TCLINCDIR) \
          819  +		$(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS)
          820  +
          821  +fulltest:	testfixture.exe sqlite3.exe
          822  +	.\testfixture.exe $(TOP)\test\all.test
          823  +
          824  +soaktest:	testfixture.exe sqlite3.exe
          825  +	.\testfixture.exe $(TOP)\test\all.test -soak=1
          826  +
          827  +test:	testfixture.exe sqlite3.exe
          828  +	.\testfixture.exe $(TOP)\test\veryquick.test
          829  +
          830  +spaceanal_tcl.h:	$(TOP)\tool\spaceanal.tcl
          831  +	$(NAWK) "/^[^#]/ { gsub(/\\/,\"\\\\\\\\\");gsub(/\\\"/,\"\\\\\\\"\");gsub(/^/,\"\\\"\");gsub(/$$/,\"\\n\\\"\");print }" \
          832  +		$(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h
          833  +
          834  +sqlite3_analyzer.exe:	$(TESTFIXTURE_SRC) spaceanal_tcl.h
          835  +	$(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \
          836  +		-DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \
          837  +		-DBUILD_sqlite -I$(TCLINCDIR) \
          838  +		$(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS)
          839  +
          840  +clean:
          841  +	del /Q *.lo *.lib *.obj sqlite3.exe libsqlite3.lib
          842  +	del /Q sqlite3.h opcodes.c opcodes.h
          843  +	del /Q lemon.exe lempar.c parse.*
          844  +	del /Q mkkeywordhash.exe keywordhash.h
          845  +	-rmdir /Q/S tsrc
          846  +	del /Q .target_source
          847  +	del /Q testfixture.exe testfixture.exp test.db
          848  +	del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
          849  +	del /Q sqlite3.c
          850  +	del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp spaceanal_tcl.h
          851  +
          852  +#
          853  +# Windows section
          854  +#
          855  +dll: sqlite3.dll
          856  +
          857  +sqlite3.def: libsqlite3.lib
          858  +	echo EXPORTS >sqlite3.def
          859  +	dumpbin /all libsqlite3.lib \
          860  +		| $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \
          861  +		| sort >>sqlite3.def
          862  +
          863  +sqlite3.dll: $(LIBOBJ) sqlite3.def
          864  +	link /DLL /OUT:$@ /DEF:sqlite3.def $(LIBOBJ)

Changes to ext/fts3/fts3.c.

   288    288   ** older data.
   289    289   **
   290    290   ** TODO(shess) Provide a VACUUM type operation to clear out all
   291    291   ** deletions and duplications.  This would basically be a forced merge
   292    292   ** into a single segment.
   293    293   */
   294    294   
          295  +#include "fts3Int.h"
   295    296   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
   296    297   
   297    298   #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
   298    299   # define SQLITE_CORE 1
   299    300   #endif
   300    301   
   301         -#include "fts3Int.h"
   302         -
   303    302   #include <assert.h>
   304    303   #include <stdlib.h>
   305    304   #include <stddef.h>
   306    305   #include <stdio.h>
   307    306   #include <string.h>
   308    307   #include <stdarg.h>
   309    308   
................................................................................
   419    418     *pVal += iVal;
   420    419   }
   421    420   
   422    421   /*
   423    422   ** When this function is called, *pp points to the first byte following a
   424    423   ** varint that is part of a doclist (or position-list, or any other list
   425    424   ** of varints). This function moves *pp to point to the start of that varint,
   426         -** and decrements the value stored in *pVal by the varint value.
          425  +** and sets *pVal by the varint value.
   427    426   **
   428    427   ** Argument pStart points to the first byte of the doclist that the
   429    428   ** varint is part of.
   430    429   */
   431         -static void fts3GetReverseDeltaVarint(
          430  +static void fts3GetReverseVarint(
   432    431     char **pp, 
   433    432     char *pStart, 
   434    433     sqlite3_int64 *pVal
   435    434   ){
   436    435     sqlite3_int64 iVal;
   437    436     char *p = *pp;
   438    437   
................................................................................
   440    439     ** interested in. So, unless the doclist is corrupt, the 0x80 bit is
   441    440     ** clear on character p[-1]. */
   442    441     for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
   443    442     p++;
   444    443     *pp = p;
   445    444   
   446    445     sqlite3Fts3GetVarint(p, &iVal);
   447         -  *pVal -= iVal;
   448         -}
   449         -
   450         -/*
   451         -** As long as *pp has not reached its end (pEnd), then do the same
   452         -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
   453         -** But if we have reached the end of the varint, just set *pp=0 and
   454         -** leave *pVal unchanged.
   455         -*/
   456         -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
   457         -  if( *pp>=pEnd ){
   458         -    *pp = 0;
   459         -  }else{
   460         -    fts3GetDeltaVarint(pp, pVal);
   461         -  }
          446  +  *pVal = iVal;
   462    447   }
   463    448   
   464    449   /*
   465    450   ** The xDisconnect() virtual table method.
   466    451   */
   467    452   static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
   468    453     Fts3Table *p = (Fts3Table *)pVtab;
................................................................................
   827    812     fts3Appendf(pRc, &zRet, "?");
   828    813     for(i=0; i<p->nColumn; i++){
   829    814       fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
   830    815     }
   831    816     sqlite3_free(zFree);
   832    817     return zRet;
   833    818   }
          819  +
          820  +static int fts3GobbleInt(const char **pp, int *pnOut){
          821  +  const char *p = *pp;
          822  +  int nInt = 0;
          823  +  for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
          824  +    nInt = nInt * 10 + (p[0] - '0');
          825  +  }
          826  +  if( p==*pp ) return SQLITE_ERROR;
          827  +  *pnOut = nInt;
          828  +  *pp = p;
          829  +  return SQLITE_OK;
          830  +}
          831  +
          832  +
          833  +static int fts3PrefixParameter(
          834  +  const char *zParam,             /* ABC in prefix=ABC parameter to parse */
          835  +  int *pnIndex,                   /* OUT: size of *apIndex[] array */
          836  +  struct Fts3Index **apIndex,     /* OUT: Array of indexes for this table */
          837  +  struct Fts3Index **apFree       /* OUT: Free this with sqlite3_free() */
          838  +){
          839  +  struct Fts3Index *aIndex;
          840  +  int nIndex = 1;
          841  +
          842  +  if( zParam && zParam[0] ){
          843  +    const char *p;
          844  +    nIndex++;
          845  +    for(p=zParam; *p; p++){
          846  +      if( *p==',' ) nIndex++;
          847  +    }
          848  +  }
          849  +
          850  +  aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
          851  +  *apIndex = *apFree = aIndex;
          852  +  *pnIndex = nIndex;
          853  +  if( !aIndex ){
          854  +    return SQLITE_NOMEM;
          855  +  }
          856  +
          857  +  memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
          858  +  if( zParam ){
          859  +    const char *p = zParam;
          860  +    int i;
          861  +    for(i=1; i<nIndex; i++){
          862  +      int nPrefix;
          863  +      if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
          864  +      aIndex[i].nPrefix = nPrefix;
          865  +      p++;
          866  +    }
          867  +  }
          868  +
          869  +  return SQLITE_OK;
          870  +}
   834    871   
   835    872   /*
   836    873   ** This function is the implementation of both the xConnect and xCreate
   837    874   ** methods of the FTS3 virtual table.
   838    875   **
   839    876   ** The argv[] array contains the following:
   840    877   **
................................................................................
   860    897     int iCol;                       /* Column index */
   861    898     int nString = 0;                /* Bytes required to hold all column names */
   862    899     int nCol = 0;                   /* Number of columns in the FTS table */
   863    900     char *zCsr;                     /* Space for holding column names */
   864    901     int nDb;                        /* Bytes required to hold database name */
   865    902     int nName;                      /* Bytes required to hold table name */
   866    903     int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
   867         -  int bNoDocsize = 0;             /* True to omit %_docsize table */
   868    904     const char **aCol;              /* Array of column names */
   869    905     sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */
   870    906   
   871         -  char *zCompress = 0;
   872         -  char *zUncompress = 0;
          907  +  int nIndex;                     /* Size of aIndex[] array */
          908  +  struct Fts3Index *aIndex;       /* Array of indexes for this table */
          909  +  struct Fts3Index *aFree = 0;    /* Free this before returning */
          910  +
          911  +  /* The results of parsing supported FTS4 key=value options: */
          912  +  int bNoDocsize = 0;             /* True to omit %_docsize table */
          913  +  int bDescIdx = 0;               /* True to store descending indexes */
          914  +  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
          915  +  char *zCompress = 0;            /* compress=? parameter (or NULL) */
          916  +  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
   873    917   
   874    918     assert( strlen(argv[0])==4 );
   875    919     assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
   876    920          || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
   877    921     );
   878    922   
   879    923     nDb = (int)strlen(argv[1]) + 1;
................................................................................
   906    950        && 0==sqlite3Fts3IsIdChar(z[8])
   907    951       ){
   908    952         rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
   909    953       }
   910    954   
   911    955       /* Check if it is an FTS4 special argument. */
   912    956       else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
          957  +      struct Fts4Option {
          958  +        const char *zOpt;
          959  +        int nOpt;
          960  +        char **pzVar;
          961  +      } aFts4Opt[] = {
          962  +        { "matchinfo",   9, 0 },            /* 0 -> MATCHINFO */
          963  +        { "prefix",      6, 0 },            /* 1 -> PREFIX */
          964  +        { "compress",    8, 0 },            /* 2 -> COMPRESS */
          965  +        { "uncompress", 10, 0 },            /* 3 -> UNCOMPRESS */
          966  +        { "order",       5, 0 }             /* 4 -> ORDER */
          967  +      };
          968  +
          969  +      int iOpt;
   913    970         if( !zVal ){
   914    971           rc = SQLITE_NOMEM;
   915         -        goto fts3_init_out;
   916         -      }
   917         -      if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
   918         -        if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
   919         -          bNoDocsize = 1;
   920         -        }else{
   921         -          *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
          972  +      }else{
          973  +        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
          974  +          struct Fts4Option *pOp = &aFts4Opt[iOpt];
          975  +          if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
          976  +            break;
          977  +          }
          978  +        }
          979  +        if( iOpt==SizeofArray(aFts4Opt) ){
          980  +          *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
   922    981             rc = SQLITE_ERROR;
   923         -        }
   924         -      }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
   925         -        zCompress = zVal;
   926         -        zVal = 0;
   927         -      }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
   928         -        zUncompress = zVal;
   929         -        zVal = 0;
   930         -      }else{
   931         -        *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
   932         -        rc = SQLITE_ERROR;
   933         -      }
   934         -      sqlite3_free(zVal);
          982  +        }else{
          983  +          switch( iOpt ){
          984  +            case 0:               /* MATCHINFO */
          985  +              if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
          986  +                *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
          987  +                rc = SQLITE_ERROR;
          988  +              }
          989  +              bNoDocsize = 1;
          990  +              break;
          991  +
          992  +            case 1:               /* PREFIX */
          993  +              sqlite3_free(zPrefix);
          994  +              zPrefix = zVal;
          995  +              zVal = 0;
          996  +              break;
          997  +
          998  +            case 2:               /* COMPRESS */
          999  +              sqlite3_free(zCompress);
         1000  +              zCompress = zVal;
         1001  +              zVal = 0;
         1002  +              break;
         1003  +
         1004  +            case 3:               /* UNCOMPRESS */
         1005  +              sqlite3_free(zUncompress);
         1006  +              zUncompress = zVal;
         1007  +              zVal = 0;
         1008  +              break;
         1009  +
         1010  +            case 4:               /* ORDER */
         1011  +              if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
         1012  +               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) 
         1013  +              ){
         1014  +                *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
         1015  +                rc = SQLITE_ERROR;
         1016  +              }
         1017  +              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
         1018  +              break;
         1019  +          }
         1020  +        }
         1021  +        sqlite3_free(zVal);
         1022  +      }
   935   1023       }
   936   1024   
   937   1025       /* Otherwise, the argument is a column name. */
   938   1026       else {
   939   1027         nString += (int)(strlen(z) + 1);
   940   1028         aCol[nCol++] = z;
   941   1029       }
................................................................................
   951   1039   
   952   1040     if( pTokenizer==0 ){
   953   1041       rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
   954   1042       if( rc!=SQLITE_OK ) goto fts3_init_out;
   955   1043     }
   956   1044     assert( pTokenizer );
   957   1045   
         1046  +  rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
         1047  +  if( rc==SQLITE_ERROR ){
         1048  +    assert( zPrefix );
         1049  +    *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
         1050  +  }
         1051  +  if( rc!=SQLITE_OK ) goto fts3_init_out;
   958   1052   
   959   1053     /* Allocate and populate the Fts3Table structure. */
   960         -  nByte = sizeof(Fts3Table) +              /* Fts3Table */
         1054  +  nByte = sizeof(Fts3Table) +                  /* Fts3Table */
   961   1055             nCol * sizeof(char *) +              /* azColumn */
         1056  +          nIndex * sizeof(struct Fts3Index) +  /* aIndex */
   962   1057             nName +                              /* zName */
   963   1058             nDb +                                /* zDb */
   964   1059             nString;                             /* Space for azColumn strings */
   965   1060     p = (Fts3Table*)sqlite3_malloc(nByte);
   966   1061     if( p==0 ){
   967   1062       rc = SQLITE_NOMEM;
   968   1063       goto fts3_init_out;
................................................................................
   969   1064     }
   970   1065     memset(p, 0, nByte);
   971   1066     p->db = db;
   972   1067     p->nColumn = nCol;
   973   1068     p->nPendingData = 0;
   974   1069     p->azColumn = (char **)&p[1];
   975   1070     p->pTokenizer = pTokenizer;
   976         -  p->nNodeSize = 1000;
   977   1071     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
   978   1072     p->bHasDocsize = (isFts4 && bNoDocsize==0);
   979   1073     p->bHasStat = isFts4;
         1074  +  p->bDescIdx = bDescIdx;
   980   1075     TESTONLY( p->inTransaction = -1 );
   981   1076     TESTONLY( p->mxSavepoint = -1 );
   982         -  fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
         1077  +
         1078  +  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
         1079  +  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
         1080  +  p->nIndex = nIndex;
         1081  +  for(i=0; i<nIndex; i++){
         1082  +    fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
         1083  +  }
   983   1084   
   984   1085     /* Fill in the zName and zDb fields of the vtab structure. */
   985         -  zCsr = (char *)&p->azColumn[nCol];
         1086  +  zCsr = (char *)&p->aIndex[nIndex];
   986   1087     p->zName = zCsr;
   987   1088     memcpy(zCsr, argv[2], nName);
   988   1089     zCsr += nName;
   989   1090     p->zDb = zCsr;
   990   1091     memcpy(zCsr, argv[1], nDb);
   991   1092     zCsr += nDb;
   992   1093   
................................................................................
  1016   1117     ** database. TODO: For xConnect(), it could verify that said tables exist.
  1017   1118     */
  1018   1119     if( isCreate ){
  1019   1120       rc = fts3CreateTables(p);
  1020   1121     }
  1021   1122   
  1022   1123     /* Figure out the page-size for the database. This is required in order to
  1023         -  ** estimate the cost of loading large doclists from the database (see 
  1024         -  ** function sqlite3Fts3SegReaderCost() for details).
  1025         -  */
         1124  +  ** estimate the cost of loading large doclists from the database.  */
  1026   1125     fts3DatabasePageSize(&rc, p);
         1126  +  p->nNodeSize = p->nPgsz-35;
  1027   1127   
  1028   1128     /* Declare the table schema to SQLite. */
  1029   1129     fts3DeclareVtab(&rc, p);
  1030   1130   
  1031   1131   fts3_init_out:
         1132  +  sqlite3_free(zPrefix);
         1133  +  sqlite3_free(aFree);
  1032   1134     sqlite3_free(zCompress);
  1033   1135     sqlite3_free(zUncompress);
  1034   1136     sqlite3_free((void *)aCol);
  1035   1137     if( rc!=SQLITE_OK ){
  1036   1138       if( p ){
  1037   1139         fts3DisconnectMethod((sqlite3_vtab *)p);
  1038   1140       }else if( pTokenizer ){
  1039   1141         pTokenizer->pModule->xDestroy(pTokenizer);
  1040   1142       }
  1041   1143     }else{
         1144  +    assert( p->pSegments==0 );
  1042   1145       *ppVTab = &p->base;
  1043   1146     }
  1044   1147     return rc;
  1045   1148   }
  1046   1149   
  1047   1150   /*
  1048   1151   ** The xConnect() and xCreate() methods for the virtual table. All the
................................................................................
  1132   1235       struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
  1133   1236       if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
  1134   1237         if( pOrder->desc ){
  1135   1238           pInfo->idxStr = "DESC";
  1136   1239         }else{
  1137   1240           pInfo->idxStr = "ASC";
  1138   1241         }
         1242  +      pInfo->orderByConsumed = 1;
  1139   1243       }
  1140         -    pInfo->orderByConsumed = 1;
  1141   1244     }
  1142   1245   
         1246  +  assert( p->pSegments==0 );
  1143   1247     return SQLITE_OK;
  1144   1248   }
  1145   1249   
  1146   1250   /*
  1147   1251   ** Implementation of xOpen method.
  1148   1252   */
  1149   1253   static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
................................................................................
  1171   1275     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  1172   1276     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1173   1277     sqlite3_finalize(pCsr->pStmt);
  1174   1278     sqlite3Fts3ExprFree(pCsr->pExpr);
  1175   1279     sqlite3Fts3FreeDeferredTokens(pCsr);
  1176   1280     sqlite3_free(pCsr->aDoclist);
  1177   1281     sqlite3_free(pCsr->aMatchinfo);
         1282  +  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1178   1283     sqlite3_free(pCsr);
  1179   1284     return SQLITE_OK;
  1180   1285   }
  1181   1286   
  1182   1287   /*
  1183   1288   ** Position the pCsr->pStmt statement so that it is on the row
  1184   1289   ** of the %_content table that contains the last match.  Return
  1185   1290   ** SQLITE_OK on success.  
  1186   1291   */
  1187   1292   static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
  1188   1293     if( pCsr->isRequireSeek ){
  1189         -    pCsr->isRequireSeek = 0;
  1190   1294       sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
         1295  +    pCsr->isRequireSeek = 0;
  1191   1296       if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
  1192   1297         return SQLITE_OK;
  1193   1298       }else{
  1194   1299         int rc = sqlite3_reset(pCsr->pStmt);
  1195   1300         if( rc==SQLITE_OK ){
  1196   1301           /* If no row was found and no error has occured, then the %_content
  1197   1302           ** table is missing a row that is present in the full-text index.
................................................................................
  1364   1469     assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
  1365   1470   
  1366   1471     if( rc==SQLITE_OK && iHeight>1 ){
  1367   1472       char *zBlob = 0;              /* Blob read from %_segments table */
  1368   1473       int nBlob;                    /* Size of zBlob in bytes */
  1369   1474   
  1370   1475       if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
  1371         -      rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
         1476  +      rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0);
  1372   1477         if( rc==SQLITE_OK ){
  1373   1478           rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
  1374   1479         }
  1375   1480         sqlite3_free(zBlob);
  1376   1481         piLeaf = 0;
  1377   1482         zBlob = 0;
  1378   1483       }
  1379   1484   
  1380   1485       if( rc==SQLITE_OK ){
  1381         -      rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
         1486  +      rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
  1382   1487       }
  1383   1488       if( rc==SQLITE_OK ){
  1384   1489         rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
  1385   1490       }
  1386   1491       sqlite3_free(zBlob);
  1387   1492     }
  1388   1493   
................................................................................
  1750   1855     }
  1751   1856     *p++ = 0x00;
  1752   1857     *pp = p;
  1753   1858     return 1;
  1754   1859   }
  1755   1860   
  1756   1861   /*
  1757         -** Merge two position-lists as required by the NEAR operator.
         1862  +** Merge two position-lists as required by the NEAR operator. The argument
         1863  +** position lists correspond to the left and right phrases of an expression 
         1864  +** like:
         1865  +**
         1866  +**     "phrase 1" NEAR "phrase number 2"
         1867  +**
         1868  +** Position list *pp1 corresponds to the left-hand side of the NEAR 
         1869  +** expression and *pp2 to the right. As usual, the indexes in the position 
         1870  +** lists are the offsets of the last token in each phrase (tokens "1" and "2" 
         1871  +** in the example above).
         1872  +**
         1873  +** The output position list - written to *pp - is a copy of *pp2 with those
         1874  +** entries that are not sufficiently NEAR entries in *pp1 removed.
  1758   1875   */
  1759   1876   static int fts3PoslistNearMerge(
  1760   1877     char **pp,                      /* Output buffer */
  1761   1878     char *aTmp,                     /* Temporary buffer space */
  1762   1879     int nRight,                     /* Maximum difference in token positions */
  1763   1880     int nLeft,                      /* Maximum difference in token positions */
  1764   1881     char **pp1,                     /* IN/OUT: Left input list */
  1765   1882     char **pp2                      /* IN/OUT: Right input list */
  1766   1883   ){
  1767   1884     char *p1 = *pp1;
  1768   1885     char *p2 = *pp2;
  1769   1886   
  1770         -  if( !pp ){
  1771         -    if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
  1772         -    *pp1 = p1;
  1773         -    *pp2 = p2;
  1774         -    return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
  1775         -  }else{
  1776         -    char *pTmp1 = aTmp;
  1777         -    char *pTmp2;
  1778         -    char *aTmp2;
  1779         -    int res = 1;
  1780         -
  1781         -    fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
  1782         -    aTmp2 = pTmp2 = pTmp1;
  1783         -    *pp1 = p1;
  1784         -    *pp2 = p2;
  1785         -    fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
  1786         -    if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
  1787         -      fts3PoslistMerge(pp, &aTmp, &aTmp2);
  1788         -    }else if( pTmp1!=aTmp ){
  1789         -      fts3PoslistCopy(pp, &aTmp);
  1790         -    }else if( pTmp2!=aTmp2 ){
  1791         -      fts3PoslistCopy(pp, &aTmp2);
  1792         -    }else{
  1793         -      res = 0;
  1794         -    }
  1795         -
  1796         -    return res;
  1797         -  }
  1798         -}
  1799         -
  1800         -/*
  1801         -** Values that may be used as the first parameter to fts3DoclistMerge().
  1802         -*/
  1803         -#define MERGE_NOT        2        /* D + D -> D */
  1804         -#define MERGE_AND        3        /* D + D -> D */
  1805         -#define MERGE_OR         4        /* D + D -> D */
  1806         -#define MERGE_POS_OR     5        /* P + P -> P */
  1807         -#define MERGE_PHRASE     6        /* P + P -> D */
  1808         -#define MERGE_POS_PHRASE 7        /* P + P -> P */
  1809         -#define MERGE_NEAR       8        /* P + P -> D */
  1810         -#define MERGE_POS_NEAR   9        /* P + P -> P */
  1811         -
  1812         -/*
  1813         -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
  1814         -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
  1815         -** which is guaranteed to be large enough to hold the results. The number
  1816         -** of bytes written to aBuffer is stored in *pnBuffer before returning.
  1817         -**
  1818         -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
  1819         -** occurs while allocating a temporary buffer as part of the merge operation,
  1820         -** SQLITE_NOMEM is returned.
  1821         -*/
  1822         -static int fts3DoclistMerge(
  1823         -  int mergetype,                  /* One of the MERGE_XXX constants */
  1824         -  int nParam1,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
  1825         -  int nParam2,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
  1826         -  char *aBuffer,                  /* Pre-allocated output buffer */
  1827         -  int *pnBuffer,                  /* OUT: Bytes written to aBuffer */
  1828         -  char *a1,                       /* Buffer containing first doclist */
  1829         -  int n1,                         /* Size of buffer a1 */
  1830         -  char *a2,                       /* Buffer containing second doclist */
  1831         -  int n2,                         /* Size of buffer a2 */
  1832         -  int *pnDoc                      /* OUT: Number of docids in output */
  1833         -){
  1834         -  sqlite3_int64 i1 = 0;
  1835         -  sqlite3_int64 i2 = 0;
  1836         -  sqlite3_int64 iPrev = 0;
  1837         -
  1838         -  char *p = aBuffer;
  1839         -  char *p1 = a1;
  1840         -  char *p2 = a2;
  1841         -  char *pEnd1 = &a1[n1];
  1842         -  char *pEnd2 = &a2[n2];
  1843         -  int nDoc = 0;
  1844         -
  1845         -  assert( mergetype==MERGE_OR     || mergetype==MERGE_POS_OR 
  1846         -       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
  1847         -       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
  1848         -       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  1849         -  );
  1850         -
  1851         -  if( !aBuffer ){
  1852         -    *pnBuffer = 0;
  1853         -    return SQLITE_NOMEM;
  1854         -  }
  1855         -
  1856         -  /* Read the first docid from each doclist */
  1857         -  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1858         -  fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1859         -
  1860         -  switch( mergetype ){
  1861         -    case MERGE_OR:
  1862         -    case MERGE_POS_OR:
  1863         -      while( p1 || p2 ){
  1864         -        if( p2 && p1 && i1==i2 ){
  1865         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1866         -          if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
  1867         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1868         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1869         -        }else if( !p2 || (p1 && i1<i2) ){
  1870         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1871         -          if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
  1872         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1873         -        }else{
  1874         -          fts3PutDeltaVarint(&p, &iPrev, i2);
  1875         -          if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
  1876         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1877         -        }
  1878         -      }
  1879         -      break;
  1880         -
  1881         -    case MERGE_AND:
  1882         -      while( p1 && p2 ){
  1883         -        if( i1==i2 ){
  1884         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1885         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1886         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1887         -          nDoc++;
  1888         -        }else if( i1<i2 ){
  1889         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1890         -        }else{
  1891         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1892         -        }
  1893         -      }
  1894         -      break;
  1895         -
  1896         -    case MERGE_NOT:
  1897         -      while( p1 ){
  1898         -        if( p2 && i1==i2 ){
  1899         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1900         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1901         -        }else if( !p2 || i1<i2 ){
  1902         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1903         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1904         -        }else{
  1905         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1906         -        }
  1907         -      }
  1908         -      break;
  1909         -
  1910         -    case MERGE_POS_PHRASE:
  1911         -    case MERGE_PHRASE: {
  1912         -      char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
  1913         -      while( p1 && p2 ){
  1914         -        if( i1==i2 ){
  1915         -          char *pSave = p;
  1916         -          sqlite3_int64 iPrevSave = iPrev;
  1917         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1918         -          if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
  1919         -            p = pSave;
  1920         -            iPrev = iPrevSave;
  1921         -          }else{
  1922         -            nDoc++;
  1923         -          }
  1924         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1925         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1926         -        }else if( i1<i2 ){
  1927         -          fts3PoslistCopy(0, &p1);
  1928         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1929         -        }else{
  1930         -          fts3PoslistCopy(0, &p2);
  1931         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1932         -        }
  1933         -      }
  1934         -      break;
  1935         -    }
  1936         -
  1937         -    default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
  1938         -      char *aTmp = 0;
  1939         -      char **ppPos = 0;
  1940         -
  1941         -      if( mergetype==MERGE_POS_NEAR ){
  1942         -        ppPos = &p;
  1943         -        aTmp = sqlite3_malloc(2*(n1+n2+1));
  1944         -        if( !aTmp ){
  1945         -          return SQLITE_NOMEM;
  1946         -        }
  1947         -      }
  1948         -
  1949         -      while( p1 && p2 ){
  1950         -        if( i1==i2 ){
  1951         -          char *pSave = p;
  1952         -          sqlite3_int64 iPrevSave = iPrev;
  1953         -          fts3PutDeltaVarint(&p, &iPrev, i1);
  1954         -
  1955         -          if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
  1956         -            iPrev = iPrevSave;
  1957         -            p = pSave;
  1958         -          }
  1959         -
  1960         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1961         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1962         -        }else if( i1<i2 ){
  1963         -          fts3PoslistCopy(0, &p1);
  1964         -          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1965         -        }else{
  1966         -          fts3PoslistCopy(0, &p2);
  1967         -          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1968         -        }
  1969         -      }
  1970         -      sqlite3_free(aTmp);
  1971         -      break;
  1972         -    }
  1973         -  }
  1974         -
  1975         -  if( pnDoc ) *pnDoc = nDoc;
  1976         -  *pnBuffer = (int)(p-aBuffer);
  1977         -  return SQLITE_OK;
         1887  +  char *pTmp1 = aTmp;
         1888  +  char *pTmp2;
         1889  +  char *aTmp2;
         1890  +  int res = 1;
         1891  +
         1892  +  fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
         1893  +  aTmp2 = pTmp2 = pTmp1;
         1894  +  *pp1 = p1;
         1895  +  *pp2 = p2;
         1896  +  fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
         1897  +  if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
         1898  +    fts3PoslistMerge(pp, &aTmp, &aTmp2);
         1899  +  }else if( pTmp1!=aTmp ){
         1900  +    fts3PoslistCopy(pp, &aTmp);
         1901  +  }else if( pTmp2!=aTmp2 ){
         1902  +    fts3PoslistCopy(pp, &aTmp2);
         1903  +  }else{
         1904  +    res = 0;
         1905  +  }
         1906  +
         1907  +  return res;
  1978   1908   }
  1979   1909   
  1980   1910   /* 
  1981   1911   ** A pointer to an instance of this structure is used as the context 
  1982   1912   ** argument to sqlite3Fts3SegReaderIterate()
  1983   1913   */
  1984   1914   typedef struct TermSelect TermSelect;
  1985   1915   struct TermSelect {
  1986   1916     int isReqPos;
  1987   1917     char *aaOutput[16];             /* Malloc'd output buffer */
  1988   1918     int anOutput[16];               /* Size of output in bytes */
  1989   1919   };
         1920  +
         1921  +
         1922  +static void fts3GetDeltaVarint3(
         1923  +  char **pp, 
         1924  +  char *pEnd, 
         1925  +  int bDescIdx,
         1926  +  sqlite3_int64 *pVal
         1927  +){
         1928  +  if( *pp>=pEnd ){
         1929  +    *pp = 0;
         1930  +  }else{
         1931  +    sqlite3_int64 iVal;
         1932  +    *pp += sqlite3Fts3GetVarint(*pp, &iVal);
         1933  +    if( bDescIdx ){
         1934  +      *pVal -= iVal;
         1935  +    }else{
         1936  +      *pVal += iVal;
         1937  +    }
         1938  +  }
         1939  +}
         1940  +
         1941  +static void fts3PutDeltaVarint3(
         1942  +  char **pp,                      /* IN/OUT: Output pointer */
         1943  +  int bDescIdx,                   /* True for descending docids */
         1944  +  sqlite3_int64 *piPrev,          /* IN/OUT: Previous value written to list */
         1945  +  int *pbFirst,                   /* IN/OUT: True after first int written */
         1946  +  sqlite3_int64 iVal              /* Write this value to the list */
         1947  +){
         1948  +  sqlite3_int64 iWrite;
         1949  +  if( bDescIdx==0 || *pbFirst==0 ){
         1950  +    iWrite = iVal - *piPrev;
         1951  +  }else{
         1952  +    iWrite = *piPrev - iVal;
         1953  +  }
         1954  +  assert( *pbFirst || *piPrev==0 );
         1955  +  assert( *pbFirst==0 || iWrite>0 );
         1956  +  *pp += sqlite3Fts3PutVarint(*pp, iWrite);
         1957  +  *piPrev = iVal;
         1958  +  *pbFirst = 1;
         1959  +}
         1960  +
         1961  +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
         1962  +
         1963  +static int fts3DoclistOrMerge(
         1964  +  int bDescIdx,                   /* True if arguments are desc */
         1965  +  char *a1, int n1,               /* First doclist */
         1966  +  char *a2, int n2,               /* Second doclist */
         1967  +  char **paOut, int *pnOut        /* OUT: Malloc'd doclist */
         1968  +){
         1969  +  sqlite3_int64 i1 = 0;
         1970  +  sqlite3_int64 i2 = 0;
         1971  +  sqlite3_int64 iPrev = 0;
         1972  +  char *pEnd1 = &a1[n1];
         1973  +  char *pEnd2 = &a2[n2];
         1974  +  char *p1 = a1;
         1975  +  char *p2 = a2;
         1976  +  char *p;
         1977  +  char *aOut;
         1978  +  int bFirstOut = 0;
         1979  +
         1980  +  *paOut = 0;
         1981  +  *pnOut = 0;
         1982  +  aOut = sqlite3_malloc(n1+n2);
         1983  +  if( !aOut ) return SQLITE_NOMEM;
         1984  +
         1985  +  p = aOut;
         1986  +  fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
         1987  +  fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
         1988  +  while( p1 || p2 ){
         1989  +    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         1990  +
         1991  +    if( p2 && p1 && iDiff==0 ){
         1992  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         1993  +      fts3PoslistMerge(&p, &p1, &p2);
         1994  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         1995  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         1996  +    }else if( !p2 || (p1 && iDiff<0) ){
         1997  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         1998  +      fts3PoslistCopy(&p, &p1);
         1999  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2000  +    }else{
         2001  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
         2002  +      fts3PoslistCopy(&p, &p2);
         2003  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2004  +    }
         2005  +  }
         2006  +
         2007  +  *paOut = aOut;
         2008  +  *pnOut = (p-aOut);
         2009  +  return SQLITE_OK;
         2010  +}
         2011  +
         2012  +static void fts3DoclistPhraseMerge(
         2013  +  int bDescIdx,                   /* True if arguments are desc */
         2014  +  int nDist,                      /* Distance from left to right (1=adjacent) */
         2015  +  char *aLeft, int nLeft,         /* Left doclist */
         2016  +  char *aRight, int *pnRight      /* IN/OUT: Right/output doclist */
         2017  +){
         2018  +  sqlite3_int64 i1 = 0;
         2019  +  sqlite3_int64 i2 = 0;
         2020  +  sqlite3_int64 iPrev = 0;
         2021  +  char *pEnd1 = &aLeft[nLeft];
         2022  +  char *pEnd2 = &aRight[*pnRight];
         2023  +  char *p1 = aLeft;
         2024  +  char *p2 = aRight;
         2025  +  char *p;
         2026  +  int bFirstOut = 0;
         2027  +  char *aOut = aRight;
         2028  +
         2029  +  assert( nDist>0 );
         2030  +
         2031  +  p = aOut;
         2032  +  fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
         2033  +  fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
         2034  +
         2035  +  while( p1 && p2 ){
         2036  +    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
         2037  +    if( iDiff==0 ){
         2038  +      char *pSave = p;
         2039  +      sqlite3_int64 iPrevSave = iPrev;
         2040  +      int bFirstOutSave = bFirstOut;
         2041  +
         2042  +      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
         2043  +      if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
         2044  +        p = pSave;
         2045  +        iPrev = iPrevSave;
         2046  +        bFirstOut = bFirstOutSave;
         2047  +      }
         2048  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2049  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2050  +    }else if( iDiff<0 ){
         2051  +      fts3PoslistCopy(0, &p1);
         2052  +      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
         2053  +    }else{
         2054  +      fts3PoslistCopy(0, &p2);
         2055  +      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
         2056  +    }
         2057  +  }
         2058  +
         2059  +  *pnRight = p - aOut;
         2060  +}
         2061  +
  1990   2062   
  1991   2063   /*
  1992   2064   ** Merge all doclists in the TermSelect.aaOutput[] array into a single
  1993   2065   ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
  1994   2066   ** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
  1995   2067   **
  1996   2068   ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
  1997   2069   ** the responsibility of the caller to free any doclists left in the
  1998   2070   ** TermSelect.aaOutput[] array.
  1999   2071   */
  2000         -static int fts3TermSelectMerge(TermSelect *pTS){
  2001         -  int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
         2072  +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
  2002   2073     char *aOut = 0;
  2003   2074     int nOut = 0;
  2004   2075     int i;
  2005   2076   
  2006   2077     /* Loop through the doclists in the aaOutput[] array. Merge them all
  2007   2078     ** into a single doclist.
  2008   2079     */
................................................................................
  2009   2080     for(i=0; i<SizeofArray(pTS->aaOutput); i++){
  2010   2081       if( pTS->aaOutput[i] ){
  2011   2082         if( !aOut ){
  2012   2083           aOut = pTS->aaOutput[i];
  2013   2084           nOut = pTS->anOutput[i];
  2014   2085           pTS->aaOutput[i] = 0;
  2015   2086         }else{
  2016         -        int nNew = nOut + pTS->anOutput[i];
  2017         -        char *aNew = sqlite3_malloc(nNew);
  2018         -        if( !aNew ){
         2087  +        int nNew;
         2088  +        char *aNew;
         2089  +
         2090  +        int rc = fts3DoclistOrMerge(p->bDescIdx, 
         2091  +            pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
         2092  +        );
         2093  +        if( rc!=SQLITE_OK ){
  2019   2094             sqlite3_free(aOut);
  2020         -          return SQLITE_NOMEM;
         2095  +          return rc;
  2021   2096           }
  2022         -        fts3DoclistMerge(mergetype, 0, 0,
  2023         -            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
  2024         -        );
         2097  +
  2025   2098           sqlite3_free(pTS->aaOutput[i]);
  2026   2099           sqlite3_free(aOut);
  2027   2100           pTS->aaOutput[i] = 0;
  2028   2101           aOut = aNew;
  2029   2102           nOut = nNew;
  2030   2103         }
  2031   2104       }
................................................................................
  2053   2126   
  2054   2127     UNUSED_PARAMETER(p);
  2055   2128     UNUSED_PARAMETER(zTerm);
  2056   2129     UNUSED_PARAMETER(nTerm);
  2057   2130   
  2058   2131     if( pTS->aaOutput[0]==0 ){
  2059   2132       /* If this is the first term selected, copy the doclist to the output
  2060         -    ** buffer using memcpy(). TODO: Add a way to transfer control of the
  2061         -    ** aDoclist buffer from the caller so as to avoid the memcpy().
  2062         -    */
         2133  +    ** buffer using memcpy(). */
  2063   2134       pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
  2064   2135       pTS->anOutput[0] = nDoclist;
  2065   2136       if( pTS->aaOutput[0] ){
  2066   2137         memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
  2067   2138       }else{
  2068   2139         return SQLITE_NOMEM;
  2069   2140       }
  2070   2141     }else{
  2071         -    int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
  2072   2142       char *aMerge = aDoclist;
  2073   2143       int nMerge = nDoclist;
  2074   2144       int iOut;
  2075   2145   
  2076   2146       for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
  2077         -      char *aNew;
  2078         -      int nNew;
  2079   2147         if( pTS->aaOutput[iOut]==0 ){
  2080   2148           assert( iOut>0 );
  2081   2149           pTS->aaOutput[iOut] = aMerge;
  2082   2150           pTS->anOutput[iOut] = nMerge;
  2083   2151           break;
  2084         -      }
  2085         -
  2086         -      nNew = nMerge + pTS->anOutput[iOut];
  2087         -      aNew = sqlite3_malloc(nNew);
  2088         -      if( !aNew ){
  2089         -        if( aMerge!=aDoclist ){
  2090         -          sqlite3_free(aMerge);
  2091         -        }
  2092         -        return SQLITE_NOMEM;
  2093         -      }
  2094         -      fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, 
  2095         -          pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
  2096         -      );
  2097         -
  2098         -      if( iOut>0 ) sqlite3_free(aMerge);
  2099         -      sqlite3_free(pTS->aaOutput[iOut]);
  2100         -      pTS->aaOutput[iOut] = 0;
  2101         -
  2102         -      aMerge = aNew;
  2103         -      nMerge = nNew;
  2104         -      if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
  2105         -        pTS->aaOutput[iOut] = aMerge;
  2106         -        pTS->anOutput[iOut] = nMerge;
  2107         -      }
  2108         -    }
  2109         -  }
         2152  +      }else{
         2153  +        char *aNew;
         2154  +        int nNew;
         2155  +
         2156  +        int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, 
         2157  +            pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
         2158  +        );
         2159  +        if( rc!=SQLITE_OK ){
         2160  +          if( aMerge!=aDoclist ) sqlite3_free(aMerge);
         2161  +          return rc;
         2162  +        }
         2163  +
         2164  +        if( aMerge!=aDoclist ) sqlite3_free(aMerge);
         2165  +        sqlite3_free(pTS->aaOutput[iOut]);
         2166  +        pTS->aaOutput[iOut] = 0;
         2167  +  
         2168  +        aMerge = aNew;
         2169  +        nMerge = nNew;
         2170  +        if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
         2171  +          pTS->aaOutput[iOut] = aMerge;
         2172  +          pTS->anOutput[iOut] = nMerge;
         2173  +        }
         2174  +      }
         2175  +    }
         2176  +  }
         2177  +  return SQLITE_OK;
         2178  +}
         2179  +
         2180  +/*
         2181  +** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
         2182  +*/
         2183  +static int fts3SegReaderCursorAppend(
         2184  +  Fts3MultiSegReader *pCsr, 
         2185  +  Fts3SegReader *pNew
         2186  +){
         2187  +  if( (pCsr->nSegment%16)==0 ){
         2188  +    Fts3SegReader **apNew;
         2189  +    int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
         2190  +    apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
         2191  +    if( !apNew ){
         2192  +      sqlite3Fts3SegReaderFree(pNew);
         2193  +      return SQLITE_NOMEM;
         2194  +    }
         2195  +    pCsr->apSegment = apNew;
         2196  +  }
         2197  +  pCsr->apSegment[pCsr->nSegment++] = pNew;
  2110   2198     return SQLITE_OK;
  2111   2199   }
  2112   2200   
  2113         -static int fts3DeferredTermSelect(
  2114         -  Fts3DeferredToken *pToken,      /* Phrase token */
  2115         -  int isTermPos,                  /* True to include positions */
  2116         -  int *pnOut,                     /* OUT: Size of list */
  2117         -  char **ppOut                    /* OUT: Body of list */
  2118         -){
  2119         -  char *aSource;
  2120         -  int nSource;
  2121         -
  2122         -  aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
  2123         -  if( !aSource ){
  2124         -    *pnOut = 0;
  2125         -    *ppOut = 0;
  2126         -  }else if( isTermPos ){
  2127         -    *ppOut = sqlite3_malloc(nSource);
  2128         -    if( !*ppOut ) return SQLITE_NOMEM;
  2129         -    memcpy(*ppOut, aSource, nSource);
  2130         -    *pnOut = nSource;
  2131         -  }else{
  2132         -    sqlite3_int64 docid;
  2133         -    *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
  2134         -    *ppOut = sqlite3_malloc(*pnOut);
  2135         -    if( !*ppOut ) return SQLITE_NOMEM;
  2136         -    sqlite3Fts3PutVarint(*ppOut, docid);
  2137         -  }
  2138         -
  2139         -  return SQLITE_OK;
  2140         -}
  2141         -
  2142         -int sqlite3Fts3SegReaderCursor(
         2201  +static int fts3SegReaderCursor(
  2143   2202     Fts3Table *p,                   /* FTS3 table handle */
         2203  +  int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
  2144   2204     int iLevel,                     /* Level of segments to scan */
  2145   2205     const char *zTerm,              /* Term to query for */
  2146   2206     int nTerm,                      /* Size of zTerm in bytes */
  2147   2207     int isPrefix,                   /* True for a prefix search */
  2148   2208     int isScan,                     /* True to scan from zTerm to EOF */
  2149         -  Fts3SegReaderCursor *pCsr       /* Cursor object to populate */
         2209  +  Fts3MultiSegReader *pCsr       /* Cursor object to populate */
  2150   2210   ){
  2151   2211     int rc = SQLITE_OK;
  2152   2212     int rc2;
  2153         -  int iAge = 0;
  2154   2213     sqlite3_stmt *pStmt = 0;
  2155         -  Fts3SegReader *pPending = 0;
  2156   2214   
  2157         -  assert( iLevel==FTS3_SEGCURSOR_ALL 
  2158         -      ||  iLevel==FTS3_SEGCURSOR_PENDING 
  2159         -      ||  iLevel>=0
  2160         -  );
  2161         -  assert( FTS3_SEGCURSOR_PENDING<0 );
  2162         -  assert( FTS3_SEGCURSOR_ALL<0 );
  2163         -  assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) );
  2164         -  assert( isPrefix==0 || isScan==0 );
  2165         -
  2166         -
  2167         -  memset(pCsr, 0, sizeof(Fts3SegReaderCursor));
  2168         -
  2169         -  /* If iLevel is less than 0, include a seg-reader for the pending-terms. */
  2170         -  assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 );
  2171         -  if( iLevel<0 && isScan==0 ){
  2172         -    rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending);
  2173         -    if( rc==SQLITE_OK && pPending ){
  2174         -      int nByte = (sizeof(Fts3SegReader *) * 16);
  2175         -      pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
  2176         -      if( pCsr->apSegment==0 ){
  2177         -        rc = SQLITE_NOMEM;
  2178         -      }else{
  2179         -        pCsr->apSegment[0] = pPending;
  2180         -        pCsr->nSegment = 1;
  2181         -        pPending = 0;
  2182         -      }
         2215  +  /* If iLevel is less than 0 and this is not a scan, include a seg-reader 
         2216  +  ** for the pending-terms. If this is a scan, then this call must be being
         2217  +  ** made by an fts4aux module, not an FTS table. In this case calling
         2218  +  ** Fts3SegReaderPending might segfault, as the data structures used by 
         2219  +  ** fts4aux are not completely populated. So it's easiest to filter these
         2220  +  ** calls out here.  */
         2221  +  if( iLevel<0 && p->aIndex ){
         2222  +    Fts3SegReader *pSeg = 0;
         2223  +    rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
         2224  +    if( rc==SQLITE_OK && pSeg ){
         2225  +      rc = fts3SegReaderCursorAppend(pCsr, pSeg);
  2183   2226       }
  2184   2227     }
  2185   2228   
  2186   2229     if( iLevel!=FTS3_SEGCURSOR_PENDING ){
  2187   2230       if( rc==SQLITE_OK ){
  2188         -      rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt);
         2231  +      rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
  2189   2232       }
         2233  +
  2190   2234       while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
         2235  +      Fts3SegReader *pSeg = 0;
  2191   2236   
  2192   2237         /* Read the values returned by the SELECT into local variables. */
  2193   2238         sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
  2194   2239         sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
  2195   2240         sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
  2196   2241         int nRoot = sqlite3_column_bytes(pStmt, 4);
  2197   2242         char const *zRoot = sqlite3_column_blob(pStmt, 4);
  2198   2243   
  2199         -      /* If nSegment is a multiple of 16 the array needs to be extended. */
  2200         -      if( (pCsr->nSegment%16)==0 ){
  2201         -        Fts3SegReader **apNew;
  2202         -        int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
  2203         -        apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
  2204         -        if( !apNew ){
  2205         -          rc = SQLITE_NOMEM;
  2206         -          goto finished;
  2207         -        }
  2208         -        pCsr->apSegment = apNew;
  2209         -      }
  2210         -
  2211   2244         /* If zTerm is not NULL, and this segment is not stored entirely on its
  2212   2245         ** root node, the range of leaves scanned can be reduced. Do this. */
  2213   2246         if( iStartBlock && zTerm ){
  2214   2247           sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
  2215   2248           rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
  2216   2249           if( rc!=SQLITE_OK ) goto finished;
  2217   2250           if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
  2218   2251         }
  2219   2252    
  2220         -      rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock,
  2221         -          iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment]
         2253  +      rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, 
         2254  +          iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
  2222   2255         );
  2223   2256         if( rc!=SQLITE_OK ) goto finished;
  2224         -      pCsr->nSegment++;
  2225         -      iAge++;
         2257  +      rc = fts3SegReaderCursorAppend(pCsr, pSeg);
  2226   2258       }
  2227   2259     }
  2228   2260   
  2229   2261    finished:
  2230   2262     rc2 = sqlite3_reset(pStmt);
  2231   2263     if( rc==SQLITE_DONE ) rc = rc2;
  2232         -  sqlite3Fts3SegReaderFree(pPending);
  2233   2264   
  2234   2265     return rc;
  2235   2266   }
  2236   2267   
         2268  +/*
         2269  +** Set up a cursor object for iterating through a full-text index or a 
         2270  +** single level therein.
         2271  +*/
         2272  +int sqlite3Fts3SegReaderCursor(
         2273  +  Fts3Table *p,                   /* FTS3 table handle */
         2274  +  int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
         2275  +  int iLevel,                     /* Level of segments to scan */
         2276  +  const char *zTerm,              /* Term to query for */
         2277  +  int nTerm,                      /* Size of zTerm in bytes */
         2278  +  int isPrefix,                   /* True for a prefix search */
         2279  +  int isScan,                     /* True to scan from zTerm to EOF */
         2280  +  Fts3MultiSegReader *pCsr       /* Cursor object to populate */
         2281  +){
         2282  +  assert( iIndex>=0 && iIndex<p->nIndex );
         2283  +  assert( iLevel==FTS3_SEGCURSOR_ALL
         2284  +      ||  iLevel==FTS3_SEGCURSOR_PENDING 
         2285  +      ||  iLevel>=0
         2286  +  );
         2287  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
         2288  +  assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
         2289  +  assert( isPrefix==0 || isScan==0 );
  2237   2290   
  2238         -static int fts3TermSegReaderCursor(
         2291  +  /* "isScan" is only set to true by the ft4aux module, an ordinary
         2292  +  ** full-text tables. */
         2293  +  assert( isScan==0 || p->aIndex==0 );
         2294  +
         2295  +  memset(pCsr, 0, sizeof(Fts3MultiSegReader));
         2296  +
         2297  +  return fts3SegReaderCursor(
         2298  +      p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
         2299  +  );
         2300  +}
         2301  +
         2302  +static int fts3SegReaderCursorAddZero(
         2303  +  Fts3Table *p,
         2304  +  const char *zTerm,
         2305  +  int nTerm,
         2306  +  Fts3MultiSegReader *pCsr
         2307  +){
         2308  +  return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
         2309  +}
         2310  +
         2311  +
         2312  +int sqlite3Fts3TermSegReaderCursor(
  2239   2313     Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  2240   2314     const char *zTerm,              /* Term to query for */
  2241   2315     int nTerm,                      /* Size of zTerm in bytes */
  2242   2316     int isPrefix,                   /* True for a prefix search */
  2243         -  Fts3SegReaderCursor **ppSegcsr  /* OUT: Allocated seg-reader cursor */
         2317  +  Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
  2244   2318   ){
  2245         -  Fts3SegReaderCursor *pSegcsr;   /* Object to allocate and return */
         2319  +  Fts3MultiSegReader *pSegcsr;   /* Object to allocate and return */
  2246   2320     int rc = SQLITE_NOMEM;          /* Return code */
  2247   2321   
  2248         -  pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor));
         2322  +  pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
  2249   2323     if( pSegcsr ){
  2250         -    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  2251   2324       int i;
  2252         -    int nCost = 0;
  2253         -    rc = sqlite3Fts3SegReaderCursor(
  2254         -        p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr);
  2255         -  
  2256         -    for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){
  2257         -      rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost);
         2325  +    int bFound = 0;               /* True once an index has been found */
         2326  +    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         2327  +
         2328  +    if( isPrefix ){
         2329  +      for(i=1; bFound==0 && i<p->nIndex; i++){
         2330  +        if( p->aIndex[i].nPrefix==nTerm ){
         2331  +          bFound = 1;
         2332  +          rc = sqlite3Fts3SegReaderCursor(
         2333  +              p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
         2334  +          pSegcsr->bLookup = 1;
         2335  +        }
         2336  +      }
         2337  +
         2338  +      for(i=1; bFound==0 && i<p->nIndex; i++){
         2339  +        if( p->aIndex[i].nPrefix==nTerm+1 ){
         2340  +          bFound = 1;
         2341  +          rc = sqlite3Fts3SegReaderCursor(
         2342  +              p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
         2343  +          );
         2344  +          if( rc==SQLITE_OK ){
         2345  +            rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr);
         2346  +          }
         2347  +        }
         2348  +      }
  2258   2349       }
  2259         -    pSegcsr->nCost = nCost;
         2350  +
         2351  +    if( bFound==0 ){
         2352  +      rc = sqlite3Fts3SegReaderCursor(
         2353  +          p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
         2354  +      );
         2355  +      pSegcsr->bLookup = !isPrefix;
         2356  +    }
  2260   2357     }
  2261   2358   
  2262   2359     *ppSegcsr = pSegcsr;
  2263   2360     return rc;
  2264   2361   }
  2265   2362   
  2266         -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){
         2363  +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
  2267   2364     sqlite3Fts3SegReaderFinish(pSegcsr);
  2268   2365     sqlite3_free(pSegcsr);
  2269   2366   }
  2270   2367   
  2271   2368   /*
  2272   2369   ** This function retreives the doclist for the specified term (or term
  2273   2370   ** prefix) from the database. 
................................................................................
  2284   2381     Fts3PhraseToken *pTok,          /* Token to query for */
  2285   2382     int iColumn,                    /* Column to query (or -ve for all columns) */
  2286   2383     int isReqPos,                   /* True to include position lists in output */
  2287   2384     int *pnOut,                     /* OUT: Size of buffer at *ppOut */
  2288   2385     char **ppOut                    /* OUT: Malloced result buffer */
  2289   2386   ){
  2290   2387     int rc;                         /* Return code */
  2291         -  Fts3SegReaderCursor *pSegcsr;   /* Seg-reader cursor for this term */
         2388  +  Fts3MultiSegReader *pSegcsr;   /* Seg-reader cursor for this term */
  2292   2389     TermSelect tsc;                 /* Context object for fts3TermSelectCb() */
  2293   2390     Fts3SegFilter filter;           /* Segment term filter configuration */
  2294   2391   
  2295   2392     pSegcsr = pTok->pSegcsr;
  2296   2393     memset(&tsc, 0, sizeof(TermSelect));
  2297   2394     tsc.isReqPos = isReqPos;
  2298   2395   
................................................................................
  2310   2407     ){
  2311   2408       rc = fts3TermSelectCb(p, (void *)&tsc, 
  2312   2409           pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
  2313   2410       );
  2314   2411     }
  2315   2412   
  2316   2413     if( rc==SQLITE_OK ){
  2317         -    rc = fts3TermSelectMerge(&tsc);
         2414  +    rc = fts3TermSelectMerge(p, &tsc);
  2318   2415     }
  2319   2416     if( rc==SQLITE_OK ){
  2320   2417       *ppOut = tsc.aaOutput[0];
  2321   2418       *pnOut = tsc.anOutput[0];
  2322   2419     }else{
  2323   2420       int i;
  2324   2421       for(i=0; i<SizeofArray(tsc.aaOutput); i++){
................................................................................
  2360   2457         }
  2361   2458       }
  2362   2459     }
  2363   2460   
  2364   2461     return nDoc;
  2365   2462   }
  2366   2463   
  2367         -/*
  2368         -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
  2369         -*/
  2370         -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
  2371         -  int rc = SQLITE_OK;
  2372         -  if( pExpr ){
  2373         -    rc = fts3DeferExpression(pCsr, pExpr->pLeft);
  2374         -    if( rc==SQLITE_OK ){
  2375         -      rc = fts3DeferExpression(pCsr, pExpr->pRight);
  2376         -    }
  2377         -    if( pExpr->eType==FTSQUERY_PHRASE ){
  2378         -      int iCol = pExpr->pPhrase->iColumn;
  2379         -      int i;
  2380         -      for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
  2381         -        Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
  2382         -        if( pToken->pDeferred==0 ){
  2383         -          rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
  2384         -        }
  2385         -      }
  2386         -    }
  2387         -  }
  2388         -  return rc;
  2389         -}
  2390         -
  2391         -/*
  2392         -** This function removes the position information from a doclist. When
  2393         -** called, buffer aList (size *pnList bytes) contains a doclist that includes
  2394         -** position information. This function removes the position information so
  2395         -** that aList contains only docids, and adjusts *pnList to reflect the new
  2396         -** (possibly reduced) size of the doclist.
  2397         -*/
  2398         -static void fts3DoclistStripPositions(
  2399         -  char *aList,                    /* IN/OUT: Buffer containing doclist */
  2400         -  int *pnList                     /* IN/OUT: Size of doclist in bytes */
  2401         -){
  2402         -  if( aList ){
  2403         -    char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
  2404         -    char *p = aList;              /* Input cursor */
  2405         -    char *pOut = aList;           /* Output cursor */
  2406         -  
  2407         -    while( p<aEnd ){
  2408         -      sqlite3_int64 delta;
  2409         -      p += sqlite3Fts3GetVarint(p, &delta);
  2410         -      fts3PoslistCopy(0, &p);
  2411         -      pOut += sqlite3Fts3PutVarint(pOut, delta);
  2412         -    }
  2413         -
  2414         -    *pnList = (int)(pOut - aList);
  2415         -  }
  2416         -}
  2417         -
  2418         -/* 
  2419         -** Return a DocList corresponding to the phrase *pPhrase.
  2420         -**
  2421         -** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
  2422         -** then no tokens in the phrase were looked up in the full-text index. This
  2423         -** is only possible when this function is called from within xFilter(). The
  2424         -** caller should assume that all documents match the phrase. The actual
  2425         -** filtering will take place in xNext().
  2426         -*/
  2427         -static int fts3PhraseSelect(
  2428         -  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  2429         -  Fts3Phrase *pPhrase,            /* Phrase to return a doclist for */
  2430         -  int isReqPos,                   /* True if output should contain positions */
  2431         -  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  2432         -  int *pnOut                      /* OUT: Size of buffer at *paOut */
  2433         -){
  2434         -  char *pOut = 0;
  2435         -  int nOut = 0;
  2436         -  int rc = SQLITE_OK;
  2437         -  int ii;
  2438         -  int iCol = pPhrase->iColumn;
  2439         -  int isTermPos = (pPhrase->nToken>1 || isReqPos);
  2440         -  Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
  2441         -  int isFirst = 1;
  2442         -
  2443         -  int iPrevTok = 0;
  2444         -  int nDoc = 0;
  2445         -
  2446         -  /* If this is an xFilter() evaluation, create a segment-reader for each
  2447         -  ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
  2448         -  ** evaluation, only create segment-readers if there are no Fts3DeferredToken
  2449         -  ** objects attached to the phrase-tokens.
  2450         -  */
  2451         -  for(ii=0; ii<pPhrase->nToken; ii++){
  2452         -    Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
  2453         -    if( pTok->pSegcsr==0 ){
  2454         -      if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
  2455         -       || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) 
  2456         -       || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) 
  2457         -      ){
  2458         -        rc = fts3TermSegReaderCursor(
  2459         -            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
  2460         -        );
  2461         -        if( rc!=SQLITE_OK ) return rc;
  2462         -      }
  2463         -    }
  2464         -  }
  2465         -
  2466         -  for(ii=0; ii<pPhrase->nToken; ii++){
  2467         -    Fts3PhraseToken *pTok;        /* Token to find doclist for */
  2468         -    int iTok = 0;                 /* The token being queried this iteration */
  2469         -    char *pList = 0;              /* Pointer to token doclist */
  2470         -    int nList = 0;                /* Size of buffer at pList */
  2471         -
  2472         -    /* Select a token to process. If this is an xFilter() call, then tokens 
  2473         -    ** are processed in order from least to most costly. Otherwise, tokens 
  2474         -    ** are processed in the order in which they occur in the phrase.
  2475         -    */
  2476         -    if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
  2477         -      assert( isReqPos );
  2478         -      iTok = ii;
  2479         -      pTok = &pPhrase->aToken[iTok];
  2480         -      if( pTok->bFulltext==0 ) continue;
  2481         -    }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
  2482         -      iTok = ii;
  2483         -      pTok = &pPhrase->aToken[iTok];
  2484         -    }else{
  2485         -      int nMinCost = 0x7FFFFFFF;
  2486         -      int jj;
  2487         -
  2488         -      /* Find the remaining token with the lowest cost. */
  2489         -      for(jj=0; jj<pPhrase->nToken; jj++){
  2490         -        Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr;
  2491         -        if( pSegcsr && pSegcsr->nCost<nMinCost ){
  2492         -          iTok = jj;
  2493         -          nMinCost = pSegcsr->nCost;
  2494         -        }
  2495         -      }
  2496         -      pTok = &pPhrase->aToken[iTok];
  2497         -
  2498         -      /* This branch is taken if it is determined that loading the doclist
  2499         -      ** for the next token would require more IO than loading all documents
  2500         -      ** currently identified by doclist pOut/nOut. No further doclists will
  2501         -      ** be loaded from the full-text index for this phrase.
  2502         -      */
  2503         -      if( nMinCost>nDoc && ii>0 ){
  2504         -        rc = fts3DeferExpression(pCsr, pCsr->pExpr);
  2505         -        break;
  2506         -      }
  2507         -    }
  2508         -
  2509         -    if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
  2510         -      rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
  2511         -    }else{
  2512         -      if( pTok->pSegcsr ){
  2513         -        rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
  2514         -      }
  2515         -      pTok->bFulltext = 1;
  2516         -    }
  2517         -    assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 );
  2518         -    if( rc!=SQLITE_OK ) break;
  2519         -
  2520         -    if( isFirst ){
  2521         -      pOut = pList;
  2522         -      nOut = nList;
  2523         -      if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
  2524         -        nDoc = fts3DoclistCountDocids(1, pOut, nOut);
  2525         -      }
  2526         -      isFirst = 0;
  2527         -      iPrevTok = iTok;
  2528         -    }else{
  2529         -      /* Merge the new term list and the current output. */
  2530         -      char *aLeft, *aRight;
  2531         -      int nLeft, nRight;
  2532         -      int nDist;
  2533         -      int mt;
  2534         -
  2535         -      /* If this is the final token of the phrase, and positions were not
  2536         -      ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
  2537         -      ** This drops the position information from the output list.
  2538         -      */
  2539         -      mt = MERGE_POS_PHRASE;
  2540         -      if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
  2541         -
  2542         -      assert( iPrevTok!=iTok );
  2543         -      if( iPrevTok<iTok ){
  2544         -        aLeft = pOut;
  2545         -        nLeft = nOut;
  2546         -        aRight = pList;
  2547         -        nRight = nList;
  2548         -        nDist = iTok-iPrevTok;
  2549         -        iPrevTok = iTok;
  2550         -      }else{
  2551         -        aRight = pOut;
  2552         -        nRight = nOut;
  2553         -        aLeft = pList;
  2554         -        nLeft = nList;
  2555         -        nDist = iPrevTok-iTok;
  2556         -      }
  2557         -      pOut = aRight;
  2558         -      fts3DoclistMerge(
  2559         -          mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
  2560         -      );
  2561         -      sqlite3_free(aLeft);
  2562         -    }
  2563         -    assert( nOut==0 || pOut!=0 );
  2564         -  }
  2565         -
  2566         -  if( rc==SQLITE_OK ){
  2567         -    if( ii!=pPhrase->nToken ){
  2568         -      assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
  2569         -      fts3DoclistStripPositions(pOut, &nOut);
  2570         -    }
  2571         -    *paOut = pOut;
  2572         -    *pnOut = nOut;
  2573         -  }else{
  2574         -    sqlite3_free(pOut);
  2575         -  }
  2576         -  return rc;
  2577         -}
  2578         -
  2579         -/*
  2580         -** This function merges two doclists according to the requirements of a
  2581         -** NEAR operator.
  2582         -**
  2583         -** Both input doclists must include position information. The output doclist 
  2584         -** includes position information if the first argument to this function
  2585         -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
  2586         -*/
  2587         -static int fts3NearMerge(
  2588         -  int mergetype,                  /* MERGE_POS_NEAR or MERGE_NEAR */
  2589         -  int nNear,                      /* Parameter to NEAR operator */
  2590         -  int nTokenLeft,                 /* Number of tokens in LHS phrase arg */
  2591         -  char *aLeft,                    /* Doclist for LHS (incl. positions) */
  2592         -  int nLeft,                      /* Size of LHS doclist in bytes */
  2593         -  int nTokenRight,                /* As nTokenLeft */
  2594         -  char *aRight,                   /* As aLeft */
  2595         -  int nRight,                     /* As nRight */
  2596         -  char **paOut,                   /* OUT: Results of merge (malloced) */
  2597         -  int *pnOut                      /* OUT: Sized of output buffer */
  2598         -){
  2599         -  char *aOut;                     /* Buffer to write output doclist to */
  2600         -  int rc;                         /* Return code */
  2601         -
  2602         -  assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
  2603         -
  2604         -  aOut = sqlite3_malloc(nLeft+nRight+1);
  2605         -  if( aOut==0 ){
  2606         -    rc = SQLITE_NOMEM;
  2607         -  }else{
  2608         -    rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, 
  2609         -      aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
  2610         -    );
  2611         -    if( rc!=SQLITE_OK ){
  2612         -      sqlite3_free(aOut);
  2613         -      aOut = 0;
  2614         -    }
  2615         -  }
  2616         -
  2617         -  *paOut = aOut;
  2618         -  return rc;
  2619         -}
  2620         -
  2621         -/*
  2622         -** This function is used as part of the processing for the snippet() and
  2623         -** offsets() functions.
  2624         -**
  2625         -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
  2626         -** have their respective doclists (including position information) loaded
  2627         -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
  2628         -** each doclist that are not within nNear tokens of a corresponding entry
  2629         -** in the other doclist.
  2630         -*/
  2631         -int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
  2632         -  int rc;                         /* Return code */
  2633         -
  2634         -  assert( pLeft->eType==FTSQUERY_PHRASE );
  2635         -  assert( pRight->eType==FTSQUERY_PHRASE );
  2636         -  assert( pLeft->isLoaded && pRight->isLoaded );
  2637         -
  2638         -  if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
  2639         -    sqlite3_free(pLeft->aDoclist);
  2640         -    sqlite3_free(pRight->aDoclist);
  2641         -    pRight->aDoclist = 0;
  2642         -    pLeft->aDoclist = 0;
  2643         -    rc = SQLITE_OK;
  2644         -  }else{
  2645         -    char *aOut;                   /* Buffer in which to assemble new doclist */
  2646         -    int nOut;                     /* Size of buffer aOut in bytes */
  2647         -
  2648         -    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
  2649         -        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
  2650         -        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
  2651         -        &aOut, &nOut
  2652         -    );
  2653         -    if( rc!=SQLITE_OK ) return rc;
  2654         -    sqlite3_free(pRight->aDoclist);
  2655         -    pRight->aDoclist = aOut;
  2656         -    pRight->nDoclist = nOut;
  2657         -
  2658         -    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
  2659         -        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
  2660         -        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
  2661         -        &aOut, &nOut
  2662         -    );
  2663         -    sqlite3_free(pLeft->aDoclist);
  2664         -    pLeft->aDoclist = aOut;
  2665         -    pLeft->nDoclist = nOut;
  2666         -  }
  2667         -  return rc;
  2668         -}
  2669         -
  2670         -
  2671         -/*
  2672         -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. 
  2673         -** The allocated objects are stored in the Fts3PhraseToken.pArray member
  2674         -** variables of each token structure.
  2675         -*/
  2676         -static int fts3ExprAllocateSegReaders(
  2677         -  Fts3Cursor *pCsr,               /* FTS3 table */
  2678         -  Fts3Expr *pExpr,                /* Expression to create seg-readers for */
  2679         -  int *pnExpr                     /* OUT: Number of AND'd expressions */
  2680         -){
  2681         -  int rc = SQLITE_OK;             /* Return code */
  2682         -
  2683         -  assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
  2684         -  if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
  2685         -    (*pnExpr)++;
  2686         -    pnExpr = 0;
  2687         -  }
  2688         -
  2689         -  if( pExpr->eType==FTSQUERY_PHRASE ){
  2690         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
  2691         -    int ii;
  2692         -
  2693         -    for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
  2694         -      Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
  2695         -      if( pTok->pSegcsr==0 ){
  2696         -        rc = fts3TermSegReaderCursor(
  2697         -            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
  2698         -        );
  2699         -      }
  2700         -    }
  2701         -  }else{ 
  2702         -    rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
  2703         -    if( rc==SQLITE_OK ){
  2704         -      rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
  2705         -    }
  2706         -  }
  2707         -  return rc;
  2708         -}
  2709         -
  2710         -/*
  2711         -** Free the Fts3SegReaderArray objects associated with each token in the
  2712         -** expression pExpr. In other words, this function frees the resources
  2713         -** allocated by fts3ExprAllocateSegReaders().
  2714         -*/
  2715         -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
  2716         -  if( pExpr ){
  2717         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
  2718         -    if( pPhrase ){
  2719         -      int kk;
  2720         -      for(kk=0; kk<pPhrase->nToken; kk++){
  2721         -        fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr);
  2722         -        pPhrase->aToken[kk].pSegcsr = 0;
  2723         -      }
  2724         -    }
  2725         -    fts3ExprFreeSegReaders(pExpr->pLeft);
  2726         -    fts3ExprFreeSegReaders(pExpr->pRight);
  2727         -  }
  2728         -}
  2729         -
  2730         -/*
  2731         -** Return the sum of the costs of all tokens in the expression pExpr. This
  2732         -** function must be called after Fts3SegReaderArrays have been allocated
  2733         -** for all tokens using fts3ExprAllocateSegReaders().
  2734         -*/
  2735         -static int fts3ExprCost(Fts3Expr *pExpr){
  2736         -  int nCost;                      /* Return value */
  2737         -  if( pExpr->eType==FTSQUERY_PHRASE ){
  2738         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
  2739         -    int ii;
  2740         -    nCost = 0;
  2741         -    for(ii=0; ii<pPhrase->nToken; ii++){
  2742         -      Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr;
  2743         -      if( pSegcsr ) nCost += pSegcsr->nCost;
  2744         -    }
  2745         -  }else{
  2746         -    nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
  2747         -  }
  2748         -  return nCost;
  2749         -}
  2750         -
  2751         -/*
  2752         -** The following is a helper function (and type) for fts3EvalExpr(). It
  2753         -** must be called after Fts3SegReaders have been allocated for every token
  2754         -** in the expression. See the context it is called from in fts3EvalExpr()
  2755         -** for further explanation.
  2756         -*/
  2757         -typedef struct ExprAndCost ExprAndCost;
  2758         -struct ExprAndCost {
  2759         -  Fts3Expr *pExpr;
  2760         -  int nCost;
  2761         -};
  2762         -static void fts3ExprAssignCosts(
  2763         -  Fts3Expr *pExpr,                /* Expression to create seg-readers for */
  2764         -  ExprAndCost **ppExprCost        /* OUT: Write to *ppExprCost */
  2765         -){
  2766         -  if( pExpr->eType==FTSQUERY_AND ){
  2767         -    fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
  2768         -    fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
  2769         -  }else{
  2770         -    (*ppExprCost)->pExpr = pExpr;
  2771         -    (*ppExprCost)->nCost = fts3ExprCost(pExpr);
  2772         -    (*ppExprCost)++;
  2773         -  }
  2774         -}
  2775         -
  2776         -/*
  2777         -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
  2778         -** the resulting doclist in *paOut and *pnOut. This routine mallocs for
  2779         -** the space needed to store the output. The caller is responsible for
  2780         -** freeing the space when it has finished.
  2781         -**
  2782         -** This function is called in two distinct contexts:
  2783         -**
  2784         -**   * From within the virtual table xFilter() method. In this case, the
  2785         -**     output doclist contains entries for all rows in the table, based on
  2786         -**     data read from the full-text index.
  2787         -**
  2788         -**     In this case, if the query expression contains one or more tokens that 
  2789         -**     are very common, then the returned doclist may contain a superset of 
  2790         -**     the documents that actually match the expression.
  2791         -**
  2792         -**   * From within the virtual table xNext() method. This call is only made
  2793         -**     if the call from within xFilter() found that there were very common 
  2794         -**     tokens in the query expression and did return a superset of the 
  2795         -**     matching documents. In this case the returned doclist contains only
  2796         -**     entries that correspond to the current row of the table. Instead of
  2797         -**     reading the data for each token from the full-text index, the data is
  2798         -**     already available in-memory in the Fts3PhraseToken.pDeferred structures.
  2799         -**     See fts3EvalDeferred() for how it gets there.
  2800         -**
  2801         -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
  2802         -** required) Fts3Cursor.doDeferred==1.
  2803         -**
  2804         -** If the SQLite invokes the snippet(), offsets() or matchinfo() function
  2805         -** as part of a SELECT on an FTS3 table, this function is called on each
  2806         -** individual phrase expression in the query. If there were very common tokens
  2807         -** found in the xFilter() call, then this function is called once for phrase
  2808         -** for each row visited, and the returned doclist contains entries for the
  2809         -** current row only. Otherwise, if there were no very common tokens, then this
  2810         -** function is called once only for each phrase in the query and the returned
  2811         -** doclist contains entries for all rows of the table.
  2812         -**
  2813         -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
  2814         -** result of a snippet(), offsets() or matchinfo() invocation.
  2815         -*/
  2816         -static int fts3EvalExpr(
  2817         -  Fts3Cursor *p,                  /* Virtual table cursor handle */
  2818         -  Fts3Expr *pExpr,                /* Parsed fts3 expression */
  2819         -  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  2820         -  int *pnOut,                     /* OUT: Size of buffer at *paOut */
  2821         -  int isReqPos                    /* Require positions in output buffer */
  2822         -){
  2823         -  int rc = SQLITE_OK;             /* Return code */
  2824         -
  2825         -  /* Zero the output parameters. */
  2826         -  *paOut = 0;
  2827         -  *pnOut = 0;
  2828         -
  2829         -  if( pExpr ){
  2830         -    assert( pExpr->eType==FTSQUERY_NEAR   || pExpr->eType==FTSQUERY_OR     
  2831         -         || pExpr->eType==FTSQUERY_AND    || pExpr->eType==FTSQUERY_NOT
  2832         -         || pExpr->eType==FTSQUERY_PHRASE
  2833         -    );
  2834         -    assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
  2835         -
  2836         -    if( pExpr->eType==FTSQUERY_PHRASE ){
  2837         -      rc = fts3PhraseSelect(p, pExpr->pPhrase,
  2838         -          isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
  2839         -          paOut, pnOut
  2840         -      );
  2841         -      fts3ExprFreeSegReaders(pExpr);
  2842         -    }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
  2843         -      ExprAndCost *aExpr = 0;     /* Array of AND'd expressions and costs */
  2844         -      int nExpr = 0;              /* Size of aExpr[] */
  2845         -      char *aRet = 0;             /* Doclist to return to caller */
  2846         -      int nRet = 0;               /* Length of aRet[] in bytes */
  2847         -      int nDoc = 0x7FFFFFFF;
  2848         -
  2849         -      assert( !isReqPos );
  2850         -
  2851         -      rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
  2852         -      if( rc==SQLITE_OK ){
  2853         -        assert( nExpr>1 );
  2854         -        aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
  2855         -        if( !aExpr ) rc = SQLITE_NOMEM;
  2856         -      }
  2857         -      if( rc==SQLITE_OK ){
  2858         -        int ii;                   /* Used to iterate through expressions */
  2859         -
  2860         -        fts3ExprAssignCosts(pExpr, &aExpr);
  2861         -        aExpr -= nExpr;
  2862         -        for(ii=0; ii<nExpr; ii++){
  2863         -          char *aNew;
  2864         -          int nNew;
  2865         -          int jj;
  2866         -          ExprAndCost *pBest = 0;
  2867         -  
  2868         -          for(jj=0; jj<nExpr; jj++){
  2869         -            ExprAndCost *pCand = &aExpr[jj];
  2870         -            if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
  2871         -              pBest = pCand;
  2872         -            }
  2873         -          }
  2874         -  
  2875         -          if( pBest->nCost>nDoc ){
  2876         -            rc = fts3DeferExpression(p, p->pExpr);
  2877         -            break;
  2878         -          }else{
  2879         -            rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
  2880         -            if( rc!=SQLITE_OK ) break;
  2881         -            pBest->pExpr = 0;
  2882         -            if( ii==0 ){
  2883         -              aRet = aNew;
  2884         -              nRet = nNew;
  2885         -              nDoc = fts3DoclistCountDocids(0, aRet, nRet);
  2886         -            }else{
  2887         -              fts3DoclistMerge(
  2888         -                  MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
  2889         -              );
  2890         -              sqlite3_free(aNew);
  2891         -            }
  2892         -          }
  2893         -        }
  2894         -      }
  2895         -
  2896         -      if( rc==SQLITE_OK ){
  2897         -        *paOut = aRet;
  2898         -        *pnOut = nRet;
  2899         -      }else{
  2900         -        assert( *paOut==0 );
  2901         -        sqlite3_free(aRet);
  2902         -      }
  2903         -      sqlite3_free(aExpr);
  2904         -      fts3ExprFreeSegReaders(pExpr);
  2905         -
  2906         -    }else{
  2907         -      char *aLeft;
  2908         -      char *aRight;
  2909         -      int nLeft;
  2910         -      int nRight;
  2911         -
  2912         -      assert( pExpr->eType==FTSQUERY_NEAR 
  2913         -           || pExpr->eType==FTSQUERY_OR
  2914         -           || pExpr->eType==FTSQUERY_NOT
  2915         -           || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
  2916         -      );
  2917         -
  2918         -      if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
  2919         -       && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
  2920         -      ){
  2921         -        switch( pExpr->eType ){
  2922         -          case FTSQUERY_NEAR: {
  2923         -            Fts3Expr *pLeft;
  2924         -            Fts3Expr *pRight;
  2925         -            int mergetype = MERGE_NEAR;
  2926         -            if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
  2927         -              mergetype = MERGE_POS_NEAR;
  2928         -            }
  2929         -            pLeft = pExpr->pLeft;
  2930         -            while( pLeft->eType==FTSQUERY_NEAR ){ 
  2931         -              pLeft=pLeft->pRight;
  2932         -            }
  2933         -            pRight = pExpr->pRight;
  2934         -            assert( pRight->eType==FTSQUERY_PHRASE );
  2935         -            assert( pLeft->eType==FTSQUERY_PHRASE );
  2936         -
  2937         -            rc = fts3NearMerge(mergetype, pExpr->nNear, 
  2938         -                pLeft->pPhrase->nToken, aLeft, nLeft,
  2939         -                pRight->pPhrase->nToken, aRight, nRight,
  2940         -                paOut, pnOut
  2941         -            );
  2942         -            sqlite3_free(aLeft);
  2943         -            break;
  2944         -          }
  2945         -
  2946         -          case FTSQUERY_OR: {
  2947         -            /* Allocate a buffer for the output. The maximum size is the
  2948         -            ** sum of the sizes of the two input buffers. The +1 term is
  2949         -            ** so that a buffer of zero bytes is never allocated - this can
  2950         -            ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
  2951         -            */
  2952         -            char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
  2953         -            rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
  2954         -                aLeft, nLeft, aRight, nRight, 0
  2955         -            );
  2956         -            *paOut = aBuffer;
  2957         -            sqlite3_free(aLeft);
  2958         -            break;
  2959         -          }
  2960         -
  2961         -          default: {
  2962         -            assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
  2963         -            fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
  2964         -                aLeft, nLeft, aRight, nRight, 0
  2965         -            );
  2966         -            *paOut = aLeft;
  2967         -            break;
  2968         -          }
  2969         -        }
  2970         -      }
  2971         -      sqlite3_free(aRight);
  2972         -    }
  2973         -  }
  2974         -
  2975         -  assert( rc==SQLITE_OK || *paOut==0 );
  2976         -  return rc;
  2977         -}
  2978         -
  2979         -/*
  2980         -** This function is called from within xNext() for each row visited by
  2981         -** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
  2982         -** was able to determine the exact set of matching rows, this function sets
  2983         -** *pbRes to true and returns SQLITE_IO immediately.
  2984         -**
  2985         -** Otherwise, if evaluating the query expression within xFilter() returned a
  2986         -** superset of the matching documents instead of an exact set (this happens
  2987         -** when the query includes very common tokens and it is deemed too expensive to
  2988         -** load their doclists from disk), this function tests if the current row
  2989         -** really does match the FTS3 query.
  2990         -**
  2991         -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
  2992         -** is returned and *pbRes is set to true if the current row matches the
  2993         -** FTS3 query (and should be included in the results returned to SQLite), or
  2994         -** false otherwise.
  2995         -*/
  2996         -static int fts3EvalDeferred(
  2997         -  Fts3Cursor *pCsr,               /* FTS3 cursor pointing at row to test */
  2998         -  int *pbRes                      /* OUT: Set to true if row is a match */
  2999         -){
  3000         -  int rc = SQLITE_OK;
  3001         -  if( pCsr->pDeferred==0 ){
  3002         -    *pbRes = 1;
  3003         -  }else{
  3004         -    rc = fts3CursorSeek(0, pCsr);
  3005         -    if( rc==SQLITE_OK ){
  3006         -      sqlite3Fts3FreeDeferredDoclists(pCsr);
  3007         -      rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
  3008         -    }
  3009         -    if( rc==SQLITE_OK ){
  3010         -      char *a = 0;
  3011         -      int n = 0;
  3012         -      rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
  3013         -      assert( n>=0 );
  3014         -      *pbRes = (n>0);
  3015         -      sqlite3_free(a);
  3016         -    }
  3017         -  }
  3018         -  return rc;
  3019         -}
  3020         -
  3021   2464   /*
  3022   2465   ** Advance the cursor to the next row in the %_content table that
  3023   2466   ** matches the search criteria.  For a MATCH search, this will be
  3024   2467   ** the next row that matches. For a full-table scan, this will be
  3025   2468   ** simply the next row in the %_content table.  For a docid lookup,
  3026   2469   ** this routine simply sets the EOF flag.
  3027   2470   **
  3028   2471   ** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
  3029   2472   ** even if we reach end-of-file.  The fts3EofMethod() will be called
  3030   2473   ** subsequently to determine whether or not an EOF was hit.
  3031   2474   */
  3032   2475   static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
  3033         -  int res;
  3034         -  int rc = SQLITE_OK;             /* Return code */
         2476  +  int rc;
  3035   2477     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  3036         -
  3037         -  pCsr->eEvalmode = FTS3_EVAL_NEXT;
  3038         -  do {
  3039         -    if( pCsr->aDoclist==0 ){
  3040         -      if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
  3041         -        pCsr->isEof = 1;
  3042         -        rc = sqlite3_reset(pCsr->pStmt);
  3043         -        break;
  3044         -      }
         2478  +  if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
         2479  +    if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
         2480  +      pCsr->isEof = 1;
         2481  +      rc = sqlite3_reset(pCsr->pStmt);
         2482  +    }else{
  3045   2483         pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
  3046         -    }else{
  3047         -      if( pCsr->desc==0 ){
  3048         -        if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
  3049         -          pCsr->isEof = 1;
  3050         -          break;
  3051         -        }
  3052         -        fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
  3053         -      }else{
  3054         -        fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId);
  3055         -        if( pCsr->pNextId<=pCsr->aDoclist ){
  3056         -          pCsr->isEof = 1;
  3057         -          break;
  3058         -        }
  3059         -      }
  3060         -      sqlite3_reset(pCsr->pStmt);
  3061         -      pCsr->isRequireSeek = 1;
  3062         -      pCsr->isMatchinfoNeeded = 1;
         2484  +      rc = SQLITE_OK;
  3063   2485       }
  3064         -  }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
  3065         -
         2486  +  }else{
         2487  +    rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
         2488  +  }
         2489  +  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  3066   2490     return rc;
  3067   2491   }
  3068   2492   
  3069   2493   /*
  3070   2494   ** This is the xFilter interface for the virtual table.  See
  3071   2495   ** the virtual table xFilter method documentation for additional
  3072   2496   ** information.
................................................................................
  3085   2509   static int fts3FilterMethod(
  3086   2510     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  3087   2511     int idxNum,                     /* Strategy index */
  3088   2512     const char *idxStr,             /* Unused */
  3089   2513     int nVal,                       /* Number of elements in apVal */
  3090   2514     sqlite3_value **apVal           /* Arguments for the indexing scheme */
  3091   2515   ){
  3092         -  const char *azSql[] = {
  3093         -    "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?",   /* non-full-scan */
  3094         -    "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */
  3095         -  };
  3096         -  int rc;                         /* Return code */
         2516  +  int rc;
  3097   2517     char *zSql;                     /* SQL statement used to access %_content */
  3098   2518     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  3099   2519     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  3100   2520   
  3101   2521     UNUSED_PARAMETER(idxStr);
  3102   2522     UNUSED_PARAMETER(nVal);
  3103   2523   
................................................................................
  3108   2528   
  3109   2529     /* In case the cursor has been used before, clear it now. */
  3110   2530     sqlite3_finalize(pCsr->pStmt);
  3111   2531     sqlite3_free(pCsr->aDoclist);
  3112   2532     sqlite3Fts3ExprFree(pCsr->pExpr);
  3113   2533     memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
  3114   2534   
         2535  +  if( idxStr ){
         2536  +    pCsr->bDesc = (idxStr[0]=='D');
         2537  +  }else{
         2538  +    pCsr->bDesc = p->bDescIdx;
         2539  +  }
         2540  +  pCsr->eSearch = (i16)idxNum;
         2541  +
  3115   2542     if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
  3116   2543       int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
  3117   2544       const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
  3118   2545   
  3119   2546       if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  3120   2547         return SQLITE_NOMEM;
  3121   2548       }
  3122   2549   
  3123   2550       rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, 
  3124   2551           iCol, zQuery, -1, &pCsr->pExpr
  3125   2552       );
  3126   2553       if( rc!=SQLITE_OK ){
  3127   2554         if( rc==SQLITE_ERROR ){
  3128         -        p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]",
  3129         -                                          zQuery);
         2555  +        static const char *zErr = "malformed MATCH expression: [%s]";
         2556  +        p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
  3130   2557         }
  3131   2558         return rc;
  3132   2559       }
  3133   2560   
  3134   2561       rc = sqlite3Fts3ReadLock(p);
  3135   2562       if( rc!=SQLITE_OK ) return rc;
  3136   2563   
  3137         -    rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
         2564  +    rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
         2565  +
  3138   2566       sqlite3Fts3SegmentsClose(p);
  3139   2567       if( rc!=SQLITE_OK ) return rc;
  3140   2568       pCsr->pNextId = pCsr->aDoclist;
  3141   2569       pCsr->iPrevId = 0;
  3142   2570     }
  3143   2571   
  3144   2572     /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  3145   2573     ** statement loops through all rows of the %_content table. For a
  3146   2574     ** full-text query or docid lookup, the statement retrieves a single
  3147   2575     ** row by docid.
  3148   2576     */
  3149         -  zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH];
  3150         -  zSql = sqlite3_mprintf(
  3151         -      zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC")
  3152         -  );
  3153         -  if( !zSql ){
  3154         -    rc = SQLITE_NOMEM;
         2577  +  if( idxNum==FTS3_FULLSCAN_SEARCH ){
         2578  +    const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
         2579  +    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
         2580  +    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
  3155   2581     }else{
  3156         -    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  3157         -    sqlite3_free(zSql);
         2582  +    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
         2583  +    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
  3158   2584     }
  3159         -  if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
  3160         -    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
  3161         -  }
  3162         -  pCsr->eSearch = (i16)idxNum;
  3163         -
  3164         -  assert( pCsr->desc==0 );
         2585  +  if( !zSql ) return SQLITE_NOMEM;
         2586  +  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         2587  +  sqlite3_free(zSql);
  3165   2588     if( rc!=SQLITE_OK ) return rc;
  3166         -  if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){
  3167         -    sqlite3_int64 iDocid = 0;
  3168         -    char *csr = pCsr->aDoclist;
  3169         -    while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){
  3170         -      fts3GetDeltaVarint(&csr, &iDocid);
  3171         -    }
  3172         -    pCsr->pNextId = csr;
  3173         -    pCsr->iPrevId = iDocid;
  3174         -    pCsr->desc = 1;
  3175         -    pCsr->isRequireSeek = 1;
  3176         -    pCsr->isMatchinfoNeeded = 1;
  3177         -    pCsr->eEvalmode = FTS3_EVAL_NEXT;
  3178         -    return SQLITE_OK;
         2589  +
         2590  +  if( idxNum==FTS3_DOCID_SEARCH ){
         2591  +    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
         2592  +    if( rc!=SQLITE_OK ) return rc;
  3179   2593     }
         2594  +
  3180   2595     return fts3NextMethod(pCursor);
  3181   2596   }
  3182   2597   
  3183   2598   /* 
  3184   2599   ** This is the xEof method of the virtual table. SQLite calls this 
  3185   2600   ** routine to find out if it has reached the end of a result set.
  3186   2601   */
................................................................................
  3192   2607   ** This is the xRowid method. The SQLite core calls this routine to
  3193   2608   ** retrieve the rowid for the current row of the result set. fts3
  3194   2609   ** exposes %_content.docid as the rowid for the virtual table. The
  3195   2610   ** rowid should be written to *pRowid.
  3196   2611   */
  3197   2612   static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  3198   2613     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  3199         -  if( pCsr->aDoclist ){
  3200         -    *pRowid = pCsr->iPrevId;
  3201         -  }else{
  3202         -    /* This branch runs if the query is implemented using a full-table scan
  3203         -    ** (not using the full-text index). In this case grab the rowid from the
  3204         -    ** SELECT statement.
  3205         -    */
  3206         -    assert( pCsr->isRequireSeek==0 );
  3207         -    *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
  3208         -  }
         2614  +  *pRowid = pCsr->iPrevId;
  3209   2615     return SQLITE_OK;
  3210   2616   }
  3211   2617   
  3212   2618   /* 
  3213   2619   ** This is the xColumn method, called by SQLite to request a value from
  3214   2620   ** the row that the supplied cursor currently points to.
  3215   2621   */
  3216   2622   static int fts3ColumnMethod(
  3217   2623     sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
  3218   2624     sqlite3_context *pContext,      /* Context for sqlite3_result_xxx() calls */
  3219   2625     int iCol                        /* Index of column to read value from */
  3220   2626   ){
  3221         -  int rc;                         /* Return Code */
         2627  +  int rc = SQLITE_OK;             /* Return Code */
  3222   2628     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  3223   2629     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  3224   2630   
  3225   2631     /* The column value supplied by SQLite must be in range. */
  3226   2632     assert( iCol>=0 && iCol<=p->nColumn+1 );
  3227   2633   
  3228   2634     if( iCol==p->nColumn+1 ){
  3229   2635       /* This call is a request for the "docid" column. Since "docid" is an 
  3230   2636       ** alias for "rowid", use the xRowid() method to obtain the value.
  3231   2637       */
  3232         -    sqlite3_int64 iRowid;
  3233         -    rc = fts3RowidMethod(pCursor, &iRowid);
  3234         -    sqlite3_result_int64(pContext, iRowid);
         2638  +    sqlite3_result_int64(pContext, pCsr->iPrevId);
  3235   2639     }else if( iCol==p->nColumn ){
  3236   2640       /* The extra column whose name is the same as the table.
  3237   2641       ** Return a blob which is a pointer to the cursor.
  3238   2642       */
  3239   2643       sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  3240         -    rc = SQLITE_OK;
  3241   2644     }else{
  3242   2645       rc = fts3CursorSeek(0, pCsr);
  3243   2646       if( rc==SQLITE_OK ){
  3244   2647         sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
  3245   2648       }
  3246   2649     }
         2650  +
         2651  +  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  3247   2652     return rc;
  3248   2653   }
  3249   2654   
  3250   2655   /* 
  3251   2656   ** This function is the implementation of the xUpdate callback used by 
  3252   2657   ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
  3253   2658   ** inserted, updated or deleted.
................................................................................
  3273   2678   
  3274   2679   /*
  3275   2680   ** Implementation of xBegin() method. This is a no-op.
  3276   2681   */
  3277   2682   static int fts3BeginMethod(sqlite3_vtab *pVtab){
  3278   2683     UNUSED_PARAMETER(pVtab);
  3279   2684     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
         2685  +  assert( p->pSegments==0 );
  3280   2686     assert( p->nPendingData==0 );
  3281   2687     assert( p->inTransaction!=1 );
  3282   2688     TESTONLY( p->inTransaction = 1 );
  3283   2689     TESTONLY( p->mxSavepoint = -1; );
  3284   2690     return SQLITE_OK;
  3285   2691   }
  3286   2692   
................................................................................
  3290   2696   ** by fts3SyncMethod().
  3291   2697   */
  3292   2698   static int fts3CommitMethod(sqlite3_vtab *pVtab){
  3293   2699     UNUSED_PARAMETER(pVtab);
  3294   2700     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  3295   2701     assert( p->nPendingData==0 );
  3296   2702     assert( p->inTransaction!=0 );
         2703  +  assert( p->pSegments==0 );
  3297   2704     TESTONLY( p->inTransaction = 0 );
  3298   2705     TESTONLY( p->mxSavepoint = -1; );
  3299   2706     return SQLITE_OK;
  3300   2707   }
  3301   2708   
  3302   2709   /*
  3303   2710   ** Implementation of xRollback(). Discard the contents of the pending-terms
................................................................................
  3307   2714     Fts3Table *p = (Fts3Table*)pVtab;
  3308   2715     sqlite3Fts3PendingTermsClear(p);
  3309   2716     assert( p->inTransaction!=0 );
  3310   2717     TESTONLY( p->inTransaction = 0 );
  3311   2718     TESTONLY( p->mxSavepoint = -1; );
  3312   2719     return SQLITE_OK;
  3313   2720   }
  3314         -
  3315         -/*
  3316         -** Load the doclist associated with expression pExpr to pExpr->aDoclist.
  3317         -** The loaded doclist contains positions as well as the document ids.
  3318         -** This is used by the matchinfo(), snippet() and offsets() auxillary
  3319         -** functions.
  3320         -*/
  3321         -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
  3322         -  int rc;
  3323         -  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
  3324         -  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
  3325         -  rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
  3326         -  return rc;
  3327         -}
  3328         -
  3329         -int sqlite3Fts3ExprLoadFtDoclist(
  3330         -  Fts3Cursor *pCsr, 
  3331         -  Fts3Expr *pExpr,
  3332         -  char **paDoclist,
  3333         -  int *pnDoclist
  3334         -){
  3335         -  int rc;
  3336         -  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
  3337         -  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
  3338         -  pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
  3339         -  rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
  3340         -  pCsr->eEvalmode = FTS3_EVAL_NEXT;
  3341         -  return rc;
  3342         -}
  3343         -
  3344   2721   
  3345   2722   /*
  3346   2723   ** When called, *ppPoslist must point to the byte immediately following the
  3347   2724   ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
  3348   2725   ** moves *ppPoslist so that it instead points to the first byte of the
  3349   2726   ** same position list.
  3350   2727   */
  3351   2728   static void fts3ReversePoslist(char *pStart, char **ppPoslist){
  3352         -  char *p = &(*ppPoslist)[-3];
  3353         -  char c = p[1];
         2729  +  char *p = &(*ppPoslist)[-2];
         2730  +  char c;
         2731  +
         2732  +  while( p>pStart && (c=*p--)==0 );
  3354   2733     while( p>pStart && (*p & 0x80) | c ){ 
  3355   2734       c = *p--; 
  3356   2735     }
  3357   2736     if( p>pStart ){ p = &p[2]; }
  3358   2737     while( *p++&0x80 );
  3359   2738     *ppPoslist = p;
  3360   2739   }
  3361   2740   
  3362         -
  3363         -/*
  3364         -** After ExprLoadDoclist() (see above) has been called, this function is
  3365         -** used to iterate/search through the position lists that make up the doclist
  3366         -** stored in pExpr->aDoclist.
  3367         -*/
  3368         -char *sqlite3Fts3FindPositions(
  3369         -  Fts3Cursor *pCursor,            /* Associate FTS3 cursor */
  3370         -  Fts3Expr *pExpr,                /* Access this expressions doclist */
  3371         -  sqlite3_int64 iDocid,           /* Docid associated with requested pos-list */
  3372         -  int iCol                        /* Column of requested pos-list */
  3373         -){
  3374         -  assert( pExpr->isLoaded );
  3375         -  if( pExpr->aDoclist ){
  3376         -    char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
  3377         -    char *pCsr;
  3378         -
  3379         -    if( pExpr->pCurrent==0 ){
  3380         -      if( pCursor->desc==0 ){
  3381         -        pExpr->pCurrent = pExpr->aDoclist;
  3382         -        pExpr->iCurrent = 0;
  3383         -        fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent);
  3384         -      }else{
  3385         -        pCsr = pExpr->aDoclist;
  3386         -        while( pCsr<pEnd ){
  3387         -          fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
  3388         -          fts3PoslistCopy(0, &pCsr);
  3389         -        }
  3390         -        fts3ReversePoslist(pExpr->aDoclist, &pCsr);
  3391         -        pExpr->pCurrent = pCsr;
  3392         -      }
  3393         -    }
  3394         -    pCsr = pExpr->pCurrent;
  3395         -    assert( pCsr );
  3396         -
  3397         -    while( (pCursor->desc==0 && pCsr<pEnd) 
  3398         -        || (pCursor->desc && pCsr>pExpr->aDoclist) 
  3399         -    ){
  3400         -      if( pCursor->desc==0 && pExpr->iCurrent<iDocid ){
  3401         -        fts3PoslistCopy(0, &pCsr);
  3402         -        if( pCsr<pEnd ){
  3403         -          fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
  3404         -        }
  3405         -        pExpr->pCurrent = pCsr;
  3406         -      }else if( pCursor->desc && pExpr->iCurrent>iDocid ){
  3407         -        fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent);
  3408         -        fts3ReversePoslist(pExpr->aDoclist, &pCsr);
  3409         -        pExpr->pCurrent = pCsr;
  3410         -      }else{
  3411         -        if( pExpr->iCurrent==iDocid ){
  3412         -          int iThis = 0;
  3413         -          if( iCol<0 ){
  3414         -            /* If iCol is negative, return a pointer to the start of the
  3415         -            ** position-list (instead of a pointer to the start of a list
  3416         -            ** of offsets associated with a specific column).
  3417         -            */
  3418         -            return pCsr;
  3419         -          }
  3420         -          while( iThis<iCol ){
  3421         -            fts3ColumnlistCopy(0, &pCsr);
  3422         -            if( *pCsr==0x00 ) return 0;
  3423         -            pCsr++;
  3424         -            pCsr += sqlite3Fts3GetVarint32(pCsr, &iThis);
  3425         -          }
  3426         -          if( iCol==iThis && (*pCsr&0xFE) ) return pCsr;
  3427         -        }
  3428         -        return 0;
  3429         -      }
  3430         -    }
  3431         -  }
  3432         -
  3433         -  return 0;
  3434         -}
  3435         -
  3436   2741   /*
  3437   2742   ** Helper function used by the implementation of the overloaded snippet(),
  3438   2743   ** offsets() and optimize() SQL functions.
  3439   2744   **
  3440   2745   ** If the value passed as the third argument is a blob of size
  3441   2746   ** sizeof(Fts3Cursor*), then the blob contents are copied to the 
  3442   2747   ** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
................................................................................
  3660   2965       "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
  3661   2966       p->zDb, p->zName, zName
  3662   2967     );
  3663   2968     return rc;
  3664   2969   }
  3665   2970   
  3666   2971   static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  3667         -  Fts3Table *p = (Fts3Table*)pVtab;
  3668   2972     UNUSED_PARAMETER(iSavepoint);
  3669         -  assert( p->inTransaction );
  3670         -  assert( p->mxSavepoint < iSavepoint );
  3671         -  TESTONLY( p->mxSavepoint = iSavepoint );
  3672         -  return sqlite3Fts3PendingTermsFlush(p);
         2973  +  assert( ((Fts3Table *)pVtab)->inTransaction );
         2974  +  assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
         2975  +  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
         2976  +  return fts3SyncMethod(pVtab);
  3673   2977   }
  3674   2978   static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  3675   2979     TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  3676   2980     UNUSED_PARAMETER(iSavepoint);
  3677   2981     UNUSED_PARAMETER(pVtab);
  3678   2982     assert( p->inTransaction );
  3679   2983     assert( p->mxSavepoint >= iSavepoint );
................................................................................
  3836   3140     const sqlite3_api_routines *pApi
  3837   3141   ){
  3838   3142     SQLITE_EXTENSION_INIT2(pApi)
  3839   3143     return sqlite3Fts3Init(db);
  3840   3144   }
  3841   3145   #endif
  3842   3146   
         3147  +
         3148  +/*
         3149  +** Allocate an Fts3MultiSegReader for each token in the expression headed
         3150  +** by pExpr. 
         3151  +**
         3152  +** An Fts3SegReader object is a cursor that can seek or scan a range of
         3153  +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
         3154  +** Fts3SegReader objects internally to provide an interface to seek or scan
         3155  +** within the union of all segments of a b-tree. Hence the name.
         3156  +**
         3157  +** If the allocated Fts3MultiSegReader just seeks to a single entry in a
         3158  +** segment b-tree (if the term is not a prefix or it is a prefix for which
         3159  +** there exists prefix b-tree of the right length) then it may be traversed
         3160  +** and merged incrementally. Otherwise, it has to be merged into an in-memory 
         3161  +** doclist and then traversed.
         3162  +*/
         3163  +static void fts3EvalAllocateReaders(
         3164  +  Fts3Cursor *pCsr, 
         3165  +  Fts3Expr *pExpr, 
         3166  +  int *pnToken,                   /* OUT: Total number of tokens in phrase. */
         3167  +  int *pnOr,                      /* OUT: Total number of OR nodes in expr. */
         3168  +  int *pRc
         3169  +){
         3170  +  if( pExpr && SQLITE_OK==*pRc ){
         3171  +    if( pExpr->eType==FTSQUERY_PHRASE ){
         3172  +      int i;
         3173  +      int nToken = pExpr->pPhrase->nToken;
         3174  +      *pnToken += nToken;
         3175  +      for(i=0; i<nToken; i++){
         3176  +        Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
         3177  +        int rc = sqlite3Fts3TermSegReaderCursor(pCsr, 
         3178  +            pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
         3179  +        );
         3180  +        if( rc!=SQLITE_OK ){
         3181  +          *pRc = rc;
         3182  +          return;
         3183  +        }
         3184  +      }
         3185  +      assert( pExpr->pPhrase->iDoclistToken==0 );
         3186  +      pExpr->pPhrase->iDoclistToken = -1;
         3187  +    }else{
         3188  +      *pnOr += (pExpr->eType==FTSQUERY_OR);
         3189  +      fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
         3190  +      fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
         3191  +    }
         3192  +  }
         3193  +}
         3194  +
         3195  +static void fts3EvalPhraseMergeToken(
         3196  +  Fts3Table *pTab,
         3197  +  Fts3Phrase *p,
         3198  +  int iToken,
         3199  +  char *pList,
         3200  +  int nList
         3201  +){
         3202  +  assert( iToken!=p->iDoclistToken );
         3203  +
         3204  +  if( pList==0 ){
         3205  +    sqlite3_free(p->doclist.aAll);
         3206  +    p->doclist.aAll = 0;
         3207  +    p->doclist.nAll = 0;
         3208  +  }
         3209  +
         3210  +  else if( p->iDoclistToken<0 ){
         3211  +    p->doclist.aAll = pList;
         3212  +    p->doclist.nAll = nList;
         3213  +  }
         3214  +
         3215  +  else if( p->doclist.aAll==0 ){
         3216  +    sqlite3_free(pList);
         3217  +  }
         3218  +
         3219  +  else {
         3220  +    char *pLeft;
         3221  +    char *pRight;
         3222  +    int nLeft;
         3223  +    int nRight;
         3224  +    int nDiff;
         3225  +
         3226  +    if( p->iDoclistToken<iToken ){
         3227  +      pLeft = p->doclist.aAll;
         3228  +      nLeft = p->doclist.nAll;
         3229  +      pRight = pList;
         3230  +      nRight = nList;
         3231  +      nDiff = iToken - p->iDoclistToken;
         3232  +    }else{
         3233  +      pRight = p->doclist.aAll;
         3234  +      nRight = p->doclist.nAll;
         3235  +      pLeft = pList;
         3236  +      nLeft = nList;
         3237  +      nDiff = p->iDoclistToken - iToken;
         3238  +    }
         3239  +
         3240  +    fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight);
         3241  +    sqlite3_free(pLeft);
         3242  +    p->doclist.aAll = pRight;
         3243  +    p->doclist.nAll = nRight;
         3244  +  }
         3245  +
         3246  +  if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
         3247  +}
         3248  +
         3249  +static int fts3EvalPhraseLoad(
         3250  +  Fts3Cursor *pCsr, 
         3251  +  Fts3Phrase *p
         3252  +){
         3253  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3254  +  int iToken;
         3255  +  int rc = SQLITE_OK;
         3256  +
         3257  +  for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
         3258  +    Fts3PhraseToken *pToken = &p->aToken[iToken];
         3259  +    assert( pToken->pDeferred==0 || pToken->pSegcsr==0 );
         3260  +
         3261  +    if( pToken->pSegcsr ){
         3262  +      int nThis = 0;
         3263  +      char *pThis = 0;
         3264  +      rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
         3265  +      if( rc==SQLITE_OK ){
         3266  +        fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
         3267  +      }
         3268  +    }
         3269  +    assert( pToken->pSegcsr==0 );
         3270  +  }
         3271  +
         3272  +  return rc;
         3273  +}
         3274  +
         3275  +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
         3276  +  int iToken;
         3277  +  int rc = SQLITE_OK;
         3278  +
         3279  +  int nMaxUndeferred = pPhrase->iDoclistToken;
         3280  +  char *aPoslist = 0;
         3281  +  int nPoslist = 0;
         3282  +  int iPrev = -1;
         3283  +
         3284  +  assert( pPhrase->doclist.bFreeList==0 );
         3285  +
         3286  +  for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
         3287  +    Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
         3288  +    Fts3DeferredToken *pDeferred = pToken->pDeferred;
         3289  +
         3290  +    if( pDeferred ){
         3291  +      char *pList;
         3292  +      int nList;
         3293  +      rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
         3294  +      if( rc!=SQLITE_OK ) return rc;
         3295  +
         3296  +      if( pList==0 ){
         3297  +        sqlite3_free(aPoslist);
         3298  +        pPhrase->doclist.pList = 0;
         3299  +        pPhrase->doclist.nList = 0;
         3300  +        return SQLITE_OK;
         3301  +
         3302  +      }else if( aPoslist==0 ){
         3303  +        aPoslist = pList;
         3304  +        nPoslist = nList;
         3305  +
         3306  +      }else{
         3307  +        char *aOut = pList;
         3308  +        char *p1 = aPoslist;
         3309  +        char *p2 = aOut;
         3310  +
         3311  +        assert( iPrev>=0 );
         3312  +        fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
         3313  +        sqlite3_free(aPoslist);
         3314  +        aPoslist = pList;
         3315  +        nPoslist = aOut - aPoslist;
         3316  +        if( nPoslist==0 ){
         3317  +          sqlite3_free(aPoslist);
         3318  +          pPhrase->doclist.pList = 0;
         3319  +          pPhrase->doclist.nList = 0;
         3320  +          return SQLITE_OK;
         3321  +        }
         3322  +      }
         3323  +      iPrev = iToken;
         3324  +    }
         3325  +  }
         3326  +
         3327  +  if( iPrev>=0 ){
         3328  +    if( nMaxUndeferred<0 ){
         3329  +      pPhrase->doclist.pList = aPoslist;
         3330  +      pPhrase->doclist.nList = nPoslist;
         3331  +      pPhrase->doclist.iDocid = pCsr->iPrevId;
         3332  +      pPhrase->doclist.bFreeList = 1;
         3333  +    }else{
         3334  +      int nDistance;
         3335  +      char *p1;
         3336  +      char *p2;
         3337  +      char *aOut;
         3338  +
         3339  +      if( nMaxUndeferred>iPrev ){
         3340  +        p1 = aPoslist;
         3341  +        p2 = pPhrase->doclist.pList;
         3342  +        nDistance = nMaxUndeferred - iPrev;
         3343  +      }else{
         3344  +        p1 = pPhrase->doclist.pList;
         3345  +        p2 = aPoslist;
         3346  +        nDistance = iPrev - nMaxUndeferred;
         3347  +      }
         3348  +
         3349  +      aOut = (char *)sqlite3_malloc(nPoslist+8);
         3350  +      if( !aOut ){
         3351  +        sqlite3_free(aPoslist);
         3352  +        return SQLITE_NOMEM;
         3353  +      }
         3354  +      
         3355  +      pPhrase->doclist.pList = aOut;
         3356  +      if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
         3357  +        pPhrase->doclist.bFreeList = 1;
         3358  +        pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList);
         3359  +      }else{
         3360  +        sqlite3_free(aOut);
         3361  +        pPhrase->doclist.pList = 0;
         3362  +        pPhrase->doclist.nList = 0;
         3363  +      }
         3364  +      sqlite3_free(aPoslist);
         3365  +    }
         3366  +  }
         3367  +
         3368  +  return SQLITE_OK;
         3369  +}
         3370  +
         3371  +/*
         3372  +** This function is called for each Fts3Phrase in a full-text query 
         3373  +** expression to initialize the mechanism for returning rows. Once this
         3374  +** function has been called successfully on an Fts3Phrase, it may be
         3375  +** used with fts3EvalPhraseNext() to iterate through the matching docids.
         3376  +*/
         3377  +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
         3378  +  int rc;
         3379  +  Fts3PhraseToken *pFirst = &p->aToken[0];
         3380  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3381  +
         3382  +  if( pCsr->bDesc==pTab->bDescIdx 
         3383  +   && bOptOk==1 
         3384  +   && p->nToken==1 
         3385  +   && pFirst->pSegcsr 
         3386  +   && pFirst->pSegcsr->bLookup 
         3387  +  ){
         3388  +    /* Use the incremental approach. */
         3389  +    int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
         3390  +    rc = sqlite3Fts3MsrIncrStart(
         3391  +        pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
         3392  +    p->bIncr = 1;
         3393  +
         3394  +  }else{
         3395  +    /* Load the full doclist for the phrase into memory. */
         3396  +    rc = fts3EvalPhraseLoad(pCsr, p);
         3397  +    p->bIncr = 0;
         3398  +  }
         3399  +
         3400  +  assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
         3401  +  return rc;
         3402  +}
         3403  +
         3404  +/*
         3405  +** This function is used to iterate backwards (from the end to start) 
         3406  +** through doclists.
         3407  +*/
         3408  +void sqlite3Fts3DoclistPrev(
         3409  +  int bDescIdx,                   /* True if the doclist is desc */
         3410  +  char *aDoclist,                 /* Pointer to entire doclist */
         3411  +  int nDoclist,                   /* Length of aDoclist in bytes */
         3412  +  char **ppIter,                  /* IN/OUT: Iterator pointer */
         3413  +  sqlite3_int64 *piDocid,         /* IN/OUT: Docid pointer */
         3414  +  int *pnList,                    /* IN/OUT: List length pointer */
         3415  +  u8 *pbEof                       /* OUT: End-of-file flag */
         3416  +){
         3417  +  char *p = *ppIter;
         3418  +
         3419  +  assert( nDoclist>0 );
         3420  +  assert( *pbEof==0 );
         3421  +  assert( p || *piDocid==0 );
         3422  +  assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
         3423  +
         3424  +  if( p==0 ){
         3425  +    sqlite3_int64 iDocid = 0;
         3426  +    char *pNext = 0;
         3427  +    char *pDocid = aDoclist;
         3428  +    char *pEnd = &aDoclist[nDoclist];
         3429  +    int iMul = 1;
         3430  +
         3431  +    while( pDocid<pEnd ){
         3432  +      sqlite3_int64 iDelta;
         3433  +      pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
         3434  +      iDocid += (iMul * iDelta);
         3435  +      pNext = pDocid;
         3436  +      fts3PoslistCopy(0, &pDocid);
         3437  +      while( pDocid<pEnd && *pDocid==0 ) pDocid++;
         3438  +      iMul = (bDescIdx ? -1 : 1);
         3439  +    }
         3440  +
         3441  +    *pnList = pEnd - pNext;
         3442  +    *ppIter = pNext;
         3443  +    *piDocid = iDocid;
         3444  +  }else{
         3445  +    int iMul = (bDescIdx ? -1 : 1);
         3446  +    sqlite3_int64 iDelta;
         3447  +    fts3GetReverseVarint(&p, aDoclist, &iDelta);
         3448  +    *piDocid -= (iMul * iDelta);
         3449  +
         3450  +    if( p==aDoclist ){
         3451  +      *pbEof = 1;
         3452  +    }else{
         3453  +      char *pSave = p;
         3454  +      fts3ReversePoslist(aDoclist, &p);
         3455  +      *pnList = (pSave - p);
         3456  +    }
         3457  +    *ppIter = p;
         3458  +  }
         3459  +}
         3460  +
         3461  +/*
         3462  +** Attempt to move the phrase iterator to point to the next matching docid. 
         3463  +** If an error occurs, return an SQLite error code. Otherwise, return 
         3464  +** SQLITE_OK.
         3465  +**
         3466  +** If there is no "next" entry and no error occurs, then *pbEof is set to
         3467  +** 1 before returning. Otherwise, if no error occurs and the iterator is
         3468  +** successfully advanced, *pbEof is set to 0.
         3469  +*/
         3470  +static int fts3EvalPhraseNext(
         3471  +  Fts3Cursor *pCsr, 
         3472  +  Fts3Phrase *p, 
         3473  +  u8 *pbEof
         3474  +){
         3475  +  int rc = SQLITE_OK;
         3476  +  Fts3Doclist *pDL = &p->doclist;
         3477  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3478  +
         3479  +  if( p->bIncr ){
         3480  +    assert( p->nToken==1 );
         3481  +    assert( pDL->pNextDocid==0 );
         3482  +    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
         3483  +        &pDL->iDocid, &pDL->pList, &pDL->nList
         3484  +    );
         3485  +    if( rc==SQLITE_OK && !pDL->pList ){
         3486  +      *pbEof = 1;
         3487  +    }
         3488  +  }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
         3489  +    sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, 
         3490  +        &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
         3491  +    );
         3492  +    pDL->pList = pDL->pNextDocid;
         3493  +  }else{
         3494  +    char *pIter;                            /* Used to iterate through aAll */
         3495  +    char *pEnd = &pDL->aAll[pDL->nAll];     /* 1 byte past end of aAll */
         3496  +    if( pDL->pNextDocid ){
         3497  +      pIter = pDL->pNextDocid;
         3498  +    }else{
         3499  +      pIter = pDL->aAll;
         3500  +    }
         3501  +
         3502  +    if( pIter>=pEnd ){
         3503  +      /* We have already reached the end of this doclist. EOF. */
         3504  +      *pbEof = 1;
         3505  +    }else{
         3506  +      sqlite3_int64 iDelta;
         3507  +      pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
         3508  +      if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
         3509  +        pDL->iDocid += iDelta;
         3510  +      }else{
         3511  +        pDL->iDocid -= iDelta;
         3512  +      }
         3513  +      pDL->pList = pIter;
         3514  +      fts3PoslistCopy(0, &pIter);
         3515  +      pDL->nList = (pIter - pDL->pList);
         3516  +
         3517  +      /* pIter now points just past the 0x00 that terminates the position-
         3518  +      ** list for document pDL->iDocid. However, if this position-list was
         3519  +      ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
         3520  +      ** point to the start of the next docid value. The following line deals
         3521  +      ** with this case by advancing pIter past the zero-padding added by
         3522  +      ** fts3EvalNearTrim2().  */
         3523  +      while( pIter<pEnd && *pIter==0 ) pIter++;
         3524  +
         3525  +      pDL->pNextDocid = pIter;
         3526  +      assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter );
         3527  +      *pbEof = 0;
         3528  +    }
         3529  +  }
         3530  +
         3531  +  return rc;
         3532  +}
         3533  +
         3534  +static void fts3EvalStartReaders(
         3535  +  Fts3Cursor *pCsr, 
         3536  +  Fts3Expr *pExpr, 
         3537  +  int bOptOk,
         3538  +  int *pRc
         3539  +){
         3540  +  if( pExpr && SQLITE_OK==*pRc ){
         3541  +    if( pExpr->eType==FTSQUERY_PHRASE ){
         3542  +      int i;
         3543  +      int nToken = pExpr->pPhrase->nToken;
         3544  +      for(i=0; i<nToken; i++){
         3545  +        if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
         3546  +      }
         3547  +      pExpr->bDeferred = (i==nToken);
         3548  +      *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase);
         3549  +    }else{
         3550  +      fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
         3551  +      fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
         3552  +      pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
         3553  +    }
         3554  +  }
         3555  +}
         3556  +
         3557  +typedef struct Fts3TokenAndCost Fts3TokenAndCost;
         3558  +struct Fts3TokenAndCost {
         3559  +  Fts3Phrase *pPhrase;            /* The phrase the token belongs to */
         3560  +  int iToken;                     /* Position of token in phrase */
         3561  +  Fts3PhraseToken *pToken;        /* The token itself */
         3562  +  Fts3Expr *pRoot; 
         3563  +  int nOvfl;
         3564  +  int iCol;                       /* The column the token must match */
         3565  +};
         3566  +
         3567  +static void fts3EvalTokenCosts(
         3568  +  Fts3Cursor *pCsr, 
         3569  +  Fts3Expr *pRoot, 
         3570  +  Fts3Expr *pExpr, 
         3571  +  Fts3TokenAndCost **ppTC,
         3572  +  Fts3Expr ***ppOr,
         3573  +  int *pRc
         3574  +){
         3575  +  if( *pRc==SQLITE_OK && pExpr ){
         3576  +    if( pExpr->eType==FTSQUERY_PHRASE ){
         3577  +      Fts3Phrase *pPhrase = pExpr->pPhrase;
         3578  +      int i;
         3579  +      for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
         3580  +        Fts3TokenAndCost *pTC = (*ppTC)++;
         3581  +        pTC->pPhrase = pPhrase;
         3582  +        pTC->iToken = i;
         3583  +        pTC->pRoot = pRoot;
         3584  +        pTC->pToken = &pPhrase->aToken[i];
         3585  +        pTC->iCol = pPhrase->iColumn;
         3586  +        *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
         3587  +      }
         3588  +    }else if( pExpr->eType!=FTSQUERY_NOT ){
         3589  +      if( pExpr->eType==FTSQUERY_OR ){
         3590  +        pRoot = pExpr->pLeft;
         3591  +        **ppOr = pRoot;
         3592  +        (*ppOr)++;
         3593  +      }
         3594  +      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
         3595  +      if( pExpr->eType==FTSQUERY_OR ){
         3596  +        pRoot = pExpr->pRight;
         3597  +        **ppOr = pRoot;
         3598  +        (*ppOr)++;
         3599  +      }
         3600  +      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
         3601  +    }
         3602  +  }
         3603  +}
         3604  +
         3605  +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
         3606  +  if( pCsr->nRowAvg==0 ){
         3607  +    /* The average document size, which is required to calculate the cost
         3608  +     ** of each doclist, has not yet been determined. Read the required 
         3609  +     ** data from the %_stat table to calculate it.
         3610  +     **
         3611  +     ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
         3612  +     ** varints, where nCol is the number of columns in the FTS3 table.
         3613  +     ** The first varint is the number of documents currently stored in
         3614  +     ** the table. The following nCol varints contain the total amount of
         3615  +     ** data stored in all rows of each column of the table, from left
         3616  +     ** to right.
         3617  +     */
         3618  +    int rc;
         3619  +    Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
         3620  +    sqlite3_stmt *pStmt;
         3621  +    sqlite3_int64 nDoc = 0;
         3622  +    sqlite3_int64 nByte = 0;
         3623  +    const char *pEnd;
         3624  +    const char *a;
         3625  +
         3626  +    rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
         3627  +    if( rc!=SQLITE_OK ) return rc;
         3628  +    a = sqlite3_column_blob(pStmt, 0);
         3629  +    assert( a );
         3630  +
         3631  +    pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
         3632  +    a += sqlite3Fts3GetVarint(a, &nDoc);
         3633  +    while( a<pEnd ){
         3634  +      a += sqlite3Fts3GetVarint(a, &nByte);
         3635  +    }
         3636  +    if( nDoc==0 || nByte==0 ){
         3637  +      sqlite3_reset(pStmt);
         3638  +      return SQLITE_CORRUPT_VTAB;
         3639  +    }
         3640  +
         3641  +    pCsr->nDoc = nDoc;
         3642  +    pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
         3643  +    assert( pCsr->nRowAvg>0 ); 
         3644  +    rc = sqlite3_reset(pStmt);
         3645  +    if( rc!=SQLITE_OK ) return rc;
         3646  +  }
         3647  +
         3648  +  *pnPage = pCsr->nRowAvg;
         3649  +  return SQLITE_OK;
         3650  +}
         3651  +
         3652  +static int fts3EvalSelectDeferred(
         3653  +  Fts3Cursor *pCsr,
         3654  +  Fts3Expr *pRoot,
         3655  +  Fts3TokenAndCost *aTC,
         3656  +  int nTC
         3657  +){
         3658  +  int nDocSize = 0;
         3659  +  int nDocEst = 0;
         3660  +  int rc = SQLITE_OK;
         3661  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3662  +  int ii;
         3663  +
         3664  +  int nOvfl = 0;
         3665  +  int nTerm = 0;
         3666  +
         3667  +  for(ii=0; ii<nTC; ii++){
         3668  +    if( aTC[ii].pRoot==pRoot ){
         3669  +      nOvfl += aTC[ii].nOvfl;
         3670  +      nTerm++;
         3671  +    }
         3672  +  }
         3673  +  if( nOvfl==0 || nTerm<2 ) return SQLITE_OK;
         3674  +
         3675  +  rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
         3676  +
         3677  +  for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
         3678  +    int jj;
         3679  +    Fts3TokenAndCost *pTC = 0;
         3680  +
         3681  +    for(jj=0; jj<nTC; jj++){
         3682  +      if( aTC[jj].pToken && aTC[jj].pRoot==pRoot 
         3683  +       && (!pTC || aTC[jj].nOvfl<pTC->nOvfl) 
         3684  +      ){
         3685  +        pTC = &aTC[jj];
         3686  +      }
         3687  +    }
         3688  +    assert( pTC );
         3689  +
         3690  +    /* At this point pTC points to the cheapest remaining token. */
         3691  +    if( ii==0 ){
         3692  +      if( pTC->nOvfl ){
         3693  +        nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
         3694  +      }else{
         3695  +        Fts3PhraseToken *pToken = pTC->pToken;
         3696  +        int nList = 0;
         3697  +        char *pList = 0;
         3698  +        rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
         3699  +        assert( rc==SQLITE_OK || pList==0 );
         3700  +
         3701  +        if( rc==SQLITE_OK ){
         3702  +          nDocEst = fts3DoclistCountDocids(1, pList, nList);
         3703  +          fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
         3704  +        }
         3705  +      }
         3706  +    }else{
         3707  +      if( pTC->nOvfl>=(nDocEst*nDocSize) ){
         3708  +        Fts3PhraseToken *pToken = pTC->pToken;
         3709  +        rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
         3710  +        fts3SegReaderCursorFree(pToken->pSegcsr);
         3711  +        pToken->pSegcsr = 0;
         3712  +      }
         3713  +      nDocEst = 1 + (nDocEst/4);
         3714  +    }
         3715  +    pTC->pToken = 0;
         3716  +  }
         3717  +
         3718  +  return rc;
         3719  +}
         3720  +
         3721  +int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
         3722  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         3723  +  int rc = SQLITE_OK;
         3724  +  int nToken = 0;
         3725  +  int nOr = 0;
         3726  +
         3727  +  /* Allocate a MultiSegReader for each token in the expression. */
         3728  +  fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
         3729  +
         3730  +  /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
         3731  +  ** This call will eventually also be responsible for determining which
         3732  +  ** tokens are 'deferred' until the document text is loaded into memory.
         3733  +  **
         3734  +  ** Each token in each phrase is dealt with using one of the following
         3735  +  ** three strategies:
         3736  +  **
         3737  +  **   1. Entire doclist loaded into memory as part of the
         3738  +  **      fts3EvalStartReaders() call.
         3739  +  **
         3740  +  **   2. Doclist loaded into memory incrementally, as part of each
         3741  +  **      sqlite3Fts3EvalNext() call.
         3742  +  **
         3743  +  **   3. Token doclist is never loaded. Instead, documents are loaded into
         3744  +  **      memory and scanned for the token as part of the sqlite3Fts3EvalNext()
         3745  +  **      call. This is known as a "deferred" token.
         3746  +  */
         3747  +
         3748  +  /* If bOptOk is true, check if there are any tokens that should be deferred.
         3749  +  */
         3750  +  if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
         3751  +    Fts3TokenAndCost *aTC;
         3752  +    Fts3Expr **apOr;
         3753  +    aTC = (Fts3TokenAndCost *)sqlite3_malloc(
         3754  +        sizeof(Fts3TokenAndCost) * nToken
         3755  +      + sizeof(Fts3Expr *) * nOr * 2
         3756  +    );
         3757  +    apOr = (Fts3Expr **)&aTC[nToken];
         3758  +
         3759  +    if( !aTC ){
         3760  +      rc = SQLITE_NOMEM;
         3761  +    }else{
         3762  +      int ii;
         3763  +      Fts3TokenAndCost *pTC = aTC;
         3764  +      Fts3Expr **ppOr = apOr;
         3765  +
         3766  +      fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
         3767  +      nToken = pTC-aTC;
         3768  +      nOr = ppOr-apOr;
         3769  +
         3770  +      if( rc==SQLITE_OK ){
         3771  +        rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
         3772  +        for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
         3773  +          rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
         3774  +        }
         3775  +      }
         3776  +
         3777  +      sqlite3_free(aTC);
         3778  +    }
         3779  +  }
         3780  +
         3781  +  fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
         3782  +  return rc;
         3783  +}
         3784  +
         3785  +static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
         3786  +  if( pPhrase->doclist.bFreeList ){
         3787  +    sqlite3_free(pPhrase->doclist.pList);
         3788  +  }
         3789  +  pPhrase->doclist.pList = 0;
         3790  +  pPhrase->doclist.nList = 0;
         3791  +  pPhrase->doclist.bFreeList = 0;
         3792  +}
         3793  +
         3794  +static int fts3EvalNearTrim2(
         3795  +  int nNear,
         3796  +  char *aTmp,                     /* Temporary space to use */
         3797  +  char **paPoslist,               /* IN/OUT: Position list */
         3798  +  int *pnToken,                   /* IN/OUT: Tokens in phrase of *paPoslist */
         3799  +  Fts3Phrase *pPhrase             /* The phrase object to trim the doclist of */
         3800  +){
         3801  +  int nParam1 = nNear + pPhrase->nToken;
         3802  +  int nParam2 = nNear + *pnToken;
         3803  +  int nNew;
         3804  +  char *p2; 
         3805  +  char *pOut; 
         3806  +  int res;
         3807  +
         3808  +  assert( pPhrase->doclist.pList );
         3809  +
         3810  +  p2 = pOut = pPhrase->doclist.pList;
         3811  +  res = fts3PoslistNearMerge(
         3812  +    &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
         3813  +  );
         3814  +  if( res ){
         3815  +    nNew = (pOut - pPhrase->doclist.pList) - 1;
         3816  +    assert( pPhrase->doclist.pList[nNew]=='\0' );
         3817  +    assert( nNew<=pPhrase->doclist.nList && nNew>0 );
         3818  +    memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
         3819  +    pPhrase->doclist.nList = nNew;
         3820  +    *paPoslist = pPhrase->doclist.pList;
         3821  +    *pnToken = pPhrase->nToken;
         3822  +  }
         3823  +
         3824  +  return res;
         3825  +}
         3826  +
         3827  +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
         3828  +  int res = 1;
         3829  +
         3830  +  /* The following block runs if pExpr is the root of a NEAR query.
         3831  +  ** For example, the query:
         3832  +  **
         3833  +  **         "w" NEAR "x" NEAR "y" NEAR "z"
         3834  +  **
         3835  +  ** which is represented in tree form as:
         3836  +  **
         3837  +  **                               |
         3838  +  **                          +--NEAR--+      <-- root of NEAR query
         3839  +  **                          |        |
         3840  +  **                     +--NEAR--+   "z"
         3841  +  **                     |        |
         3842  +  **                +--NEAR--+   "y"
         3843  +  **                |        |
         3844  +  **               "w"      "x"
         3845  +  **
         3846  +  ** The right-hand child of a NEAR node is always a phrase. The 
         3847  +  ** left-hand child may be either a phrase or a NEAR node. There are
         3848  +  ** no exceptions to this.
         3849  +  */
         3850  +  if( *pRc==SQLITE_OK 
         3851  +   && pExpr->eType==FTSQUERY_NEAR 
         3852  +   && pExpr->bEof==0
         3853  +   && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
         3854  +  ){
         3855  +    Fts3Expr *p; 
         3856  +    int nTmp = 0;                 /* Bytes of temp space */
         3857  +    char *aTmp;                   /* Temp space for PoslistNearMerge() */
         3858  +
         3859  +    /* Allocate temporary working space. */
         3860  +    for(p=pExpr; p->pLeft; p=p->pLeft){
         3861  +      nTmp += p->pRight->pPhrase->doclist.nList;
         3862  +    }
         3863  +    nTmp += p->pPhrase->doclist.nList;
         3864  +    aTmp = sqlite3_malloc(nTmp*2);
         3865  +    if( !aTmp ){
         3866  +      *pRc = SQLITE_NOMEM;
         3867  +      res = 0;
         3868  +    }else{
         3869  +      char *aPoslist = p->pPhrase->doclist.pList;
         3870  +      int nToken = p->pPhrase->nToken;
         3871  +
         3872  +      for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
         3873  +        Fts3Phrase *pPhrase = p->pRight->pPhrase;
         3874  +        int nNear = p->nNear;
         3875  +        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         3876  +      }
         3877  +  
         3878  +      aPoslist = pExpr->pRight->pPhrase->doclist.pList;
         3879  +      nToken = pExpr->pRight->pPhrase->nToken;
         3880  +      for(p=pExpr->pLeft; p && res; p=p->pLeft){
         3881  +        int nNear = p->pParent->nNear;
         3882  +        Fts3Phrase *pPhrase = (
         3883  +            p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
         3884  +        );
         3885  +        res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         3886  +      }
         3887  +    }
         3888  +
         3889  +    sqlite3_free(aTmp);
         3890  +  }
         3891  +
         3892  +  return res;
         3893  +}
         3894  +
         3895  +/*
         3896  +** This macro is used by the fts3EvalNext() function. The two arguments are
         3897  +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
         3898  +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
         3899  +** it returns (i2 - i1). This allows the same code to be used for merging
         3900  +** doclists in ascending or descending order.
         3901  +*/
         3902  +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
         3903  +
         3904  +static void fts3EvalNext(
         3905  +  Fts3Cursor *pCsr, 
         3906  +  Fts3Expr *pExpr, 
         3907  +  int *pRc
         3908  +){
         3909  +  if( *pRc==SQLITE_OK ){
         3910  +    assert( pExpr->bEof==0 );
         3911  +    pExpr->bStart = 1;
         3912  +
         3913  +    switch( pExpr->eType ){
         3914  +      case FTSQUERY_NEAR:
         3915  +      case FTSQUERY_AND: {
         3916  +        Fts3Expr *pLeft = pExpr->pLeft;
         3917  +        Fts3Expr *pRight = pExpr->pRight;
         3918  +        assert( !pLeft->bDeferred || !pRight->bDeferred );
         3919  +        if( pLeft->bDeferred ){
         3920  +          fts3EvalNext(pCsr, pRight, pRc);
         3921  +          pExpr->iDocid = pRight->iDocid;
         3922  +          pExpr->bEof = pRight->bEof;
         3923  +        }else if( pRight->bDeferred ){
         3924  +          fts3EvalNext(pCsr, pLeft, pRc);
         3925  +          pExpr->iDocid = pLeft->iDocid;
         3926  +          pExpr->bEof = pLeft->bEof;
         3927  +        }else{
         3928  +          fts3EvalNext(pCsr, pLeft, pRc);
         3929  +          fts3EvalNext(pCsr, pRight, pRc);
         3930  +
         3931  +          while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
         3932  +            sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         3933  +            if( iDiff==0 ) break;
         3934  +            if( iDiff<0 ){
         3935  +              fts3EvalNext(pCsr, pLeft, pRc);
         3936  +            }else{
         3937  +              fts3EvalNext(pCsr, pRight, pRc);
         3938  +            }
         3939  +          }
         3940  +
         3941  +          pExpr->iDocid = pLeft->iDocid;
         3942  +          pExpr->bEof = (pLeft->bEof || pRight->bEof);
         3943  +        }
         3944  +        break;
         3945  +      }
         3946  +  
         3947  +      case FTSQUERY_OR: {
         3948  +        Fts3Expr *pLeft = pExpr->pLeft;
         3949  +        Fts3Expr *pRight = pExpr->pRight;
         3950  +        sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         3951  +
         3952  +        assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
         3953  +        assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
         3954  +
         3955  +        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
         3956  +          fts3EvalNext(pCsr, pLeft, pRc);
         3957  +        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
         3958  +          fts3EvalNext(pCsr, pRight, pRc);
         3959  +        }else{
         3960  +          fts3EvalNext(pCsr, pLeft, pRc);
         3961  +          fts3EvalNext(pCsr, pRight, pRc);
         3962  +        }
         3963  +
         3964  +        pExpr->bEof = (pLeft->bEof && pRight->bEof);
         3965  +        iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
         3966  +        if( pRight->bEof || (pLeft->bEof==0 &&  iCmp<0) ){
         3967  +          pExpr->iDocid = pLeft->iDocid;
         3968  +        }else{
         3969  +          pExpr->iDocid = pRight->iDocid;
         3970  +        }
         3971  +
         3972  +        break;
         3973  +      }
         3974  +
         3975  +      case FTSQUERY_NOT: {
         3976  +        Fts3Expr *pLeft = pExpr->pLeft;
         3977  +        Fts3Expr *pRight = pExpr->pRight;
         3978  +
         3979  +        if( pRight->bStart==0 ){
         3980  +          fts3EvalNext(pCsr, pRight, pRc);
         3981  +          assert( *pRc!=SQLITE_OK || pRight->bStart );
         3982  +        }
         3983  +
         3984  +        fts3EvalNext(pCsr, pLeft, pRc);
         3985  +        if( pLeft->bEof==0 ){
         3986  +          while( !*pRc 
         3987  +              && !pRight->bEof 
         3988  +              && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 
         3989  +          ){
         3990  +            fts3EvalNext(pCsr, pRight, pRc);
         3991  +          }
         3992  +        }
         3993  +        pExpr->iDocid = pLeft->iDocid;
         3994  +        pExpr->bEof = pLeft->bEof;
         3995  +        break;
         3996  +      }
         3997  +
         3998  +      default: {
         3999  +        Fts3Phrase *pPhrase = pExpr->pPhrase;
         4000  +        fts3EvalZeroPoslist(pPhrase);
         4001  +        *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
         4002  +        pExpr->iDocid = pPhrase->doclist.iDocid;
         4003  +        break;
         4004  +      }
         4005  +    }
         4006  +  }
         4007  +}
         4008  +
         4009  +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
         4010  +  int bHit = 1;
         4011  +  if( *pRc==SQLITE_OK ){
         4012  +    switch( pExpr->eType ){
         4013  +      case FTSQUERY_NEAR:
         4014  +      case FTSQUERY_AND:
         4015  +        bHit = (
         4016  +            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
         4017  +         && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         4018  +         && fts3EvalNearTest(pExpr, pRc)
         4019  +        );
         4020  +
         4021  +        /* If the NEAR expression does not match any rows, zero the doclist for 
         4022  +        ** all phrases involved in the NEAR. This is because the snippet(),
         4023  +        ** offsets() and matchinfo() functions are not supposed to recognize 
         4024  +        ** any instances of phrases that are part of unmatched NEAR queries. 
         4025  +        ** For example if this expression:
         4026  +        **
         4027  +        **    ... MATCH 'a OR (b NEAR c)'
         4028  +        **
         4029  +        ** is matched against a row containing:
         4030  +        **
         4031  +        **        'a b d e'
         4032  +        **
         4033  +        ** then any snippet() should ony highlight the "a" term, not the "b"
         4034  +        ** (as "b" is part of a non-matching NEAR clause).
         4035  +        */
         4036  +        if( bHit==0 
         4037  +         && pExpr->eType==FTSQUERY_NEAR 
         4038  +         && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
         4039  +        ){
         4040  +          Fts3Expr *p;
         4041  +          for(p=pExpr; p->pPhrase==0; p=p->pLeft){
         4042  +            if( p->pRight->iDocid==pCsr->iPrevId ){
         4043  +              fts3EvalZeroPoslist(p->pRight->pPhrase);
         4044  +            }
         4045  +          }
         4046  +          if( p->iDocid==pCsr->iPrevId ){
         4047  +            fts3EvalZeroPoslist(p->pPhrase);
         4048  +          }
         4049  +        }
         4050  +
         4051  +        break;
         4052  +
         4053  +      case FTSQUERY_OR: {
         4054  +        int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
         4055  +        int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
         4056  +        bHit = bHit1 || bHit2;
         4057  +        break;
         4058  +      }
         4059  +
         4060  +      case FTSQUERY_NOT:
         4061  +        bHit = (
         4062  +            fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
         4063  +         && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
         4064  +        );
         4065  +        break;
         4066  +
         4067  +      default: {
         4068  +        if( pCsr->pDeferred 
         4069  +         && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
         4070  +        ){
         4071  +          Fts3Phrase *pPhrase = pExpr->pPhrase;
         4072  +          assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
         4073  +          if( pExpr->bDeferred ){
         4074  +            fts3EvalZeroPoslist(pPhrase);
         4075  +          }
         4076  +          *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
         4077  +          bHit = (pPhrase->doclist.pList!=0);
         4078  +          pExpr->iDocid = pCsr->iPrevId;
         4079  +        }else{
         4080  +          bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
         4081  +        }
         4082  +        break;
         4083  +      }
         4084  +    }
         4085  +  }
         4086  +  return bHit;
         4087  +}
         4088  +
         4089  +/*
         4090  +** Return 1 if both of the following are true:
         4091  +**
         4092  +**   1. *pRc is SQLITE_OK when this function returns, and
         4093  +**
         4094  +**   2. After scanning the current FTS table row for the deferred tokens,
         4095  +**      it is determined that the row does not match the query.
         4096  +**
         4097  +** Or, if no error occurs and it seems the current row does match the FTS
         4098  +** query, return 0.
         4099  +*/
         4100  +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){
         4101  +  int rc = *pRc;
         4102  +  int bMiss = 0;
         4103  +  if( rc==SQLITE_OK ){
         4104  +    if( pCsr->pDeferred ){
         4105  +      rc = fts3CursorSeek(0, pCsr);
         4106  +      if( rc==SQLITE_OK ){
         4107  +        rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
         4108  +      }
         4109  +    }
         4110  +    bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
         4111  +    sqlite3Fts3FreeDeferredDoclists(pCsr);
         4112  +    *pRc = rc;
         4113  +  }
         4114  +  return (rc==SQLITE_OK && bMiss);
         4115  +}
         4116  +
         4117  +/*
         4118  +** Advance to the next document that matches the FTS expression in
         4119  +** Fts3Cursor.pExpr.
         4120  +*/
         4121  +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
         4122  +  int rc = SQLITE_OK;             /* Return Code */
         4123  +  Fts3Expr *pExpr = pCsr->pExpr;
         4124  +  assert( pCsr->isEof==0 );
         4125  +  if( pExpr==0 ){
         4126  +    pCsr->isEof = 1;
         4127  +  }else{
         4128  +    do {
         4129  +      if( pCsr->isRequireSeek==0 ){
         4130  +        sqlite3_reset(pCsr->pStmt);
         4131  +      }
         4132  +      assert( sqlite3_data_count(pCsr->pStmt)==0 );
         4133  +      fts3EvalNext(pCsr, pExpr, &rc);
         4134  +      pCsr->isEof = pExpr->bEof;
         4135  +      pCsr->isRequireSeek = 1;
         4136  +      pCsr->isMatchinfoNeeded = 1;
         4137  +      pCsr->iPrevId = pExpr->iDocid;
         4138  +    }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
         4139  +  }
         4140  +  return rc;
         4141  +}
         4142  +
         4143  +/*
         4144  +** Restart interation for expression pExpr so that the next call to
         4145  +** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental 
         4146  +** loading or merging of phrase doclists for this iteration.
         4147  +**
         4148  +** If *pRc is other than SQLITE_OK when this function is called, it is
         4149  +** a no-op. If an error occurs within this function, *pRc is set to an
         4150  +** SQLite error code before returning.
         4151  +*/
         4152  +static void fts3EvalRestart(
         4153  +  Fts3Cursor *pCsr,
         4154  +  Fts3Expr *pExpr,
         4155  +  int *pRc
         4156  +){
         4157  +  if( pExpr && *pRc==SQLITE_OK ){
         4158  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
         4159  +
         4160  +    if( pPhrase ){
         4161  +      fts3EvalZeroPoslist(pPhrase);
         4162  +      if( pPhrase->bIncr ){
         4163  +        assert( pPhrase->nToken==1 );
         4164  +        assert( pPhrase->aToken[0].pSegcsr );
         4165  +        sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
         4166  +        *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
         4167  +      }
         4168  +
         4169  +      pPhrase->doclist.pNextDocid = 0;
         4170  +      pPhrase->doclist.iDocid = 0;
         4171  +    }
         4172  +
         4173  +    pExpr->iDocid = 0;
         4174  +    pExpr->bEof = 0;
         4175  +    pExpr->bStart = 0;
         4176  +
         4177  +    fts3EvalRestart(pCsr, pExpr->pLeft, pRc);
         4178  +    fts3EvalRestart(pCsr, pExpr->pRight, pRc);
         4179  +  }
         4180  +}
         4181  +
         4182  +/*
         4183  +** After allocating the Fts3Expr.aMI[] array for each phrase in the 
         4184  +** expression rooted at pExpr, the cursor iterates through all rows matched
         4185  +** by pExpr, calling this function for each row. This function increments
         4186  +** the values in Fts3Expr.aMI[] according to the position-list currently
         4187  +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase 
         4188  +** expression nodes.
         4189  +*/
         4190  +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
         4191  +  if( pExpr ){
         4192  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
         4193  +    if( pPhrase && pPhrase->doclist.pList ){
         4194  +      int iCol = 0;
         4195  +      char *p = pPhrase->doclist.pList;
         4196  +
         4197  +      assert( *p );
         4198  +      while( 1 ){
         4199  +        u8 c = 0;
         4200  +        int iCnt = 0;
         4201  +        while( 0xFE & (*p | c) ){
         4202  +          if( (c&0x80)==0 ) iCnt++;
         4203  +          c = *p++ & 0x80;
         4204  +        }
         4205  +
         4206  +        /* aMI[iCol*3 + 1] = Number of occurrences
         4207  +        ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
         4208  +        */
         4209  +        pExpr->aMI[iCol*3 + 1] += iCnt;
         4210  +        pExpr->aMI[iCol*3 + 2] += (iCnt>0);
         4211  +        if( *p==0x00 ) break;
         4212  +        p++;
         4213  +        p += sqlite3Fts3GetVarint32(p, &iCol);
         4214  +      }
         4215  +    }
         4216  +
         4217  +    fts3EvalUpdateCounts(pExpr->pLeft);
         4218  +    fts3EvalUpdateCounts(pExpr->pRight);
         4219  +  }
         4220  +}
         4221  +
         4222  +/*
         4223  +** Expression pExpr must be of type FTSQUERY_PHRASE.
         4224  +**
         4225  +** If it is not already allocated and populated, this function allocates and
         4226  +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part
         4227  +** of a NEAR expression, then it also allocates and populates the same array
         4228  +** for all other phrases that are part of the NEAR expression.
         4229  +**
         4230  +** SQLITE_OK is returned if the aMI[] array is successfully allocated and
         4231  +** populated. Otherwise, if an error occurs, an SQLite error code is returned.
         4232  +*/
         4233  +static int fts3EvalGatherStats(
         4234  +  Fts3Cursor *pCsr,               /* Cursor object */
         4235  +  Fts3Expr *pExpr                 /* FTSQUERY_PHRASE expression */
         4236  +){
         4237  +  int rc = SQLITE_OK;             /* Return code */
         4238  +
         4239  +  assert( pExpr->eType==FTSQUERY_PHRASE );
         4240  +  if( pExpr->aMI==0 ){
         4241  +    Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         4242  +    Fts3Expr *pRoot;                /* Root of NEAR expression */
         4243  +    Fts3Expr *p;                    /* Iterator used for several purposes */
         4244  +
         4245  +    sqlite3_int64 iPrevId = pCsr->iPrevId;
         4246  +    sqlite3_int64 iDocid;
         4247  +    u8 bEof;
         4248  +
         4249  +    /* Find the root of the NEAR expression */
         4250  +    pRoot = pExpr;
         4251  +    while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
         4252  +      pRoot = pRoot->pParent;
         4253  +    }
         4254  +    iDocid = pRoot->iDocid;
         4255  +    bEof = pRoot->bEof;
         4256  +    assert( pRoot->bStart );
         4257  +
         4258  +    /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
         4259  +    for(p=pRoot; p; p=p->pLeft){
         4260  +      Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
         4261  +      assert( pE->aMI==0 );
         4262  +      pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
         4263  +      if( !pE->aMI ) return SQLITE_NOMEM;
         4264  +      memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
         4265  +    }
         4266  +
         4267  +    fts3EvalRestart(pCsr, pRoot, &rc);
         4268  +
         4269  +    while( pCsr->isEof==0 && rc==SQLITE_OK ){
         4270  +
         4271  +      do {
         4272  +        /* Ensure the %_content statement is reset. */
         4273  +        if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
         4274  +        assert( sqlite3_data_count(pCsr->pStmt)==0 );
         4275  +
         4276  +        /* Advance to the next document */
         4277  +        fts3EvalNext(pCsr, pRoot, &rc);
         4278  +        pCsr->isEof = pRoot->bEof;
         4279  +        pCsr->isRequireSeek = 1;
         4280  +        pCsr->isMatchinfoNeeded = 1;
         4281  +        pCsr->iPrevId = pRoot->iDocid;
         4282  +      }while( pCsr->isEof==0 
         4283  +           && pRoot->eType==FTSQUERY_NEAR 
         4284  +           && fts3EvalLoadDeferred(pCsr, &rc) 
         4285  +      );
         4286  +
         4287  +      if( rc==SQLITE_OK && pCsr->isEof==0 ){
         4288  +        fts3EvalUpdateCounts(pRoot);
         4289  +      }
         4290  +    }
         4291  +
         4292  +    pCsr->isEof = 0;
         4293  +    pCsr->iPrevId = iPrevId;
         4294  +
         4295  +    if( bEof ){
         4296  +      pRoot->bEof = bEof;
         4297  +    }else{
         4298  +      /* Caution: pRoot may iterate through docids in ascending or descending
         4299  +      ** order. For this reason, even though it seems more defensive, the 
         4300  +      ** do loop can not be written:
         4301  +      **
         4302  +      **   do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
         4303  +      */
         4304  +      fts3EvalRestart(pCsr, pRoot, &rc);
         4305  +      do {
         4306  +        fts3EvalNext(pCsr, pRoot, &rc);
         4307  +        assert( pRoot->bEof==0 );
         4308  +      }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
         4309  +      fts3EvalLoadDeferred(pCsr, &rc);
         4310  +    }
         4311  +  }
         4312  +  return rc;
         4313  +}
         4314  +
         4315  +/*
         4316  +** This function is used by the matchinfo() module to query a phrase 
         4317  +** expression node for the following information:
         4318  +**
         4319  +**   1. The total number of occurrences of the phrase in each column of 
         4320  +**      the FTS table (considering all rows), and
         4321  +**
         4322  +**   2. For each column, the number of rows in the table for which the
         4323  +**      column contains at least one instance of the phrase.
         4324  +**
         4325  +** If no error occurs, SQLITE_OK is returned and the values for each column
         4326  +** written into the array aiOut as follows:
         4327  +**
         4328  +**   aiOut[iCol*3 + 1] = Number of occurrences
         4329  +**   aiOut[iCol*3 + 2] = Number of rows containing at least one instance
         4330  +**
         4331  +** Caveats:
         4332  +**
         4333  +**   * If a phrase consists entirely of deferred tokens, then all output 
         4334  +**     values are set to the number of documents in the table. In other
         4335  +**     words we assume that very common tokens occur exactly once in each 
         4336  +**     column of each row of the table.
         4337  +**
         4338  +**   * If a phrase contains some deferred tokens (and some non-deferred 
         4339  +**     tokens), count the potential occurrence identified by considering
         4340  +**     the non-deferred tokens instead of actual phrase occurrences.
         4341  +**
         4342  +**   * If the phrase is part of a NEAR expression, then only phrase instances
         4343  +**     that meet the NEAR constraint are included in the counts.
         4344  +*/
         4345  +int sqlite3Fts3EvalPhraseStats(
         4346  +  Fts3Cursor *pCsr,               /* FTS cursor handle */
         4347  +  Fts3Expr *pExpr,                /* Phrase expression */
         4348  +  u32 *aiOut                      /* Array to write results into (see above) */
         4349  +){
         4350  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         4351  +  int rc = SQLITE_OK;
         4352  +  int iCol;
         4353  +
         4354  +  if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
         4355  +    assert( pCsr->nDoc>0 );
         4356  +    for(iCol=0; iCol<pTab->nColumn; iCol++){
         4357  +      aiOut[iCol*3 + 1] = pCsr->nDoc;
         4358  +      aiOut[iCol*3 + 2] = pCsr->nDoc;
         4359  +    }
         4360  +  }else{
         4361  +    rc = fts3EvalGatherStats(pCsr, pExpr);
         4362  +    if( rc==SQLITE_OK ){
         4363  +      assert( pExpr->aMI );
         4364  +      for(iCol=0; iCol<pTab->nColumn; iCol++){
         4365  +        aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1];
         4366  +        aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2];
         4367  +      }
         4368  +    }
         4369  +  }
         4370  +
         4371  +  return rc;
         4372  +}
         4373  +
         4374  +/*
         4375  +** The expression pExpr passed as the second argument to this function
         4376  +** must be of type FTSQUERY_PHRASE. 
         4377  +**
         4378  +** The returned value is either NULL or a pointer to a buffer containing
         4379  +** a position-list indicating the occurrences of the phrase in column iCol
         4380  +** of the current row. 
         4381  +**
         4382  +** More specifically, the returned buffer contains 1 varint for each 
         4383  +** occurence of the phrase in the column, stored using the normal (delta+2) 
         4384  +** compression and is terminated by either an 0x01 or 0x00 byte. For example,
         4385  +** if the requested column contains "a b X c d X X" and the position-list
         4386  +** for 'X' is requested, the buffer returned may contain:
         4387  +**
         4388  +**     0x04 0x05 0x03 0x01   or   0x04 0x05 0x03 0x00
         4389  +**
         4390  +** This function works regardless of whether or not the phrase is deferred,
         4391  +** incremental, or neither.
         4392  +*/
         4393  +char *sqlite3Fts3EvalPhrasePoslist(
         4394  +  Fts3Cursor *pCsr,               /* FTS3 cursor object */
         4395  +  Fts3Expr *pExpr,                /* Phrase to return doclist for */
         4396  +  int iCol                        /* Column to return position list for */
         4397  +){
         4398  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
         4399  +  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
         4400  +  char *pIter = pPhrase->doclist.pList;
         4401  +  int iThis;
         4402  +
         4403  +  assert( iCol>=0 && iCol<pTab->nColumn );
         4404  +  if( !pIter 
         4405  +   || pExpr->bEof 
         4406  +   || pExpr->iDocid!=pCsr->iPrevId
         4407  +   || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) 
         4408  +  ){
         4409  +    return 0;
         4410  +  }
         4411  +
         4412  +  assert( pPhrase->doclist.nList>0 );
         4413  +  if( *pIter==0x01 ){
         4414  +    pIter++;
         4415  +    pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
         4416  +  }else{
         4417  +    iThis = 0;
         4418  +  }
         4419  +  while( iThis<iCol ){
         4420  +    fts3ColumnlistCopy(0, &pIter);
         4421  +    if( *pIter==0x00 ) return 0;
         4422  +    pIter++;
         4423  +    pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
         4424  +  }
         4425  +
         4426  +  return ((iCol==iThis)?pIter:0);
         4427  +}
         4428  +
         4429  +/*
         4430  +** Free all components of the Fts3Phrase structure that were allocated by
         4431  +** the eval module. Specifically, this means to free:
         4432  +**
         4433  +**   * the contents of pPhrase->doclist, and
         4434  +**   * any Fts3MultiSegReader objects held by phrase tokens.
         4435  +*/
         4436  +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
         4437  +  if( pPhrase ){
         4438  +    int i;
         4439  +    sqlite3_free(pPhrase->doclist.aAll);
         4440  +    fts3EvalZeroPoslist(pPhrase);
         4441  +    memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
         4442  +    for(i=0; i<pPhrase->nToken; i++){
         4443  +      fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
         4444  +      pPhrase->aToken[i].pSegcsr = 0;
         4445  +    }
         4446  +  }
         4447  +}
         4448  +
  3843   4449   #endif

Changes to ext/fts3/fts3Int.h.

     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   */
    14         -
    15     14   #ifndef _FTSINT_H
    16     15   #define _FTSINT_H
    17     16   
    18     17   #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
    19     18   # define NDEBUG 1
    20     19   #endif
    21     20   
           21  +/*
           22  +** FTS4 is really an extension for FTS3.  It is enabled using the
           23  +** SQLITE_ENABLE_FTS3 macro.  But to avoid confusion we also all
           24  +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
           25  +*/
           26  +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
           27  +# define SQLITE_ENABLE_FTS3
           28  +#endif
           29  +
           30  +#ifdef SQLITE_ENABLE_FTS3
    22     31   #include "sqlite3.h"
    23     32   #include "fts3_tokenizer.h"
    24     33   #include "fts3_hash.h"
    25     34   
    26     35   /*
    27     36   ** This constant controls how often segments are merged. Once there are
    28     37   ** FTS3_MERGE_COUNT segments of level N, they are merged into a single
................................................................................
    43     52   /*
    44     53   ** Macro to return the number of elements in an array. SQLite has a
    45     54   ** similar macro called ArraySize(). Use a different name to avoid
    46     55   ** a collision when building an amalgamation with built-in FTS3.
    47     56   */
    48     57   #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
    49     58   
           59  +
           60  +#ifndef MIN
           61  +# define MIN(x,y) ((x)<(y)?(x):(y))
           62  +#endif
           63  +
    50     64   /*
    51     65   ** Maximum length of a varint encoded integer. The varint format is different
    52     66   ** from that used by SQLite, so the maximum length is 10, not 9.
    53     67   */
    54     68   #define FTS3_VARINT_MAX 10
    55     69   
           70  +/*
           71  +** FTS4 virtual tables may maintain multiple indexes - one index of all terms
           72  +** in the document set and zero or more prefix indexes. All indexes are stored
           73  +** as one or more b+-trees in the %_segments and %_segdir tables. 
           74  +**
           75  +** It is possible to determine which index a b+-tree belongs to based on the
           76  +** value stored in the "%_segdir.level" column. Given this value L, the index
           77  +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
           78  +** level values between 0 and 1023 (inclusive) belong to index 0, all levels
           79  +** between 1024 and 2047 to index 1, and so on.
           80  +**
           81  +** It is considered impossible for an index to use more than 1024 levels. In 
           82  +** theory though this may happen, but only after at least 
           83  +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
           84  +*/
           85  +#define FTS3_SEGDIR_MAXLEVEL      1024
           86  +#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
           87  +
    56     88   /*
    57     89   ** The testcase() macro is only used by the amalgamation.  If undefined,
    58     90   ** make it a no-op.
    59     91   */
    60     92   #ifndef testcase
    61     93   # define testcase(X)
    62     94   #endif
................................................................................
   120    152   
   121    153   typedef struct Fts3Table Fts3Table;
   122    154   typedef struct Fts3Cursor Fts3Cursor;
   123    155   typedef struct Fts3Expr Fts3Expr;
   124    156   typedef struct Fts3Phrase Fts3Phrase;
   125    157   typedef struct Fts3PhraseToken Fts3PhraseToken;
   126    158   
          159  +typedef struct Fts3Doclist Fts3Doclist;
   127    160   typedef struct Fts3SegFilter Fts3SegFilter;
   128    161   typedef struct Fts3DeferredToken Fts3DeferredToken;
   129    162   typedef struct Fts3SegReader Fts3SegReader;
   130         -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
          163  +typedef struct Fts3MultiSegReader Fts3MultiSegReader;
   131    164   
   132    165   /*
   133    166   ** A connection to a fulltext index is an instance of the following
   134    167   ** structure. The xCreate and xConnect methods create an instance
   135    168   ** of this structure and xDestroy and xDisconnect free that instance.
   136    169   ** All other methods receive a pointer to the structure as one of their
   137    170   ** arguments.
................................................................................
   144    177     int nColumn;                    /* number of named columns in virtual table */
   145    178     char **azColumn;                /* column names.  malloced */
   146    179     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   147    180   
   148    181     /* Precompiled statements used by the implementation. Each of these 
   149    182     ** statements is run and reset within a single virtual table API call. 
   150    183     */
   151         -  sqlite3_stmt *aStmt[24];
          184  +  sqlite3_stmt *aStmt[27];
   152    185   
   153    186     char *zReadExprlist;
   154    187     char *zWriteExprlist;
   155    188   
   156    189     int nNodeSize;                  /* Soft limit for node size */
   157    190     u8 bHasStat;                    /* True if %_stat table exists */
   158    191     u8 bHasDocsize;                 /* True if %_docsize table exists */
          192  +  u8 bDescIdx;                    /* True if doclists are in reverse order */
   159    193     int nPgsz;                      /* Page size for host database */
   160    194     char *zSegmentsTbl;             /* Name of %_segments table */
   161    195     sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
   162    196   
   163         -  /* The following hash table is used to buffer pending index updates during
          197  +  /* TODO: Fix the first paragraph of this comment.
          198  +  **
          199  +  ** The following hash table is used to buffer pending index updates during
   164    200     ** transactions. Variable nPendingData estimates the memory size of the 
   165    201     ** pending data, including hash table overhead, but not malloc overhead. 
   166    202     ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 
   167    203     ** automatically. Variable iPrevDocid is the docid of the most recently
   168    204     ** inserted record.
          205  +  **
          206  +  ** A single FTS4 table may have multiple full-text indexes. For each index
          207  +  ** there is an entry in the aIndex[] array. Index 0 is an index of all the
          208  +  ** terms that appear in the document set. Each subsequent index in aIndex[]
          209  +  ** is an index of prefixes of a specific length.
   169    210     */
   170         -  int nMaxPendingData;
   171         -  int nPendingData;
   172         -  sqlite_int64 iPrevDocid;
   173         -  Fts3Hash pendingTerms;
          211  +  int nIndex;                     /* Size of aIndex[] */
          212  +  struct Fts3Index {
          213  +    int nPrefix;                  /* Prefix length (0 for main terms index) */
          214  +    Fts3Hash hPending;            /* Pending terms table for this index */
          215  +  } *aIndex;
          216  +  int nMaxPendingData;            /* Max pending data before flush to disk */
          217  +  int nPendingData;               /* Current bytes of pending data */
          218  +  sqlite_int64 iPrevDocid;        /* Docid of most recently inserted document */
   174    219   
   175    220   #if defined(SQLITE_DEBUG)
   176    221     /* State variables used for validating that the transaction control
   177    222     ** methods of the virtual table are called at appropriate times.  These
   178    223     ** values do not contribution to the FTS computation; they are used for
   179    224     ** verifying the SQLite core.
   180    225     */
................................................................................
   197    242     Fts3Expr *pExpr;                /* Parsed MATCH query string */
   198    243     int nPhrase;                    /* Number of matchable phrases in query */
   199    244     Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
   200    245     sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
   201    246     char *pNextId;                  /* Pointer into the body of aDoclist */
   202    247     char *aDoclist;                 /* List of docids for full-text queries */
   203    248     int nDoclist;                   /* Size of buffer at aDoclist */
   204         -  int desc;                       /* True to sort in descending order */
          249  +  u8 bDesc;                       /* True to sort in descending order */
   205    250     int eEvalmode;                  /* An FTS3_EVAL_XX constant */
   206    251     int nRowAvg;                    /* Average size of database rows, in pages */
          252  +  int nDoc;                       /* Documents in table */
   207    253   
   208    254     int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
   209    255     u32 *aMatchinfo;                /* Information about most recent match */
   210    256     int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
   211    257     char *zMatchinfo;               /* Matchinfo specification */
   212    258   };
   213    259   
................................................................................
   230    276   ** indicating that all columns should be searched,
   231    277   ** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
   232    278   */
   233    279   #define FTS3_FULLSCAN_SEARCH 0    /* Linear scan of %_content table */
   234    280   #define FTS3_DOCID_SEARCH    1    /* Lookup by rowid on %_content table */
   235    281   #define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */
   236    282   
          283  +
          284  +struct Fts3Doclist {
          285  +  char *aAll;                    /* Array containing doclist (or NULL) */
          286  +  int nAll;                      /* Size of a[] in bytes */
          287  +  char *pNextDocid;              /* Pointer to next docid */
          288  +
          289  +  sqlite3_int64 iDocid;          /* Current docid (if pList!=0) */
          290  +  int bFreeList;                 /* True if pList should be sqlite3_free()d */
          291  +  char *pList;                   /* Pointer to position list following iDocid */
          292  +  int nList;                     /* Length of position list */
          293  +} doclist;
          294  +
   237    295   /*
   238    296   ** A "phrase" is a sequence of one or more tokens that must match in
   239    297   ** sequence.  A single token is the base case and the most common case.
   240    298   ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
   241    299   ** nToken will be the number of tokens in the string.
   242         -**
   243         -** The nDocMatch and nMatch variables contain data that may be used by the
   244         -** matchinfo() function. They are populated when the full-text index is 
   245         -** queried for hits on the phrase. If one or more tokens in the phrase
   246         -** are deferred, the nDocMatch and nMatch variables are populated based
   247         -** on the assumption that the 
   248    300   */
   249    301   struct Fts3PhraseToken {
   250    302     char *z;                        /* Text of the token */
   251    303     int n;                          /* Number of bytes in buffer z */
   252    304     int isPrefix;                   /* True if token ends with a "*" character */
   253         -  int bFulltext;                  /* True if full-text index was used */
   254         -  Fts3SegReaderCursor *pSegcsr;   /* Segment-reader for this token */
          305  +
          306  +  /* Variables above this point are populated when the expression is
          307  +  ** parsed (by code in fts3_expr.c). Below this point the variables are
          308  +  ** used when evaluating the expression. */
   255    309     Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
          310  +  Fts3MultiSegReader *pSegcsr;    /* Segment-reader for this token */
   256    311   };
   257    312   
   258    313   struct Fts3Phrase {
   259         -  /* Variables populated by fts3_expr.c when parsing a MATCH expression */
          314  +  /* Cache of doclist for this phrase. */
          315  +  Fts3Doclist doclist;
          316  +  int bIncr;                 /* True if doclist is loaded incrementally */
          317  +  int iDoclistToken;
          318  +
          319  +  /* Variables below this point are populated by fts3_expr.c when parsing 
          320  +  ** a MATCH expression. Everything above is part of the evaluation phase. 
          321  +  */
   260    322     int nToken;                /* Number of tokens in the phrase */
   261    323     int iColumn;               /* Index of column this phrase must match */
   262         -  int isNot;                 /* Phrase prefixed by unary not (-) operator */
   263    324     Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
   264    325   };
   265    326   
   266    327   /*
   267    328   ** A tree of these objects forms the RHS of a MATCH operator.
   268    329   **
   269         -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
   270         -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, 
   271         -** containing the results of the NEAR or phrase query in FTS3 doclist
   272         -** format. As usual, the initial "Length" field found in doclists stored
   273         -** on disk is omitted from this buffer.
          330  +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist 
          331  +** points to a malloced buffer, size nDoclist bytes, containing the results 
          332  +** of this phrase query in FTS3 doclist format. As usual, the initial 
          333  +** "Length" field found in doclists stored on disk is omitted from this 
          334  +** buffer.
          335  +**
          336  +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
          337  +** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
          338  +** where nCol is the number of columns in the queried FTS table. The array
          339  +** is populated as follows:
          340  +**
          341  +**   aMI[iCol*3 + 0] = Undefined
          342  +**   aMI[iCol*3 + 1] = Number of occurrences
          343  +**   aMI[iCol*3 + 2] = Number of rows containing at least one instance
   274    344   **
   275         -** Variable pCurrent always points to the start of a docid field within
   276         -** aDoclist. Since the doclist is usually scanned in docid order, this can
   277         -** be used to accelerate seeking to the required docid within the doclist.
          345  +** The aMI array is allocated using sqlite3_malloc(). It should be freed 
          346  +** when the expression node is.
   278    347   */
   279    348   struct Fts3Expr {
   280    349     int eType;                 /* One of the FTSQUERY_XXX values defined below */
   281    350     int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
   282    351     Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
   283    352     Fts3Expr *pLeft;           /* Left operand */
   284    353     Fts3Expr *pRight;          /* Right operand */
   285    354     Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
   286    355   
   287         -  int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
   288         -  char *aDoclist;            /* Buffer containing doclist */
   289         -  int nDoclist;              /* Size of aDoclist in bytes */
          356  +  /* The following are used by the fts3_eval.c module. */
          357  +  sqlite3_int64 iDocid;      /* Current docid */
          358  +  u8 bEof;                   /* True this expression is at EOF already */
          359  +  u8 bStart;                 /* True if iDocid is valid */
          360  +  u8 bDeferred;              /* True if this expression is entirely deferred */
   290    361   
   291         -  sqlite3_int64 iCurrent;
   292         -  char *pCurrent;
          362  +  u32 *aMI;
   293    363   };
   294    364   
   295    365   /*
   296    366   ** Candidate values for Fts3Query.eType. Note that the order of the first
   297    367   ** four values is in order of precedence when parsing expressions. For 
   298    368   ** example, the following:
   299    369   **
................................................................................
   313    383   /* fts3_write.c */
   314    384   int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
   315    385   int sqlite3Fts3PendingTermsFlush(Fts3Table *);
   316    386   void sqlite3Fts3PendingTermsClear(Fts3Table *);
   317    387   int sqlite3Fts3Optimize(Fts3Table *);
   318    388   int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
   319    389     sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
   320         -int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
          390  +int sqlite3Fts3SegReaderPending(
          391  +  Fts3Table*,int,const char*,int,int,Fts3SegReader**);
   321    392   void sqlite3Fts3SegReaderFree(Fts3SegReader *);
   322         -int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
   323         -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
          393  +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
   324    394   int sqlite3Fts3ReadLock(Fts3Table *);
   325         -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
          395  +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
   326    396   
   327    397   int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
   328    398   int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
   329    399   
   330    400   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
   331    401   int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
   332    402   int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
   333    403   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
   334         -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
   335    404   void sqlite3Fts3SegmentsClose(Fts3Table *);
   336    405   
   337         -#define FTS3_SEGCURSOR_PENDING -1
   338         -#define FTS3_SEGCURSOR_ALL     -2
          406  +/* Special values interpreted by sqlite3SegReaderCursor() */
          407  +#define FTS3_SEGCURSOR_PENDING        -1
          408  +#define FTS3_SEGCURSOR_ALL            -2
   339    409   
   340         -int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
   341         -int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
   342         -void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
          410  +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
          411  +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
          412  +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
          413  +
   343    414   int sqlite3Fts3SegReaderCursor(
   344         -    Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
          415  +    Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
   345    416   
   346    417   /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
   347    418   #define FTS3_SEGMENT_REQUIRE_POS   0x00000001
   348    419   #define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
   349    420   #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
   350    421   #define FTS3_SEGMENT_PREFIX        0x00000008
   351    422   #define FTS3_SEGMENT_SCAN          0x00000010
................................................................................
   354    425   struct Fts3SegFilter {
   355    426     const char *zTerm;
   356    427     int nTerm;
   357    428     int iCol;
   358    429     int flags;
   359    430   };
   360    431   
   361         -struct Fts3SegReaderCursor {
          432  +struct Fts3MultiSegReader {
   362    433     /* Used internally by sqlite3Fts3SegReaderXXX() calls */
   363    434     Fts3SegReader **apSegment;      /* Array of Fts3SegReader objects */
   364    435     int nSegment;                   /* Size of apSegment array */
   365    436     int nAdvance;                   /* How many seg-readers to advance */
   366    437     Fts3SegFilter *pFilter;         /* Pointer to filter object */
   367    438     char *aBuffer;                  /* Buffer to merge doclists in */
   368    439     int nBuffer;                    /* Allocated size of aBuffer[] in bytes */
   369    440   
   370         -  /* Cost of running this iterator. Used by fts3.c only. */
   371         -  int nCost;
          441  +  int iColFilter;                 /* If >=0, filter for this column */
          442  +  int bRestart;
          443  +
          444  +  /* Used by fts3.c only. */
          445  +  int nCost;                      /* Cost of running iterator */
          446  +  int bLookup;                    /* True if a lookup of a single entry. */
   372    447   
   373    448     /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
   374    449     char *zTerm;                    /* Pointer to term buffer */
   375    450     int nTerm;                      /* Size of zTerm in bytes */
   376    451     char *aDoclist;                 /* Pointer to doclist buffer */
   377    452     int nDoclist;                   /* Size of aDoclist[] in bytes */
   378    453   };
................................................................................
   379    454   
   380    455   /* fts3.c */
   381    456   int sqlite3Fts3PutVarint(char *, sqlite3_int64);
   382    457   int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
   383    458   int sqlite3Fts3GetVarint32(const char *, int *);
   384    459   int sqlite3Fts3VarintLen(sqlite3_uint64);
   385    460   void sqlite3Fts3Dequote(char *);
          461  +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
   386    462   
   387         -char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
   388         -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
   389         -int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
   390         -int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
          463  +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
   391    464   
   392    465   /* fts3_tokenizer.c */
   393    466   const char *sqlite3Fts3NextToken(const char *, int *);
   394    467   int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
   395    468   int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, 
   396    469       sqlite3_tokenizer **, char **
   397    470   );
................................................................................
   413    486   int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
   414    487   int sqlite3Fts3InitTerm(sqlite3 *db);
   415    488   #endif
   416    489   
   417    490   /* fts3_aux.c */
   418    491   int sqlite3Fts3InitAux(sqlite3 *db);
   419    492   
          493  +int sqlite3Fts3TermSegReaderCursor(
          494  +  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
          495  +  const char *zTerm,              /* Term to query for */
          496  +  int nTerm,                      /* Size of zTerm in bytes */
          497  +  int isPrefix,                   /* True for a prefix search */
          498  +  Fts3MultiSegReader **ppSegcsr   /* OUT: Allocated seg-reader cursor */
          499  +);
          500  +
          501  +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
          502  +
          503  +int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
          504  +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
          505  +
          506  +int sqlite3Fts3MsrIncrStart(
          507  +    Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
          508  +int sqlite3Fts3MsrIncrNext(
          509  +    Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
          510  +char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); 
          511  +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
          512  +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
          513  +
          514  +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
          515  +
          516  +#endif /* SQLITE_ENABLE_FTS3 */
   420    517   #endif /* _FTSINT_H */

Changes to ext/fts3/fts3_aux.c.

     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   */
    14         -
           14  +#include "fts3Int.h"
    15     15   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    16     16   
    17         -#include "fts3Int.h"
    18     17   #include <string.h>
    19     18   #include <assert.h>
    20     19   
    21     20   typedef struct Fts3auxTable Fts3auxTable;
    22     21   typedef struct Fts3auxCursor Fts3auxCursor;
    23     22   
    24     23   struct Fts3auxTable {
    25     24     sqlite3_vtab base;              /* Base class used by SQLite core */
    26     25     Fts3Table *pFts3Tab;
    27     26   };
    28     27   
    29     28   struct Fts3auxCursor {
    30     29     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
    31         -  Fts3SegReaderCursor csr;        /* Must be right after "base" */
           30  +  Fts3MultiSegReader csr;        /* Must be right after "base" */
    32     31     Fts3SegFilter filter;
    33     32     char *zStop;
    34     33     int nStop;                      /* Byte-length of string zStop */
    35     34     int isEof;                      /* True if cursor is at EOF */
    36     35     sqlite3_int64 iRowid;           /* Current rowid */
    37     36   
    38     37     int iCol;                       /* Current value of 'col' column */
................................................................................
    92     91     if( !p ) return SQLITE_NOMEM;
    93     92     memset(p, 0, nByte);
    94     93   
    95     94     p->pFts3Tab = (Fts3Table *)&p[1];
    96     95     p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
    97     96     p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
    98     97     p->pFts3Tab->db = db;
           98  +  p->pFts3Tab->nIndex = 1;
    99     99   
   100    100     memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
   101    101     memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
   102    102     sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
   103    103   
   104    104     *ppVtab = (sqlite3_vtab *)p;
   105    105     return SQLITE_OK;
................................................................................
   372    372     if( idxNum&FTS4AUX_LE_CONSTRAINT ){
   373    373       int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0;
   374    374       pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
   375    375       pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
   376    376       if( pCsr->zStop==0 ) return SQLITE_NOMEM;
   377    377     }
   378    378   
   379         -  rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
          379  +  rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
   380    380         pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
   381    381     );
   382    382     if( rc==SQLITE_OK ){
   383    383       rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
   384    384     }
   385    385   
   386    386     if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);

Changes to ext/fts3/fts3_expr.c.

    11     11   ******************************************************************************
    12     12   **
    13     13   ** This module contains code that implements a parser for fts3 query strings
    14     14   ** (the right-hand argument to the MATCH operator). Because the supported 
    15     15   ** syntax is relatively simple, the whole tokenizer/parser system is
    16     16   ** hand-coded. 
    17     17   */
           18  +#include "fts3Int.h"
    18     19   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    19     20   
    20     21   /*
    21     22   ** By default, this module parses the legacy syntax that has been 
    22     23   ** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
    23     24   ** is defined, then it uses the new syntax. The differences between
    24     25   ** the new and the old syntaxes are:
................................................................................
    73     74   #endif
    74     75   
    75     76   /*
    76     77   ** Default span for NEAR operators.
    77     78   */
    78     79   #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
    79     80   
    80         -#include "fts3Int.h"
    81     81   #include <string.h>
    82     82   #include <assert.h>
    83     83   
           84  +/*
           85  +** isNot:
           86  +**   This variable is used by function getNextNode(). When getNextNode() is
           87  +**   called, it sets ParseContext.isNot to true if the 'next node' is a 
           88  +**   FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
           89  +**   FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
           90  +**   zero.
           91  +*/
    84     92   typedef struct ParseContext ParseContext;
    85     93   struct ParseContext {
    86     94     sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
    87     95     const char **azCol;                 /* Array of column names for fts3 table */
    88     96     int nCol;                           /* Number of entries in azCol[] */
    89     97     int iDefaultCol;                    /* Default column to query */
           98  +  int isNot;                          /* True if getNextNode() sees a unary - */
    90     99     sqlite3_context *pCtx;              /* Write error message here */
    91    100     int nNest;                          /* Number of nested brackets */
    92    101   };
    93    102   
    94    103   /*
    95    104   ** This function is equivalent to the standard isspace() function. 
    96    105   **
................................................................................
   168    177           memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
   169    178   
   170    179           if( iEnd<n && z[iEnd]=='*' ){
   171    180             pRet->pPhrase->aToken[0].isPrefix = 1;
   172    181             iEnd++;
   173    182           }
   174    183           if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
   175         -          pRet->pPhrase->isNot = 1;
          184  +          pParse->isNot = 1;
   176    185           }
   177    186         }
   178    187         nConsumed = iEnd;
   179    188       }
   180    189   
   181    190       pModule->xClose(pCursor);
   182    191     }
................................................................................
   220    229     sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
   221    230     int rc;
   222    231     Fts3Expr *p = 0;
   223    232     sqlite3_tokenizer_cursor *pCursor = 0;
   224    233     char *zTemp = 0;
   225    234     int nTemp = 0;
   226    235   
          236  +  const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
          237  +  int nToken = 0;
          238  +
          239  +  /* The final Fts3Expr data structure, including the Fts3Phrase,
          240  +  ** Fts3PhraseToken structures token buffers are all stored as a single 
          241  +  ** allocation so that the expression can be freed with a single call to
          242  +  ** sqlite3_free(). Setting this up requires a two pass approach.
          243  +  **
          244  +  ** The first pass, in the block below, uses a tokenizer cursor to iterate
          245  +  ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
          246  +  ** to assemble data in two dynamic buffers:
          247  +  **
          248  +  **   Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
          249  +  **             structure, followed by the array of Fts3PhraseToken 
          250  +  **             structures. This pass only populates the Fts3PhraseToken array.
          251  +  **
          252  +  **   Buffer zTemp: Contains copies of all tokens.
          253  +  **
          254  +  ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
          255  +  ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
          256  +  ** structures.
          257  +  */
   227    258     rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
   228    259     if( rc==SQLITE_OK ){
   229    260       int ii;
   230    261       pCursor->pTokenizer = pTokenizer;
   231    262       for(ii=0; rc==SQLITE_OK; ii++){
   232         -      const char *zToken;
   233         -      int nToken, iBegin, iEnd, iPos;
   234         -      rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
          263  +      const char *zByte;
          264  +      int nByte, iBegin, iEnd, iPos;
          265  +      rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
   235    266         if( rc==SQLITE_OK ){
   236         -        int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   237         -        p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
   238         -        zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
   239         -        if( !p || !zTemp ){
   240         -          goto no_mem;
   241         -        }
   242         -        if( ii==0 ){
   243         -          memset(p, 0, nByte);
   244         -          p->pPhrase = (Fts3Phrase *)&p[1];
   245         -        }
   246         -        p->pPhrase = (Fts3Phrase *)&p[1];
   247         -        memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
   248         -        p->pPhrase->nToken = ii+1;
   249         -        p->pPhrase->aToken[ii].n = nToken;
   250         -        memcpy(&zTemp[nTemp], zToken, nToken);
   251         -        nTemp += nToken;
   252         -        if( iEnd<nInput && zInput[iEnd]=='*' ){
   253         -          p->pPhrase->aToken[ii].isPrefix = 1;
   254         -        }else{
   255         -          p->pPhrase->aToken[ii].isPrefix = 0;
   256         -        }
          267  +        Fts3PhraseToken *pToken;
          268  +
          269  +        p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
          270  +        if( !p ) goto no_mem;
          271  +
          272  +        zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
          273  +        if( !zTemp ) goto no_mem;
          274  +
          275  +        assert( nToken==ii );
          276  +        pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
          277  +        memset(pToken, 0, sizeof(Fts3PhraseToken));
          278  +
          279  +        memcpy(&zTemp[nTemp], zByte, nByte);
          280  +        nTemp += nByte;
          281  +
          282  +        pToken->n = nByte;
          283  +        pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
          284  +        nToken = ii+1;
   257    285         }
   258    286       }
   259    287   
   260    288       pModule->xClose(pCursor);
   261    289       pCursor = 0;
   262    290     }
   263    291   
   264    292     if( rc==SQLITE_DONE ){
   265    293       int jj;
   266         -    char *zNew = NULL;
   267         -    int nNew = 0;
   268         -    int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   269         -    nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
   270         -    p = fts3ReallocOrFree(p, nByte + nTemp);
   271         -    if( !p ){
   272         -      goto no_mem;
   273         -    }
   274         -    if( zTemp ){
   275         -      zNew = &(((char *)p)[nByte]);
   276         -      memcpy(zNew, zTemp, nTemp);
   277         -    }else{
   278         -      memset(p, 0, nByte+nTemp);
   279         -    }
          294  +    char *zBuf = 0;
          295  +
          296  +    p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
          297  +    if( !p ) goto no_mem;
          298  +    memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
          299  +    p->eType = FTSQUERY_PHRASE;
   280    300       p->pPhrase = (Fts3Phrase *)&p[1];
          301  +    p->pPhrase->iColumn = pParse->iDefaultCol;
          302  +    p->pPhrase->nToken = nToken;
          303  +
          304  +    zBuf = (char *)&p->pPhrase->aToken[nToken];
          305  +    memcpy(zBuf, zTemp, nTemp);
          306  +    sqlite3_free(zTemp);
          307  +
   281    308       for(jj=0; jj<p->pPhrase->nToken; jj++){
   282         -      p->pPhrase->aToken[jj].z = &zNew[nNew];
   283         -      nNew += p->pPhrase->aToken[jj].n;
          309  +      p->pPhrase->aToken[jj].z = zBuf;
          310  +      zBuf += p->pPhrase->aToken[jj].n;
   284    311       }
   285         -    sqlite3_free(zTemp);
   286         -    p->eType = FTSQUERY_PHRASE;
   287         -    p->pPhrase->iColumn = pParse->iDefaultCol;
   288    312       rc = SQLITE_OK;
   289    313     }
   290    314   
   291    315     *ppExpr = p;
   292    316     return rc;
   293    317   no_mem:
   294    318   
................................................................................
   336    360     int iCol;
   337    361     int iColLen;
   338    362     int rc;
   339    363     Fts3Expr *pRet = 0;
   340    364   
   341    365     const char *zInput = z;
   342    366     int nInput = n;
          367  +
          368  +  pParse->isNot = 0;
   343    369   
   344    370     /* Skip over any whitespace before checking for a keyword, an open or
   345    371     ** close bracket, or a quoted string. 
   346    372     */
   347    373     while( nInput>0 && fts3isspace(*zInput) ){
   348    374       nInput--;
   349    375       zInput++;
................................................................................
   555    581       Fts3Expr *p = 0;
   556    582       int nByte = 0;
   557    583       rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
   558    584       if( rc==SQLITE_OK ){
   559    585         int isPhrase;
   560    586   
   561    587         if( !sqlite3_fts3_enable_parentheses 
   562         -       && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot 
          588  +       && p->eType==FTSQUERY_PHRASE && pParse->isNot 
   563    589         ){
   564    590           /* Create an implicit NOT operator. */
   565    591           Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
   566    592           if( !pNot ){
   567    593             sqlite3Fts3ExprFree(p);
   568    594             rc = SQLITE_NOMEM;
   569    595             goto exprparse_out;
................................................................................
   573    599           if( pNotBranch ){
   574    600             pNot->pLeft = pNotBranch;
   575    601           }
   576    602           pNotBranch = pNot;
   577    603           p = pPrev;
   578    604         }else{
   579    605           int eType = p->eType;
   580         -        assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
   581    606           isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
   582    607   
   583    608           /* The isRequirePhrase variable is set to true if a phrase or
   584    609           ** an expression contained in parenthesis is required. If a
   585    610           ** binary operator (AND, OR, NOT or NEAR) is encounted when
   586    611           ** isRequirePhrase is set, this is a syntax error.
   587    612           */
................................................................................
   736    761   }
   737    762   
   738    763   /*
   739    764   ** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
   740    765   */
   741    766   void sqlite3Fts3ExprFree(Fts3Expr *p){
   742    767     if( p ){
          768  +    assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
   743    769       sqlite3Fts3ExprFree(p->pLeft);
   744    770       sqlite3Fts3ExprFree(p->pRight);
   745         -    sqlite3_free(p->aDoclist);
          771  +    sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
          772  +    sqlite3_free(p->aMI);
   746    773       sqlite3_free(p);
   747    774     }
   748    775   }
   749    776   
   750    777   /****************************************************************************
   751    778   *****************************************************************************
   752    779   ** Everything after this point is just test code.
................................................................................
   796    823   */
   797    824   static char *exprToString(Fts3Expr *pExpr, char *zBuf){
   798    825     switch( pExpr->eType ){
   799    826       case FTSQUERY_PHRASE: {
   800    827         Fts3Phrase *pPhrase = pExpr->pPhrase;
   801    828         int i;
   802    829         zBuf = sqlite3_mprintf(
   803         -          "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
          830  +          "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
   804    831         for(i=0; zBuf && i<pPhrase->nToken; i++){
   805    832           zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, 
   806    833               pPhrase->aToken[i].n, pPhrase->aToken[i].z,
   807    834               (pPhrase->aToken[i].isPrefix?"+":"")
   808    835           );
   809    836         }
   810    837         return zBuf;

Changes to ext/fts3/fts3_hash.c.

    19     19   **
    20     20   **     * The FTS3 module is being built as an extension
    21     21   **       (in which case SQLITE_CORE is not defined), or
    22     22   **
    23     23   **     * The FTS3 module is being built into the core of
    24     24   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    25     25   */
           26  +#include "fts3Int.h"
    26     27   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    27     28   
    28     29   #include <assert.h>
    29     30   #include <stdlib.h>
    30     31   #include <string.h>
    31     32   
    32     33   #include "sqlite3.h"

Changes to ext/fts3/fts3_icu.c.

     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** This file implements a tokenizer for fts3 based on the ICU library.
    13         -** 
    14         -** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $
    15     13   */
    16         -
           14  +#include "fts3Int.h"
    17     15   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    18     16   #ifdef SQLITE_ENABLE_ICU
    19     17   
    20     18   #include <assert.h>
    21     19   #include <string.h>
    22     20   #include "fts3_tokenizer.h"
    23     21   

Changes to ext/fts3/fts3_porter.c.

    18     18   **
    19     19   **     * The FTS3 module is being built as an extension
    20     20   **       (in which case SQLITE_CORE is not defined), or
    21     21   **
    22     22   **     * The FTS3 module is being built into the core of
    23     23   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    24     24   */
    25         -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    26         -
    27     25   #include "fts3Int.h"
           26  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    28     27   
    29     28   #include <assert.h>
    30     29   #include <stdlib.h>
    31     30   #include <stdio.h>
    32     31   #include <string.h>
    33     32   
    34     33   #include "fts3_tokenizer.h"

Changes to ext/fts3/fts3_snippet.c.

     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   */
    13     13   
           14  +#include "fts3Int.h"
    14     15   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    15     16   
    16         -#include "fts3Int.h"
    17     17   #include <string.h>
    18     18   #include <assert.h>
    19     19   
    20     20   /*
    21     21   ** Characters that may appear in the second argument to matchinfo().
    22     22   */
    23     23   #define FTS3_MATCHINFO_NPHRASE   'p'        /* 1 value */
................................................................................
   172    172     int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   173    173     void *pCtx                      /* Second argument to pass to callback */
   174    174   ){
   175    175     int iPhrase = 0;                /* Variable used as the phrase counter */
   176    176     return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
   177    177   }
   178    178   
   179         -/*
   180         -** The argument to this function is always a phrase node. Its doclist 
   181         -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
   182         -** to the left of this one in the query tree have already been loaded.
   183         -**
   184         -** If this phrase node is part of a series of phrase nodes joined by 
   185         -** NEAR operators (and is not the left-most of said series), then elements are
   186         -** removed from the phrases doclist consistent with the NEAR restriction. If
   187         -** required, elements may be removed from the doclists of phrases to the
   188         -** left of this one that are part of the same series of NEAR operator 
   189         -** connected phrases.
   190         -**
   191         -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
   192         -*/
   193         -static int fts3ExprNearTrim(Fts3Expr *pExpr){
   194         -  int rc = SQLITE_OK;
   195         -  Fts3Expr *pParent = pExpr->pParent;
   196         -
   197         -  assert( pExpr->eType==FTSQUERY_PHRASE );
   198         -  while( rc==SQLITE_OK
   199         -   && pParent 
   200         -   && pParent->eType==FTSQUERY_NEAR 
   201         -   && pParent->pRight==pExpr 
   202         -  ){
   203         -    /* This expression (pExpr) is the right-hand-side of a NEAR operator. 
   204         -    ** Find the expression to the left of the same operator.
   205         -    */
   206         -    int nNear = pParent->nNear;
   207         -    Fts3Expr *pLeft = pParent->pLeft;
   208         -
   209         -    if( pLeft->eType!=FTSQUERY_PHRASE ){
   210         -      assert( pLeft->eType==FTSQUERY_NEAR );
   211         -      assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
   212         -      pLeft = pLeft->pRight;
   213         -    }
   214         -
   215         -    rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
   216         -
   217         -    pExpr = pLeft;
   218         -    pParent = pExpr->pParent;
   219         -  }
   220         -
   221         -  return rc;
   222         -}
   223         -
   224    179   /*
   225    180   ** This is an fts3ExprIterate() callback used while loading the doclists
   226    181   ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
   227    182   ** fts3ExprLoadDoclists().
   228    183   */
   229    184   static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   230    185     int rc = SQLITE_OK;
          186  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
   231    187     LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
   232    188   
   233    189     UNUSED_PARAMETER(iPhrase);
   234    190   
   235    191     p->nPhrase++;
   236         -  p->nToken += pExpr->pPhrase->nToken;
   237         -
   238         -  if( pExpr->isLoaded==0 ){
   239         -    rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
   240         -    pExpr->isLoaded = 1;
   241         -    if( rc==SQLITE_OK ){
   242         -      rc = fts3ExprNearTrim(pExpr);
   243         -    }
   244         -  }
          192  +  p->nToken += pPhrase->nToken;
   245    193   
   246    194     return rc;
   247    195   }
   248    196   
   249    197   /*
   250    198   ** Load the doclists for each phrase in the query associated with FTS3 cursor
   251    199   ** pCsr. 
................................................................................
   411    359   static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
   412    360     SnippetIter *p = (SnippetIter *)ctx;
   413    361     SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
   414    362     char *pCsr;
   415    363   
   416    364     pPhrase->nToken = pExpr->pPhrase->nToken;
   417    365   
   418         -  pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol);
          366  +  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
   419    367     if( pCsr ){
   420    368       int iFirst = 0;
   421    369       pPhrase->pList = pCsr;
   422    370       fts3GetDeltaPosition(&pCsr, &iFirst);
   423    371       pPhrase->pHead = pCsr;
   424    372       pPhrase->pTail = pCsr;
   425    373       pPhrase->iHead = iFirst;
................................................................................
   768    716       if( !c ) nEntry++;
   769    717     }
   770    718   
   771    719     *ppCollist = pEnd;
   772    720     return nEntry;
   773    721   }
   774    722   
   775         -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
   776         -  char *pCsr = *pp;
   777         -  while( *pCsr ){
   778         -    int nHit;
   779         -    sqlite3_int64 iCol = 0;
   780         -    if( *pCsr==0x01 ){
   781         -      pCsr++;
   782         -      pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
   783         -    }
   784         -    nHit = fts3ColumnlistCount(&pCsr);
   785         -    assert( nHit>0 );
   786         -    if( isGlobal ){
   787         -      aOut[iCol*3+1]++;
   788         -    }
   789         -    aOut[iCol*3] += nHit;
   790         -  }
   791         -  pCsr++;
   792         -  *pp = pCsr;
   793         -}
   794         -
   795    723   /*
   796    724   ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
   797    725   ** for a single query. 
   798    726   **
   799    727   ** fts3ExprIterate() callback to load the 'global' elements of a
   800    728   ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
   801    729   ** of the matchinfo array that are constant for all rows returned by the 
................................................................................
   821    749   */
   822    750   static int fts3ExprGlobalHitsCb(
   823    751     Fts3Expr *pExpr,                /* Phrase expression node */
   824    752     int iPhrase,                    /* Phrase number (numbered from zero) */
   825    753     void *pCtx                      /* Pointer to MatchInfo structure */
   826    754   ){
   827    755     MatchInfo *p = (MatchInfo *)pCtx;
   828         -  Fts3Cursor *pCsr = p->pCursor;
   829         -  char *pIter;
   830         -  char *pEnd;
   831         -  char *pFree = 0;
   832         -  u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
   833         -
   834         -  assert( pExpr->isLoaded );
   835         -  assert( pExpr->eType==FTSQUERY_PHRASE );
   836         -
   837         -  if( pCsr->pDeferred ){
   838         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
   839         -    int ii;
   840         -    for(ii=0; ii<pPhrase->nToken; ii++){
   841         -      if( pPhrase->aToken[ii].bFulltext ) break;
   842         -    }
   843         -    if( ii<pPhrase->nToken ){
   844         -      int nFree = 0;
   845         -      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
   846         -      if( rc!=SQLITE_OK ) return rc;
   847         -      pIter = pFree;
   848         -      pEnd = &pFree[nFree];
   849         -    }else{
   850         -      int iCol;                   /* Column index */
   851         -      for(iCol=0; iCol<p->nCol; iCol++){
   852         -        aOut[iCol*3 + 1] = (u32)p->nDoc;
   853         -        aOut[iCol*3 + 2] = (u32)p->nDoc;
   854         -      }
   855         -      return SQLITE_OK;
   856         -    }
   857         -  }else{
   858         -    pIter = pExpr->aDoclist;
   859         -    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
   860         -  }
   861         -
   862         -  /* Fill in the global hit count matrix row for this phrase. */
   863         -  while( pIter<pEnd ){
   864         -    while( *pIter++ & 0x80 );      /* Skip past docid. */
   865         -    fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
   866         -  }
   867         -
   868         -  sqlite3_free(pFree);
   869         -  return SQLITE_OK;
          756  +  return sqlite3Fts3EvalPhraseStats(
          757  +      p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
          758  +  );
   870    759   }
   871    760   
   872    761   /*
   873    762   ** fts3ExprIterate() callback used to collect the "local" part of the
   874    763   ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the 
   875    764   ** array that are different for each row returned by the query.
   876    765   */
................................................................................
   879    768     int iPhrase,                    /* Phrase number */
   880    769     void *pCtx                      /* Pointer to MatchInfo structure */
   881    770   ){
   882    771     MatchInfo *p = (MatchInfo *)pCtx;
   883    772     int iStart = iPhrase * p->nCol * 3;
   884    773     int i;
   885    774   
   886         -  for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
   887         -
   888         -  if( pExpr->aDoclist ){
          775  +  for(i=0; i<p->nCol; i++){
   889    776       char *pCsr;
   890         -
   891         -    pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
          777  +    pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
   892    778       if( pCsr ){
   893         -      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
          779  +      p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
          780  +    }else{
          781  +      p->aMatchinfo[iStart+i*3] = 0;
   894    782       }
   895    783     }
   896    784   
   897    785     return SQLITE_OK;
   898    786   }
   899    787   
   900    788   static int fts3MatchinfoCheck(
................................................................................
   972    860   ** iterating through a multi-column position-list corresponding to the
   973    861   ** hits for a single phrase on a single row in order to calculate the
   974    862   ** values for a matchinfo() FTS3_MATCHINFO_LCS request.
   975    863   */
   976    864   typedef struct LcsIterator LcsIterator;
   977    865   struct LcsIterator {
   978    866     Fts3Expr *pExpr;                /* Pointer to phrase expression */
   979         -  char *pRead;                    /* Cursor used to iterate through aDoclist */
   980    867     int iPosOffset;                 /* Tokens count up to end of this phrase */
   981         -  int iCol;                       /* Current column number */
          868  +  char *pRead;                    /* Cursor used to iterate through aDoclist */
   982    869     int iPos;                       /* Current position */
   983    870   };
   984    871   
   985    872   /* 
   986    873   ** If LcsIterator.iCol is set to the following value, the iterator has
   987    874   ** finished iterating through all offsets for all columns.
   988    875   */
................................................................................
  1005    892   */
  1006    893   static int fts3LcsIteratorAdvance(LcsIterator *pIter){
  1007    894     char *pRead = pIter->pRead;
  1008    895     sqlite3_int64 iRead;
  1009    896     int rc = 0;
  1010    897   
  1011    898     pRead += sqlite3Fts3GetVarint(pRead, &iRead);
  1012         -  if( iRead==0 ){
  1013         -    pIter->iCol = LCS_ITERATOR_FINISHED;
          899  +  if( iRead==0 || iRead==1 ){
          900  +    pRead = 0;
  1014    901       rc = 1;
  1015    902     }else{
  1016         -    if( iRead==1 ){
  1017         -      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
  1018         -      pIter->iCol = (int)iRead;
  1019         -      pIter->iPos = pIter->iPosOffset;
  1020         -      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
  1021         -      rc = 1;
  1022         -    }
  1023    903       pIter->iPos += (int)(iRead-2);
  1024    904     }
  1025    905   
  1026    906     pIter->pRead = pRead;
  1027    907     return rc;
  1028    908   }
  1029    909     
................................................................................
  1047    927     /* Allocate and populate the array of LcsIterator objects. The array
  1048    928     ** contains one element for each matchable phrase in the query.
  1049    929     **/
  1050    930     aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
  1051    931     if( !aIter ) return SQLITE_NOMEM;
  1052    932     memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
  1053    933     (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
          934  +
  1054    935     for(i=0; i<pInfo->nPhrase; i++){
  1055    936       LcsIterator *pIter = &aIter[i];
  1056    937       nToken -= pIter->pExpr->pPhrase->nToken;
  1057    938       pIter->iPosOffset = nToken;
  1058         -    pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
  1059         -    if( pIter->pRead ){
  1060         -      pIter->iPos = pIter->iPosOffset;
  1061         -      fts3LcsIteratorAdvance(&aIter[i]);
  1062         -    }else{
  1063         -      pIter->iCol = LCS_ITERATOR_FINISHED;
  1064         -    }
  1065    939     }
  1066    940   
  1067    941     for(iCol=0; iCol<pInfo->nCol; iCol++){
  1068    942       int nLcs = 0;                 /* LCS value for this column */
  1069    943       int nLive = 0;                /* Number of iterators in aIter not at EOF */
  1070    944   
  1071         -    /* Loop through the iterators in aIter[]. Set nLive to the number of
  1072         -    ** iterators that point to a position-list corresponding to column iCol.
  1073         -    */
  1074    945       for(i=0; i<pInfo->nPhrase; i++){
  1075         -      assert( aIter[i].iCol>=iCol );
  1076         -      if( aIter[i].iCol==iCol ) nLive++;
          946  +      LcsIterator *pIt = &aIter[i];
          947  +      pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
          948  +      if( pIt->pRead ){
          949  +        pIt->iPos = pIt->iPosOffset;
          950  +        fts3LcsIteratorAdvance(&aIter[i]);
          951  +        nLive++;
          952  +      }
  1077    953       }
  1078    954   
  1079         -    /* The following loop runs until all iterators in aIter[] have finished
  1080         -    ** iterating through positions in column iCol. Exactly one of the 
  1081         -    ** iterators is advanced each time the body of the loop is run.
  1082         -    */
  1083    955       while( nLive>0 ){
  1084    956         LcsIterator *pAdv = 0;      /* The iterator to advance by one position */
  1085    957         int nThisLcs = 0;           /* LCS for the current iterator positions */
  1086    958   
  1087    959         for(i=0; i<pInfo->nPhrase; i++){
  1088    960           LcsIterator *pIter = &aIter[i];
  1089         -        if( iCol!=pIter->iCol ){  
          961  +        if( pIter->pRead==0 ){
  1090    962             /* This iterator is already at EOF for this column. */
  1091    963             nThisLcs = 0;
  1092    964           }else{
  1093    965             if( pAdv==0 || pIter->iPos<pAdv->iPos ){
  1094    966               pAdv = pIter;
  1095    967             }
  1096    968             if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
................................................................................
  1422   1294     TermOffsetCtx *p = (TermOffsetCtx *)ctx;
  1423   1295     int nTerm;                      /* Number of tokens in phrase */
  1424   1296     int iTerm;                      /* For looping through nTerm phrase terms */
  1425   1297     char *pList;                    /* Pointer to position list for phrase */
  1426   1298     int iPos = 0;                   /* First position in position-list */
  1427   1299   
  1428   1300     UNUSED_PARAMETER(iPhrase);
  1429         -  pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol);
         1301  +  pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
  1430   1302     nTerm = pExpr->pPhrase->nToken;
  1431   1303     if( pList ){
  1432   1304       fts3GetDeltaPosition(&pList, &iPos);
  1433   1305       assert( iPos>=0 );
  1434   1306     }
  1435   1307   
  1436   1308     for(iTerm=0; iTerm<nTerm; iTerm++){

Changes to ext/fts3/fts3_term.c.

    11     11   ******************************************************************************
    12     12   **
    13     13   ** This file is not part of the production FTS code. It is only used for
    14     14   ** testing. It contains a virtual table implementation that provides direct 
    15     15   ** access to the full-text index of an FTS table. 
    16     16   */
    17     17   
           18  +#include "fts3Int.h"
    18     19   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    19     20   #ifdef SQLITE_TEST
    20     21   
    21         -#include "fts3Int.h"
    22     22   #include <string.h>
    23     23   #include <assert.h>
    24     24   
    25     25   typedef struct Fts3termTable Fts3termTable;
    26     26   typedef struct Fts3termCursor Fts3termCursor;
    27     27   
    28     28   struct Fts3termTable {
    29     29     sqlite3_vtab base;              /* Base class used by SQLite core */
           30  +  int iIndex;                     /* Index for Fts3Table.aIndex[] */
    30     31     Fts3Table *pFts3Tab;
    31     32   };
    32     33   
    33     34   struct Fts3termCursor {
    34     35     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
    35         -  Fts3SegReaderCursor csr;        /* Must be right after "base" */
           36  +  Fts3MultiSegReader csr;        /* Must be right after "base" */
    36     37     Fts3SegFilter filter;
    37     38   
    38     39     int isEof;                      /* True if cursor is at EOF */
    39     40     char *pNext;
    40     41   
    41     42     sqlite3_int64 iRowid;           /* Current 'rowid' value */
    42     43     sqlite3_int64 iDocid;           /* Current 'docid' value */
................................................................................
    52     53   /*
    53     54   ** This function does all the work for both the xConnect and xCreate methods.
    54     55   ** These tables have no persistent representation of their own, so xConnect
    55     56   ** and xCreate are identical operations.
    56     57   */
    57     58   static int fts3termConnectMethod(
    58     59     sqlite3 *db,                    /* Database connection */
    59         -  void *pUnused,                  /* Unused */
           60  +  void *pCtx,                     /* Non-zero for an fts4prefix table */
    60     61     int argc,                       /* Number of elements in argv array */
    61     62     const char * const *argv,       /* xCreate/xConnect argument array */
    62     63     sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
    63     64     char **pzErr                    /* OUT: sqlite3_malloc'd error message */
    64     65   ){
    65     66     char const *zDb;                /* Name of database (e.g. "main") */
    66     67     char const *zFts3;              /* Name of fts3 table */
    67     68     int nDb;                        /* Result of strlen(zDb) */
    68     69     int nFts3;                      /* Result of strlen(zFts3) */
    69     70     int nByte;                      /* Bytes of space to allocate here */
    70     71     int rc;                         /* value returned by declare_vtab() */
    71     72     Fts3termTable *p;                /* Virtual table object to return */
           73  +  int iIndex = 0;
    72     74   
    73         -  UNUSED_PARAMETER(pUnused);
           75  +  if( argc==5 ){
           76  +    iIndex = atoi(argv[4]);
           77  +    argc--;
           78  +  }
    74     79   
    75     80     /* The user should specify a single argument - the name of an fts3 table. */
    76     81     if( argc!=4 ){
    77     82       *pzErr = sqlite3_mprintf(
    78     83           "wrong number of arguments to fts4term constructor"
    79     84       );
    80     85       return SQLITE_ERROR;
................................................................................
    93     98     if( !p ) return SQLITE_NOMEM;
    94     99     memset(p, 0, nByte);
    95    100   
    96    101     p->pFts3Tab = (Fts3Table *)&p[1];
    97    102     p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
    98    103     p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
    99    104     p->pFts3Tab->db = db;
          105  +  p->pFts3Tab->nIndex = iIndex+1;
          106  +  p->iIndex = iIndex;
   100    107   
   101    108     memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
   102    109     memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
   103    110     sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
   104    111   
   105    112     *ppVtab = (sqlite3_vtab *)p;
   106    113     return SQLITE_OK;
................................................................................
   240    247     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
   241    248     int idxNum,                     /* Strategy index */
   242    249     const char *idxStr,             /* Unused */
   243    250     int nVal,                       /* Number of elements in apVal */
   244    251     sqlite3_value **apVal           /* Arguments for the indexing scheme */
   245    252   ){
   246    253     Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
   247         -  Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
          254  +  Fts3termTable *p = (Fts3termTable *)pCursor->pVtab;
          255  +  Fts3Table *pFts3 = p->pFts3Tab;
   248    256     int rc;
   249    257   
   250    258     UNUSED_PARAMETER(nVal);
   251    259     UNUSED_PARAMETER(idxNum);
   252    260     UNUSED_PARAMETER(idxStr);
   253    261     UNUSED_PARAMETER(apVal);
   254    262   
................................................................................
   258    266     testcase(pCsr->filter.zTerm);
   259    267     sqlite3Fts3SegReaderFinish(&pCsr->csr);
   260    268     memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
   261    269   
   262    270     pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
   263    271     pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
   264    272   
   265         -  rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
          273  +  rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL,
   266    274         pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr
   267    275     );
   268    276     if( rc==SQLITE_OK ){
   269    277       rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
   270    278     }
   271    279     if( rc==SQLITE_OK ){
   272    280       rc = fts3termNextMethod(pCursor);

Added ext/fts3/fts3_test.c.

            1  +/*
            2  +** 2011 Jun 13
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file is not part of the production FTS code. It is only used for
           14  +** testing. It contains a Tcl command that can be used to test if a document
           15  +** matches an FTS NEAR expression.
           16  +*/
           17  +
           18  +#include <tcl.h>
           19  +#include <string.h>
           20  +#include <assert.h>
           21  +
           22  +/* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */
           23  +#include "fts3Int.h"
           24  +
           25  +#define NM_MAX_TOKEN 12
           26  +
           27  +typedef struct NearPhrase NearPhrase;
           28  +typedef struct NearDocument NearDocument;
           29  +typedef struct NearToken NearToken;
           30  +
           31  +struct NearDocument {
           32  +  int nToken;                     /* Length of token in bytes */
           33  +  NearToken *aToken;              /* Token array */
           34  +};
           35  +
           36  +struct NearToken {
           37  +  int n;                          /* Length of token in bytes */
           38  +  const char *z;                  /* Pointer to token string */
           39  +};
           40  +
           41  +struct NearPhrase {
           42  +  int nNear;                      /* Preceding NEAR value */
           43  +  int nToken;                     /* Number of tokens in this phrase */
           44  +  NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */
           45  +};
           46  +
           47  +static int nm_phrase_match(
           48  +  NearPhrase *p,
           49  +  NearToken *aToken
           50  +){
           51  +  int ii;
           52  +
           53  +  for(ii=0; ii<p->nToken; ii++){
           54  +    NearToken *pToken = &p->aToken[ii];
           55  +    if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){
           56  +      if( aToken[ii].n<(pToken->n-1) ) return 0;
           57  +      if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0;
           58  +    }else{
           59  +      if( aToken[ii].n!=pToken->n ) return 0;
           60  +      if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0;
           61  +    }
           62  +  }
           63  +
           64  +  return 1;
           65  +}
           66  +
           67  +static int nm_near_chain(
           68  +  int iDir,                       /* Direction to iterate through aPhrase[] */
           69  +  NearDocument *pDoc,             /* Document to match against */
           70  +  int iPos,                       /* Position at which iPhrase was found */
           71  +  int nPhrase,                    /* Size of phrase array */
           72  +  NearPhrase *aPhrase,            /* Phrase array */
           73  +  int iPhrase                     /* Index of phrase found */
           74  +){
           75  +  int iStart;
           76  +  int iStop;
           77  +  int ii;
           78  +  int nNear;
           79  +  int iPhrase2;
           80  +  NearPhrase *p;
           81  +  NearPhrase *pPrev;
           82  +
           83  +  assert( iDir==1 || iDir==-1 );
           84  +
           85  +  if( iDir==1 ){
           86  +    if( (iPhrase+1)==nPhrase ) return 1;
           87  +    nNear = aPhrase[iPhrase+1].nNear;
           88  +  }else{
           89  +    if( iPhrase==0 ) return 1;
           90  +    nNear = aPhrase[iPhrase].nNear;
           91  +  }
           92  +  pPrev = &aPhrase[iPhrase];
           93  +  iPhrase2 = iPhrase+iDir;
           94  +  p = &aPhrase[iPhrase2];
           95  +
           96  +  iStart = iPos - nNear - p->nToken;
           97  +  iStop = iPos + nNear + pPrev->nToken;
           98  +
           99  +  if( iStart<0 ) iStart = 0;
          100  +  if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken;
          101  +
          102  +  for(ii=iStart; ii<=iStop; ii++){
          103  +    if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
          104  +      if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1;
          105  +    }
          106  +  }
          107  +
          108  +  return 0;
          109  +}
          110  +
          111  +static int nm_match_count(
          112  +  NearDocument *pDoc,             /* Document to match against */
          113  +  int nPhrase,                    /* Size of phrase array */
          114  +  NearPhrase *aPhrase,            /* Phrase array */
          115  +  int iPhrase                     /* Index of phrase to count matches for */
          116  +){
          117  +  int nOcc = 0;
          118  +  int ii;
          119  +  NearPhrase *p = &aPhrase[iPhrase];
          120  +
          121  +  for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){
          122  +    if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
          123  +      /* Test forward NEAR chain (i>iPhrase) */
          124  +      if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
          125  +
          126  +      /* Test reverse NEAR chain (i<iPhrase) */
          127  +      if( 0==nm_near_chain(-1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
          128  +
          129  +      /* This is a real match. Increment the counter. */
          130  +      nOcc++;
          131  +    }
          132  +  } 
          133  +
          134  +  return nOcc;
          135  +}
          136  +
          137  +/*
          138  +** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
          139  +*/
          140  +static int fts3_near_match_cmd(
          141  +  ClientData clientData,
          142  +  Tcl_Interp *interp,
          143  +  int objc,
          144  +  Tcl_Obj *CONST objv[]
          145  +){
          146  +  int nTotal = 0;
          147  +  int rc;
          148  +  int ii;
          149  +  int nPhrase;
          150  +  NearPhrase *aPhrase = 0;
          151  +  NearDocument doc = {0, 0};
          152  +  Tcl_Obj **apDocToken;
          153  +  Tcl_Obj *pRet;
          154  +  Tcl_Obj *pPhrasecount = 0;
          155  +  
          156  +  Tcl_Obj **apExprToken;
          157  +  int nExprToken;
          158  +
          159  +  /* Must have 3 or more arguments. */
          160  +  if( objc<3 || (objc%2)==0 ){
          161  +    Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?...");
          162  +    rc = TCL_ERROR;
          163  +    goto near_match_out;
          164  +  }
          165  +
          166  +  for(ii=3; ii<objc; ii+=2){
          167  +    enum NM_enum { NM_PHRASECOUNTS };
          168  +    struct TestnmSubcmd {
          169  +      char *zName;
          170  +      enum NM_enum eOpt;
          171  +    } aOpt[] = {
          172  +      { "-phrasecountvar", NM_PHRASECOUNTS },
          173  +      { 0, 0 }
          174  +    };
          175  +    int iOpt;
          176  +    if( Tcl_GetIndexFromObjStruct(
          177  +        interp, objv[ii], aOpt, sizeof(aOpt[0]), "option", 0, &iOpt) 
          178  +    ){
          179  +      return TCL_ERROR;
          180  +    }
          181  +
          182  +    switch( aOpt[iOpt].eOpt ){
          183  +      case NM_PHRASECOUNTS:
          184  +        pPhrasecount = objv[ii+1];
          185  +        break;
          186  +    }
          187  +  }
          188  +
          189  +  rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
          190  +  if( rc!=TCL_OK ) goto near_match_out;
          191  +  doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
          192  +  for(ii=0; ii<doc.nToken; ii++){
          193  +    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
          194  +  }
          195  +
          196  +  rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
          197  +  if( rc!=TCL_OK ) goto near_match_out;
          198  +
          199  +  nPhrase = (nExprToken + 1) / 2;
          200  +  aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
          201  +  memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
          202  +  for(ii=0; ii<nPhrase; ii++){
          203  +    Tcl_Obj *pPhrase = apExprToken[ii*2];
          204  +    Tcl_Obj **apToken;
          205  +    int nToken;
          206  +    int jj;
          207  +
          208  +    rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
          209  +    if( rc!=TCL_OK ) goto near_match_out;
          210  +    if( nToken>NM_MAX_TOKEN ){
          211  +      Tcl_AppendResult(interp, "Too many tokens in phrase", 0);
          212  +      rc = TCL_ERROR;
          213  +      goto near_match_out;
          214  +    }
          215  +    for(jj=0; jj<nToken; jj++){
          216  +      NearToken *pT = &aPhrase[ii].aToken[jj];
          217  +      pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
          218  +    }
          219  +    aPhrase[ii].nToken = nToken;
          220  +  }
          221  +  for(ii=1; ii<nPhrase; ii++){
          222  +    Tcl_Obj *pNear = apExprToken[2*ii-1];
          223  +    int nNear;
          224  +    rc = Tcl_GetIntFromObj(interp, pNear, &nNear);
          225  +    if( rc!=TCL_OK ) goto near_match_out;
          226  +    aPhrase[ii].nNear = nNear;
          227  +  }
          228  +
          229  +  pRet = Tcl_NewObj();
          230  +  Tcl_IncrRefCount(pRet);
          231  +  for(ii=0; ii<nPhrase; ii++){
          232  +    int nOcc = nm_match_count(&doc, nPhrase, aPhrase, ii);
          233  +    Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nOcc));
          234  +    nTotal += nOcc;
          235  +  }
          236  +  if( pPhrasecount ){
          237  +    Tcl_ObjSetVar2(interp, pPhrasecount, 0, pRet, 0);
          238  +  }
          239  +  Tcl_DecrRefCount(pRet);
          240  +  Tcl_SetObjResult(interp, Tcl_NewBooleanObj(nTotal>0));
          241  +
          242  + near_match_out: 
          243  +  ckfree((char *)aPhrase);
          244  +  ckfree((char *)doc.aToken);
          245  +  return rc;
          246  +}
          247  +
          248  +/*
          249  +**   Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD?
          250  +**
          251  +** Normally, FTS uses hard-coded values to determine the minimum doclist
          252  +** size eligible for incremental loading, and the size of the chunks loaded
          253  +** when a doclist is incrementally loaded. This command allows the built-in
          254  +** values to be overridden for testing purposes.
          255  +**
          256  +** If present, the first argument is the chunksize in bytes to load doclists
          257  +** in. The second argument is the minimum doclist size in bytes to use
          258  +** incremental loading with.
          259  +**
          260  +** Whether or not the arguments are present, this command returns a list of
          261  +** two integers - the initial chunksize and threshold when the command is
          262  +** invoked. This can be used to restore the default behaviour after running
          263  +** tests. For example:
          264  +**
          265  +**    # Override incr-load settings for testing:
          266  +**    set cfg [fts3_configure_incr_load $new_chunksize $new_threshold]
          267  +**
          268  +**    .... run tests ....
          269  +**
          270  +**    # Restore initial incr-load settings:
          271  +**    eval fts3_configure_incr_load $cfg
          272  +*/
          273  +static int fts3_configure_incr_load_cmd(
          274  +  ClientData clientData,
          275  +  Tcl_Interp *interp,
          276  +  int objc,
          277  +  Tcl_Obj *CONST objv[]
          278  +){
          279  +#ifdef SQLITE_ENABLE_FTS3
          280  +  extern int test_fts3_node_chunksize;
          281  +  extern int test_fts3_node_chunk_threshold;
          282  +  int iArg1;
          283  +  int iArg2;
          284  +  Tcl_Obj *pRet;
          285  +
          286  +  if( objc!=1 && objc!=3 ){
          287  +    Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?");
          288  +    return TCL_ERROR;
          289  +  }
          290  +
          291  +  pRet = Tcl_NewObj();
          292  +  Tcl_IncrRefCount(pRet);
          293  +  Tcl_ListObjAppendElement(
          294  +      interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize));
          295  +  Tcl_ListObjAppendElement(
          296  +      interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold));
          297  +
          298  +  if( objc==3 ){
          299  +    int iArg1;
          300  +    int iArg2;
          301  +    if( Tcl_GetIntFromObj(interp, objv[1], &iArg1)
          302  +     || Tcl_GetIntFromObj(interp, objv[2], &iArg2)
          303  +    ){
          304  +      Tcl_DecrRefCount(pRet);
          305  +      return TCL_ERROR;
          306  +    }
          307  +    test_fts3_node_chunksize = iArg1;
          308  +    test_fts3_node_chunk_threshold = iArg2;
          309  +  }
          310  +
          311  +  Tcl_SetObjResult(interp, pRet);
          312  +  Tcl_DecrRefCount(pRet);
          313  +#endif
          314  +  return TCL_OK;
          315  +}
          316  +
          317  +int Sqlitetestfts3_Init(Tcl_Interp *interp){
          318  +  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
          319  +  Tcl_CreateObjCommand(interp, 
          320  +      "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
          321  +  );
          322  +  return TCL_OK;
          323  +}

Changes to ext/fts3/fts3_tokenizer.c.

    19     19   **
    20     20   **     * The FTS3 module is being built as an extension
    21     21   **       (in which case SQLITE_CORE is not defined), or
    22     22   **
    23     23   **     * The FTS3 module is being built into the core of
    24     24   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    25     25   */
    26         -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    27         -
    28     26   #include "sqlite3ext.h"
    29     27   #ifndef SQLITE_CORE
    30     28     SQLITE_EXTENSION_INIT1
    31     29   #endif
    32         -
    33     30   #include "fts3Int.h"
           31  +
           32  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
           33  +
    34     34   #include <assert.h>
    35     35   #include <string.h>
    36     36   
    37     37   /*
    38     38   ** Implementation of the SQL scalar function for accessing the underlying 
    39     39   ** hash table. This function may be called as follows:
    40     40   **

Changes to ext/fts3/fts3_tokenizer1.c.

    18     18   **
    19     19   **     * The FTS3 module is being built as an extension
    20     20   **       (in which case SQLITE_CORE is not defined), or
    21     21   **
    22     22   **     * The FTS3 module is being built into the core of
    23     23   **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
    24     24   */
    25         -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    26         -
    27     25   #include "fts3Int.h"
           26  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    28     27   
    29     28   #include <assert.h>
    30     29   #include <stdlib.h>
    31     30   #include <stdio.h>
    32     31   #include <string.h>
    33     32   
    34     33   #include "fts3_tokenizer.h"

Changes to ext/fts3/fts3_write.c.

    13     13   ** This file is part of the SQLite FTS3 extension module. Specifically,
    14     14   ** this file contains code to insert, update and delete rows from FTS3
    15     15   ** tables. It also contains code to merge FTS3 b-tree segments. Some
    16     16   ** of the sub-routines used to merge segments are also used by the query 
    17     17   ** code in fts3.c.
    18     18   */
    19     19   
           20  +#include "fts3Int.h"
    20     21   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    21     22   
    22         -#include "fts3Int.h"
    23     23   #include <string.h>
    24     24   #include <assert.h>
    25     25   #include <stdlib.h>
    26     26   
    27     27   /*
    28     28   ** When full-text index nodes are loaded from disk, the buffer that they
    29     29   ** are loaded into has the following number of bytes of padding at the end 
................................................................................
    32     32   **
    33     33   ** This means that if we have a pointer into a buffer containing node data,
    34     34   ** it is always safe to read up to two varints from it without risking an
    35     35   ** overread, even if the node data is corrupted.
    36     36   */
    37     37   #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
    38     38   
           39  +/*
           40  +** Under certain circumstances, b-tree nodes (doclists) can be loaded into
           41  +** memory incrementally instead of all at once. This can be a big performance
           42  +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
           43  +** method before retrieving all query results (as may happen, for example,
           44  +** if a query has a LIMIT clause).
           45  +**
           46  +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD 
           47  +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
           48  +** The code is written so that the hard lower-limit for each of these values 
           49  +** is 1. Clearly such small values would be inefficient, but can be useful 
           50  +** for testing purposes.
           51  +**
           52  +** If this module is built with SQLITE_TEST defined, these constants may
           53  +** be overridden at runtime for testing purposes. File fts3_test.c contains
           54  +** a Tcl interface to read and write the values.
           55  +*/
           56  +#ifdef SQLITE_TEST
           57  +int test_fts3_node_chunksize = (4*1024);
           58  +int test_fts3_node_chunk_threshold = (4*1024)*4;
           59  +# define FTS3_NODE_CHUNKSIZE       test_fts3_node_chunksize
           60  +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
           61  +#else
           62  +# define FTS3_NODE_CHUNKSIZE (4*1024) 
           63  +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
           64  +#endif
           65  +
    39     66   typedef struct PendingList PendingList;
    40     67   typedef struct SegmentNode SegmentNode;
    41     68   typedef struct SegmentWriter SegmentWriter;
    42     69   
    43     70   /*
    44         -** Data structure used while accumulating terms in the pending-terms hash
    45         -** table. The hash table entry maps from term (a string) to a malloc'd
    46         -** instance of this structure.
           71  +** An instance of the following data structure is used to build doclists
           72  +** incrementally. See function fts3PendingListAppend() for details.
    47     73   */
    48     74   struct PendingList {
    49     75     int nData;
    50     76     char *aData;
    51     77     int nSpace;
    52     78     sqlite3_int64 iLastDocid;
    53     79     sqlite3_int64 iLastCol;
................................................................................
    70     96   ** a contiguous set of segment b-tree leaf nodes. Although the details of
    71     97   ** this structure are only manipulated by code in this file, opaque handles
    72     98   ** of type Fts3SegReader* are also used by code in fts3.c to iterate through
    73     99   ** terms when querying the full-text index. See functions:
    74    100   **
    75    101   **   sqlite3Fts3SegReaderNew()
    76    102   **   sqlite3Fts3SegReaderFree()
    77         -**   sqlite3Fts3SegReaderCost()
    78    103   **   sqlite3Fts3SegReaderIterate()
    79    104   **
    80    105   ** Methods used to manipulate Fts3SegReader structures:
    81    106   **
    82    107   **   fts3SegReaderNext()
    83    108   **   fts3SegReaderFirstDocid()
    84    109   **   fts3SegReaderNextDocid()
................................................................................
    89    114     sqlite3_int64 iStartBlock;      /* Rowid of first leaf block to traverse */
    90    115     sqlite3_int64 iLeafEndBlock;    /* Rowid of final leaf block to traverse */
    91    116     sqlite3_int64 iEndBlock;        /* Rowid of final block in segment (or 0) */
    92    117     sqlite3_int64 iCurrentBlock;    /* Current leaf block (or 0) */
    93    118   
    94    119     char *aNode;                    /* Pointer to node data (or NULL) */
    95    120     int nNode;                      /* Size of buffer at aNode (or 0) */
          121  +  int nPopulate;                  /* If >0, bytes of buffer aNode[] loaded */
          122  +  sqlite3_blob *pBlob;            /* If not NULL, blob handle to read node */
          123  +
    96    124     Fts3HashElem **ppNextElem;
    97    125   
    98    126     /* Variables set by fts3SegReaderNext(). These may be read directly
    99    127     ** by the caller. They are valid from the time SegmentReaderNew() returns
   100    128     ** until SegmentReaderNext() returns something other than SQLITE_OK
   101    129     ** (i.e. SQLITE_DONE).
   102    130     */
   103    131     int nTerm;                      /* Number of bytes in current term */
   104    132     char *zTerm;                    /* Pointer to current term */
   105    133     int nTermAlloc;                 /* Allocated size of zTerm buffer */
   106    134     char *aDoclist;                 /* Pointer to doclist of current entry */
   107    135     int nDoclist;                   /* Size of doclist in current entry */
   108    136   
   109         -  /* The following variables are used to iterate through the current doclist */
          137  +  /* The following variables are used by fts3SegReaderNextDocid() to iterate 
          138  +  ** through the current doclist (aDoclist/nDoclist).
          139  +  */
   110    140     char *pOffsetList;
          141  +  int nOffsetList;                /* For descending pending seg-readers only */
   111    142     sqlite3_int64 iDocid;
   112    143   };
   113    144   
   114    145   #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
   115    146   #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
   116    147   
   117    148   /*
................................................................................
   141    172   ** the interior part of the segment b+-tree structures (everything except
   142    173   ** the leaf nodes). These functions and type are only ever used by code
   143    174   ** within the fts3SegWriterXXX() family of functions described above.
   144    175   **
   145    176   **   fts3NodeAddTerm()
   146    177   **   fts3NodeWrite()
   147    178   **   fts3NodeFree()
          179  +**
          180  +** When a b+tree is written to the database (either as a result of a merge
          181  +** or the pending-terms table being flushed), leaves are written into the 
          182  +** database file as soon as they are completely populated. The interior of
          183  +** the tree is assembled in memory and written out only once all leaves have
          184  +** been populated and stored. This is Ok, as the b+-tree fanout is usually
          185  +** very large, meaning that the interior of the tree consumes relatively 
          186  +** little memory.
   148    187   */
   149    188   struct SegmentNode {
   150    189     SegmentNode *pParent;           /* Parent node (or NULL for root node) */
   151    190     SegmentNode *pRight;            /* Pointer to right-sibling */
   152    191     SegmentNode *pLeftmost;         /* Pointer to left-most node of this depth */
   153    192     int nEntry;                     /* Number of terms written to node so far */
   154    193     char *zTerm;                    /* Pointer to previous term buffer */
................................................................................
   171    210   #define SQL_DELETE_ALL_STAT            6
   172    211   #define SQL_SELECT_CONTENT_BY_ROWID    7
   173    212   #define SQL_NEXT_SEGMENT_INDEX         8
   174    213   #define SQL_INSERT_SEGMENTS            9
   175    214   #define SQL_NEXT_SEGMENTS_ID          10
   176    215   #define SQL_INSERT_SEGDIR             11
   177    216   #define SQL_SELECT_LEVEL              12
   178         -#define SQL_SELECT_ALL_LEVEL          13
          217  +#define SQL_SELECT_LEVEL_RANGE        13
   179    218   #define SQL_SELECT_LEVEL_COUNT        14
   180         -#define SQL_SELECT_SEGDIR_COUNT_MAX   15
   181         -#define SQL_DELETE_SEGDIR_BY_LEVEL    16
          219  +#define SQL_SELECT_SEGDIR_MAX_LEVEL   15
          220  +#define SQL_DELETE_SEGDIR_LEVEL       16
   182    221   #define SQL_DELETE_SEGMENTS_RANGE     17
   183    222   #define SQL_CONTENT_INSERT            18
   184    223   #define SQL_DELETE_DOCSIZE            19
   185    224   #define SQL_REPLACE_DOCSIZE           20
   186    225   #define SQL_SELECT_DOCSIZE            21
   187    226   #define SQL_SELECT_DOCTOTAL           22
   188    227   #define SQL_REPLACE_DOCTOTAL          23
          228  +
          229  +#define SQL_SELECT_ALL_PREFIX_LEVEL   24
          230  +#define SQL_DELETE_ALL_TERMS_SEGDIR   25
          231  +
          232  +#define SQL_DELETE_SEGDIR_RANGE       26
   189    233   
   190    234   /*
   191    235   ** This function is used to obtain an SQLite prepared statement handle
   192    236   ** for the statement identified by the second argument. If successful,
   193    237   ** *pp is set to the requested statement handle and SQLITE_OK returned.
   194    238   ** Otherwise, an SQLite error code is returned and *pp is set to 0.
   195    239   **
................................................................................
   218    262   /* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
   219    263   /* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
   220    264   
   221    265             /* Return segments in order from oldest to newest.*/ 
   222    266   /* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   223    267               "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
   224    268   /* 13 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   225         -            "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
          269  +            "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
          270  +            "ORDER BY level DESC, idx ASC",
   226    271   
   227    272   /* 14 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
   228         -/* 15 */  "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
          273  +/* 15 */  "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
   229    274   
   230    275   /* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
   231    276   /* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
   232    277   /* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
   233    278   /* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
   234    279   /* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
   235    280   /* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
   236    281   /* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
   237    282   /* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
          283  +/* 24 */  "",
          284  +/* 25 */  "",
          285  +
          286  +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
          287  +
   238    288     };
   239    289     int rc = SQLITE_OK;
   240    290     sqlite3_stmt *pStmt;
   241    291   
   242    292     assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
   243    293     assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
   244    294     
................................................................................
   386    436   **
   387    437   **   0: idx
   388    438   **   1: start_block
   389    439   **   2: leaves_end_block
   390    440   **   3: end_block
   391    441   **   4: root
   392    442   */
   393         -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){
          443  +int sqlite3Fts3AllSegdirs(
          444  +  Fts3Table *p,                   /* FTS3 table */
          445  +  int iIndex,                     /* Index for p->aIndex[] */
          446  +  int iLevel,                     /* Level to select */
          447  +  sqlite3_stmt **ppStmt           /* OUT: Compiled statement */
          448  +){
   394    449     int rc;
   395    450     sqlite3_stmt *pStmt = 0;
          451  +
          452  +  assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
          453  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
          454  +  assert( iIndex>=0 && iIndex<p->nIndex );
          455  +
   396    456     if( iLevel<0 ){
   397         -    rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0);
          457  +    /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
          458  +    rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
          459  +    if( rc==SQLITE_OK ){ 
          460  +      sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
          461  +      sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
          462  +    }
   398    463     }else{
          464  +    /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
   399    465       rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
   400         -    if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel);
          466  +    if( rc==SQLITE_OK ){ 
          467  +      sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
          468  +    }
   401    469     }
   402    470     *ppStmt = pStmt;
   403    471     return rc;
   404    472   }
   405    473   
   406    474   
   407    475   /*
................................................................................
   507    575     *pRc = rc;
   508    576     if( p!=*pp ){
   509    577       *pp = p;
   510    578       return 1;
   511    579     }
   512    580     return 0;
   513    581   }
          582  +
          583  +/*
          584  +** Free a PendingList object allocated by fts3PendingListAppend().
          585  +*/
          586  +static void fts3PendingListDelete(PendingList *pList){
          587  +  sqlite3_free(pList);
          588  +}
          589  +
          590  +/*
          591  +** Add an entry to one of the pending-terms hash tables.
          592  +*/
          593  +static int fts3PendingTermsAddOne(
          594  +  Fts3Table *p,
          595  +  int iCol,
          596  +  int iPos,
          597  +  Fts3Hash *pHash,                /* Pending terms hash table to add entry to */
          598  +  const char *zToken,
          599  +  int nToken
          600  +){
          601  +  PendingList *pList;
          602  +  int rc = SQLITE_OK;
          603  +
          604  +  pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
          605  +  if( pList ){
          606  +    p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
          607  +  }
          608  +  if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
          609  +    if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){
          610  +      /* Malloc failed while inserting the new entry. This can only 
          611  +      ** happen if there was no previous entry for this token.
          612  +      */
          613  +      assert( 0==fts3HashFind(pHash, zToken, nToken) );
          614  +      sqlite3_free(pList);
          615  +      rc = SQLITE_NOMEM;
          616  +    }
          617  +  }
          618  +  if( rc==SQLITE_OK ){
          619  +    p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
          620  +  }
          621  +  return rc;
          622  +}
   514    623   
   515    624   /*
   516    625   ** Tokenize the nul-terminated string zText and add all tokens to the
   517    626   ** pending-terms hash-table. The docid used is that currently stored in
   518    627   ** p->iPrevDocid, and the column is specified by argument iCol.
   519    628   **
   520    629   ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
................................................................................
   556    665     }
   557    666     pCsr->pTokenizer = pTokenizer;
   558    667   
   559    668     xNext = pModule->xNext;
   560    669     while( SQLITE_OK==rc
   561    670         && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
   562    671     ){
   563         -    PendingList *pList;
   564         - 
          672  +    int i;
   565    673       if( iPos>=nWord ) nWord = iPos+1;
   566    674   
   567    675       /* Positions cannot be negative; we use -1 as a terminator internally.
   568    676       ** Tokens must have a non-zero length.
   569    677       */
   570    678       if( iPos<0 || !zToken || nToken<=0 ){
   571    679         rc = SQLITE_ERROR;
   572    680         break;
   573    681       }
   574    682   
   575         -    pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
   576         -    if( pList ){
   577         -      p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
   578         -    }
   579         -    if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
   580         -      if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
   581         -        /* Malloc failed while inserting the new entry. This can only 
   582         -        ** happen if there was no previous entry for this token.
   583         -        */
   584         -        assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
   585         -        sqlite3_free(pList);
   586         -        rc = SQLITE_NOMEM;
   587         -      }
   588         -    }
   589         -    if( rc==SQLITE_OK ){
   590         -      p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
          683  +    /* Add the term to the terms index */
          684  +    rc = fts3PendingTermsAddOne(
          685  +        p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
          686  +    );
          687  +    
          688  +    /* Add the term to each of the prefix indexes that it is not too 
          689  +    ** short for. */
          690  +    for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
          691  +      struct Fts3Index *pIndex = &p->aIndex[i];
          692  +      if( nToken<pIndex->nPrefix ) continue;
          693  +      rc = fts3PendingTermsAddOne(
          694  +          p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
          695  +      );
   591    696       }
   592    697     }
   593    698   
   594    699     pModule->xClose(pCsr);
   595    700     *pnWord = nWord;
   596    701     return (rc==SQLITE_DONE ? SQLITE_OK : rc);
   597    702   }
................................................................................
   613    718       if( rc!=SQLITE_OK ) return rc;
   614    719     }
   615    720     p->iPrevDocid = iDocid;
   616    721     return SQLITE_OK;
   617    722   }
   618    723   
   619    724   /*
   620         -** Discard the contents of the pending-terms hash table. 
          725  +** Discard the contents of the pending-terms hash tables. 
   621    726   */
   622    727   void sqlite3Fts3PendingTermsClear(Fts3Table *p){
   623         -  Fts3HashElem *pElem;
   624         -  for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
   625         -    sqlite3_free(fts3HashData(pElem));
          728  +  int i;
          729  +  for(i=0; i<p->nIndex; i++){
          730  +    Fts3HashElem *pElem;
          731  +    Fts3Hash *pHash = &p->aIndex[i].hPending;
          732  +    for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
          733  +      PendingList *pList = (PendingList *)fts3HashData(pElem);
          734  +      fts3PendingListDelete(pList);
          735  +    }
          736  +    fts3HashClear(pHash);
   626    737     }
   627         -  fts3HashClear(&p->pendingTerms);
   628    738     p->nPendingData = 0;
   629    739   }
   630    740   
   631    741   /*
   632    742   ** This function is called by the xUpdate() method as part of an INSERT
   633    743   ** operation. It adds entries for each term in the new record to the
   634    744   ** pendingTerms hash table.
................................................................................
   776    886     *pRC = rc;
   777    887   }
   778    888   
   779    889   /*
   780    890   ** Forward declaration to account for the circular dependency between
   781    891   ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
   782    892   */
   783         -static int fts3SegmentMerge(Fts3Table *, int);
          893  +static int fts3SegmentMerge(Fts3Table *, int, int);
   784    894   
   785    895   /* 
   786    896   ** This function allocates a new level iLevel index in the segdir table.
   787    897   ** Usually, indexes are allocated within a level sequentially starting
   788    898   ** with 0, so the allocated index is one greater than the value returned
   789    899   ** by:
   790    900   **
................................................................................
   793    903   ** However, if there are already FTS3_MERGE_COUNT indexes at the requested
   794    904   ** level, they are merged into a single level (iLevel+1) segment and the 
   795    905   ** allocated index is 0.
   796    906   **
   797    907   ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
   798    908   ** returned. Otherwise, an SQLite error code is returned.
   799    909   */
   800         -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
          910  +static int fts3AllocateSegdirIdx(
          911  +  Fts3Table *p, 
          912  +  int iIndex,                     /* Index for p->aIndex */
          913  +  int iLevel, 
          914  +  int *piIdx
          915  +){
   801    916     int rc;                         /* Return Code */
   802    917     sqlite3_stmt *pNextIdx;         /* Query for next idx at level iLevel */
   803    918     int iNext = 0;                  /* Result of query pNextIdx */
   804    919   
   805    920     /* Set variable iNext to the next available segdir index at level iLevel. */
   806    921     rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
   807    922     if( rc==SQLITE_OK ){
   808         -    sqlite3_bind_int(pNextIdx, 1, iLevel);
          923  +    sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
   809    924       if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
   810    925         iNext = sqlite3_column_int(pNextIdx, 0);
   811    926       }
   812    927       rc = sqlite3_reset(pNextIdx);
   813    928     }
   814    929   
   815    930     if( rc==SQLITE_OK ){
   816    931       /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
   817    932       ** full, merge all segments in level iLevel into a single iLevel+1
   818    933       ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
   819    934       ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
   820    935       */
   821    936       if( iNext>=FTS3_MERGE_COUNT ){
   822         -      rc = fts3SegmentMerge(p, iLevel);
          937  +      rc = fts3SegmentMerge(p, iIndex, iLevel);
   823    938         *piIdx = 0;
   824    939       }else{
   825    940         *piIdx = iNext;
   826    941       }
   827    942     }
   828    943   
   829    944     return rc;
................................................................................
   856    971   ** method (xFilter etc.) that may directly or indirectly call this function
   857    972   ** must call sqlite3Fts3SegmentsClose() before returning.
   858    973   */
   859    974   int sqlite3Fts3ReadBlock(
   860    975     Fts3Table *p,                   /* FTS3 table handle */
   861    976     sqlite3_int64 iBlockid,         /* Access the row with blockid=$iBlockid */
   862    977     char **paBlob,                  /* OUT: Blob data in malloc'd buffer */
   863         -  int *pnBlob                     /* OUT: Size of blob data */
          978  +  int *pnBlob,                    /* OUT: Size of blob data */
          979  +  int *pnLoad                     /* OUT: Bytes actually loaded */
   864    980   ){
   865    981     int rc;                         /* Return code */
   866    982   
   867    983     /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
   868    984     assert( pnBlob);
   869    985   
   870    986     if( p->pSegments ){
................................................................................
   877    993       rc = sqlite3_blob_open(
   878    994          p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
   879    995       );
   880    996     }
   881    997   
   882    998     if( rc==SQLITE_OK ){
   883    999       int nByte = sqlite3_blob_bytes(p->pSegments);
         1000  +    *pnBlob = nByte;
   884   1001       if( paBlob ){
   885   1002         char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
   886   1003         if( !aByte ){
   887   1004           rc = SQLITE_NOMEM;
   888   1005         }else{
         1006  +        if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
         1007  +          nByte = FTS3_NODE_CHUNKSIZE;
         1008  +          *pnLoad = nByte;
         1009  +        }
   889   1010           rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
   890   1011           memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
   891   1012           if( rc!=SQLITE_OK ){
   892   1013             sqlite3_free(aByte);
   893   1014             aByte = 0;
   894   1015           }
   895   1016         }
   896   1017         *paBlob = aByte;
   897   1018       }
   898         -    *pnBlob = nByte;
   899   1019     }
   900   1020   
   901   1021     return rc;
   902   1022   }
   903   1023   
   904   1024   /*
   905   1025   ** Close the blob handle at p->pSegments, if it is open. See comments above
   906   1026   ** the sqlite3Fts3ReadBlock() function for details.
   907   1027   */
   908   1028   void sqlite3Fts3SegmentsClose(Fts3Table *p){
   909   1029     sqlite3_blob_close(p->pSegments);
   910   1030     p->pSegments = 0;
   911   1031   }
         1032  +    
         1033  +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
         1034  +  int nRead;                      /* Number of bytes to read */
         1035  +  int rc;                         /* Return code */
         1036  +
         1037  +  nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
         1038  +  rc = sqlite3_blob_read(
         1039  +      pReader->pBlob, 
         1040  +      &pReader->aNode[pReader->nPopulate],
         1041  +      nRead,
         1042  +      pReader->nPopulate
         1043  +  );
         1044  +
         1045  +  if( rc==SQLITE_OK ){
         1046  +    pReader->nPopulate += nRead;
         1047  +    memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
         1048  +    if( pReader->nPopulate==pReader->nNode ){
         1049  +      sqlite3_blob_close(pReader->pBlob);
         1050  +      pReader->pBlob = 0;
         1051  +      pReader->nPopulate = 0;
         1052  +    }
         1053  +  }
         1054  +  return rc;
         1055  +}
         1056  +
         1057  +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
         1058  +  int rc = SQLITE_OK;
         1059  +  assert( !pReader->pBlob 
         1060  +       || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
         1061  +  );
         1062  +  while( pReader->pBlob && rc==SQLITE_OK 
         1063  +     &&  (pFrom - pReader->aNode + nByte)>pReader->nPopulate
         1064  +  ){
         1065  +    rc = fts3SegReaderIncrRead(pReader);
         1066  +  }
         1067  +  return rc;
         1068  +}
   912   1069   
   913   1070   /*
   914   1071   ** Move the iterator passed as the first argument to the next term in the
   915   1072   ** segment. If successful, SQLITE_OK is returned. If there is no next term,
   916   1073   ** SQLITE_DONE. Otherwise, an SQLite error code.
   917   1074   */
   918         -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
         1075  +static int fts3SegReaderNext(
         1076  +  Fts3Table *p, 
         1077  +  Fts3SegReader *pReader,
         1078  +  int bIncr
         1079  +){
         1080  +  int rc;                         /* Return code of various sub-routines */
   919   1081     char *pNext;                    /* Cursor variable */
   920   1082     int nPrefix;                    /* Number of bytes in term prefix */
   921   1083     int nSuffix;                    /* Number of bytes in term suffix */
   922   1084   
   923   1085     if( !pReader->aDoclist ){
   924   1086       pNext = pReader->aNode;
   925   1087     }else{
   926   1088       pNext = &pReader->aDoclist[pReader->nDoclist];
   927   1089     }
   928   1090   
   929   1091     if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
   930         -    int rc;                       /* Return code from Fts3ReadBlock() */
   931   1092   
   932   1093       if( fts3SegReaderIsPending(pReader) ){
   933   1094         Fts3HashElem *pElem = *(pReader->ppNextElem);
   934   1095         if( pElem==0 ){
   935   1096           pReader->aNode = 0;
   936   1097         }else{
   937   1098           PendingList *pList = (PendingList *)fts3HashData(pElem);
................................................................................
   943   1104           assert( pReader->aNode );
   944   1105         }
   945   1106         return SQLITE_OK;
   946   1107       }
   947   1108   
   948   1109       if( !fts3SegReaderIsRootOnly(pReader) ){
   949   1110         sqlite3_free(pReader->aNode);
         1111  +      sqlite3_blob_close(pReader->pBlob);
         1112  +      pReader->pBlob = 0;
   950   1113       }
   951   1114       pReader->aNode = 0;
   952   1115   
   953   1116       /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 
   954   1117       ** blocks have already been traversed.  */
   955   1118       assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
   956   1119       if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
   957   1120         return SQLITE_OK;
   958   1121       }
   959   1122   
   960   1123       rc = sqlite3Fts3ReadBlock(
   961         -        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
         1124  +        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, 
         1125  +        (bIncr ? &pReader->nPopulate : 0)
   962   1126       );
   963   1127       if( rc!=SQLITE_OK ) return rc;
         1128  +    assert( pReader->pBlob==0 );
         1129  +    if( bIncr && pReader->nPopulate<pReader->nNode ){
         1130  +      pReader->pBlob = p->pSegments;
         1131  +      p->pSegments = 0;
         1132  +    }
   964   1133       pNext = pReader->aNode;
   965   1134     }
         1135  +
         1136  +  assert( !fts3SegReaderIsPending(pReader) );
         1137  +
         1138  +  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
         1139  +  if( rc!=SQLITE_OK ) return rc;
   966   1140     
   967   1141     /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
   968         -  ** safe (no risk of overread) even if the node data is corrupted.  
   969         -  */
         1142  +  ** safe (no risk of overread) even if the node data is corrupted. */
   970   1143     pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
   971   1144     pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
   972   1145     if( nPrefix<0 || nSuffix<=0 
   973   1146      || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
   974   1147     ){
   975   1148       return SQLITE_CORRUPT_VTAB;
   976   1149     }
................................................................................
   980   1153       char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
   981   1154       if( !zNew ){
   982   1155         return SQLITE_NOMEM;
   983   1156       }
   984   1157       pReader->zTerm = zNew;
   985   1158       pReader->nTermAlloc = nNew;
   986   1159     }
         1160  +
         1161  +  rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
         1162  +  if( rc!=SQLITE_OK ) return rc;
         1163  +
   987   1164     memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
   988   1165     pReader->nTerm = nPrefix+nSuffix;
   989   1166     pNext += nSuffix;
   990   1167     pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
   991   1168     pReader->aDoclist = pNext;
   992   1169     pReader->pOffsetList = 0;
   993   1170   
   994   1171     /* Check that the doclist does not appear to extend past the end of the
   995   1172     ** b-tree node. And that the final byte of the doclist is 0x00. If either 
   996   1173     ** of these statements is untrue, then the data structure is corrupt.
   997   1174     */
   998   1175     if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
   999         -   || pReader->aDoclist[pReader->nDoclist-1]
         1176  +   || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  1000   1177     ){
  1001   1178       return SQLITE_CORRUPT_VTAB;
  1002   1179     }
  1003   1180     return SQLITE_OK;
  1004   1181   }
  1005   1182   
  1006   1183   /*
  1007   1184   ** Set the SegReader to point to the first docid in the doclist associated
  1008   1185   ** with the current term.
  1009   1186   */
  1010         -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
  1011         -  int n;
         1187  +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
         1188  +  int rc = SQLITE_OK;
  1012   1189     assert( pReader->aDoclist );
  1013   1190     assert( !pReader->pOffsetList );
  1014         -  n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
  1015         -  pReader->pOffsetList = &pReader->aDoclist[n];
         1191  +  if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
         1192  +    u8 bEof = 0;
         1193  +    pReader->iDocid = 0;
         1194  +    pReader->nOffsetList = 0;
         1195  +    sqlite3Fts3DoclistPrev(0,
         1196  +        pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, 
         1197  +        &pReader->iDocid, &pReader->nOffsetList, &bEof
         1198  +    );
         1199  +  }else{
         1200  +    rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
         1201  +    if( rc==SQLITE_OK ){
         1202  +      int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
         1203  +      pReader->pOffsetList = &pReader->aDoclist[n];
         1204  +    }
         1205  +  }
         1206  +  return rc;
  1016   1207   }
  1017   1208   
  1018   1209   /*
  1019   1210   ** Advance the SegReader to point to the next docid in the doclist
  1020   1211   ** associated with the current term.
  1021   1212   ** 
  1022   1213   ** If arguments ppOffsetList and pnOffsetList are not NULL, then 
  1023   1214   ** *ppOffsetList is set to point to the first column-offset list
  1024   1215   ** in the doclist entry (i.e. immediately past the docid varint).
  1025   1216   ** *pnOffsetList is set to the length of the set of column-offset
  1026   1217   ** lists, not including the nul-terminator byte. For example:
  1027   1218   */
  1028         -static void fts3SegReaderNextDocid(
  1029         -  Fts3SegReader *pReader,
  1030         -  char **ppOffsetList,
  1031         -  int *pnOffsetList
         1219  +static int fts3SegReaderNextDocid(
         1220  +  Fts3Table *pTab,
         1221  +  Fts3SegReader *pReader,         /* Reader to advance to next docid */
         1222  +  char **ppOffsetList,            /* OUT: Pointer to current position-list */
         1223  +  int *pnOffsetList               /* OUT: Length of *ppOffsetList in bytes */
  1032   1224   ){
         1225  +  int rc = SQLITE_OK;
  1033   1226     char *p = pReader->pOffsetList;
  1034   1227     char c = 0;
  1035   1228   
  1036         -  /* Pointer p currently points at the first byte of an offset list. The
  1037         -  ** following two lines advance it to point one byte past the end of
  1038         -  ** the same offset list.
  1039         -  */
  1040         -  while( *p | c ) c = *p++ & 0x80;
  1041         -  p++;
  1042         -
  1043         -  /* If required, populate the output variables with a pointer to and the
  1044         -  ** size of the previous offset-list.
  1045         -  */
  1046         -  if( ppOffsetList ){
  1047         -    *ppOffsetList = pReader->pOffsetList;
  1048         -    *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
  1049         -  }
  1050         -
  1051         -  /* If there are no more entries in the doclist, set pOffsetList to
  1052         -  ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
  1053         -  ** Fts3SegReader.pOffsetList to point to the next offset list before
  1054         -  ** returning.
  1055         -  */
  1056         -  if( p>=&pReader->aDoclist[pReader->nDoclist] ){
  1057         -    pReader->pOffsetList = 0;
  1058         -  }else{
  1059         -    sqlite3_int64 iDelta;
  1060         -    pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
  1061         -    pReader->iDocid += iDelta;
  1062         -  }
  1063         -}
  1064         -
  1065         -/*
  1066         -** This function is called to estimate the amount of data that will be 
  1067         -** loaded from the disk If SegReaderIterate() is called on this seg-reader,
  1068         -** in units of average document size.
  1069         -** 
  1070         -** This can be used as follows: If the caller has a small doclist that 
  1071         -** contains references to N documents, and is considering merging it with
  1072         -** a large doclist (size X "average documents"), it may opt not to load
  1073         -** the large doclist if X>N.
  1074         -*/
  1075         -int sqlite3Fts3SegReaderCost(
  1076         -  Fts3Cursor *pCsr,               /* FTS3 cursor handle */
  1077         -  Fts3SegReader *pReader,         /* Segment-reader handle */
  1078         -  int *pnCost                     /* IN/OUT: Number of bytes read */
         1229  +  assert( p );
         1230  +
         1231  +  if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
         1232  +    /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
         1233  +    ** Pending-terms doclists are always built up in ascending order, so
         1234  +    ** we have to iterate through them backwards here. */
         1235  +    u8 bEof = 0;
         1236  +    if( ppOffsetList ){
         1237  +      *ppOffsetList = pReader->pOffsetList;
         1238  +      *pnOffsetList = pReader->nOffsetList - 1;
         1239  +    }
         1240  +    sqlite3Fts3DoclistPrev(0,
         1241  +        pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
         1242  +        &pReader->nOffsetList, &bEof
         1243  +    );
         1244  +    if( bEof ){
         1245  +      pReader->pOffsetList = 0;
         1246  +    }else{
         1247  +      pReader->pOffsetList = p;
         1248  +    }
         1249  +  }else{
         1250  +    char *pEnd = &pReader->aDoclist[pReader->nDoclist];
         1251  +
         1252  +    /* Pointer p currently points at the first byte of an offset list. The
         1253  +    ** following block advances it to point one byte past the end of
         1254  +    ** the same offset list. */
         1255  +    while( 1 ){
         1256  +  
         1257  +      /* The following line of code (and the "p++" below the while() loop) is
         1258  +      ** normally all that is required to move pointer p to the desired 
         1259  +      ** position. The exception is if this node is being loaded from disk
         1260  +      ** incrementally and pointer "p" now points to the first byte passed
         1261  +      ** the populated part of pReader->aNode[].
         1262  +      */
         1263  +      while( *p | c ) c = *p++ & 0x80;
         1264  +      assert( *p==0 );
         1265  +  
         1266  +      if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
         1267  +      rc = fts3SegReaderIncrRead(pReader);
         1268  +      if( rc!=SQLITE_OK ) return rc;
         1269  +    }
         1270  +    p++;
         1271  +  
         1272  +    /* If required, populate the output variables with a pointer to and the
         1273  +    ** size of the previous offset-list.
         1274  +    */
         1275  +    if( ppOffsetList ){
         1276  +      *ppOffsetList = pReader->pOffsetList;
         1277  +      *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
         1278  +    }
         1279  +
         1280  +    while( p<pEnd && *p==0 ) p++;
         1281  +  
         1282  +    /* If there are no more entries in the doclist, set pOffsetList to
         1283  +    ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
         1284  +    ** Fts3SegReader.pOffsetList to point to the next offset list before
         1285  +    ** returning.
         1286  +    */
         1287  +    if( p>=pEnd ){
         1288  +      pReader->pOffsetList = 0;
         1289  +    }else{
         1290  +      rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
         1291  +      if( rc==SQLITE_OK ){
         1292  +        sqlite3_int64 iDelta;
         1293  +        pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
         1294  +        if( pTab->bDescIdx ){
         1295  +          pReader->iDocid -= iDelta;
         1296  +        }else{
         1297  +          pReader->iDocid += iDelta;
         1298  +        }
         1299  +      }
         1300  +    }
         1301  +  }
         1302  +
         1303  +  return SQLITE_OK;
         1304  +}
         1305  +
         1306  +
         1307  +int sqlite3Fts3MsrOvfl(
         1308  +  Fts3Cursor *pCsr, 
         1309  +  Fts3MultiSegReader *pMsr,
         1310  +  int *pnOvfl
  1079   1311   ){
  1080   1312     Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
  1081         -  int rc = SQLITE_OK;             /* Return code */
  1082         -  int nCost = 0;                  /* Cost in bytes to return */
  1083         -  int pgsz = p->nPgsz;            /* Database page size */
  1084         -
  1085         -  /* If this seg-reader is reading the pending-terms table, or if all data
  1086         -  ** for the segment is stored on the root page of the b-tree, then the cost
  1087         -  ** is zero. In this case all required data is already in main memory.
  1088         -  */
  1089         -  if( p->bHasStat 
  1090         -   && !fts3SegReaderIsPending(pReader) 
  1091         -   && !fts3SegReaderIsRootOnly(pReader) 
  1092         -  ){
  1093         -    int nBlob = 0;
  1094         -    sqlite3_int64 iBlock;
  1095         -
  1096         -    if( pCsr->nRowAvg==0 ){
  1097         -      /* The average document size, which is required to calculate the cost
  1098         -      ** of each doclist, has not yet been determined. Read the required 
  1099         -      ** data from the %_stat table to calculate it.
  1100         -      **
  1101         -      ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
  1102         -      ** varints, where nCol is the number of columns in the FTS3 table.
  1103         -      ** The first varint is the number of documents currently stored in
  1104         -      ** the table. The following nCol varints contain the total amount of
  1105         -      ** data stored in all rows of each column of the table, from left
  1106         -      ** to right.
  1107         -      */
  1108         -      sqlite3_stmt *pStmt;
  1109         -      sqlite3_int64 nDoc = 0;
  1110         -      sqlite3_int64 nByte = 0;
  1111         -      const char *pEnd;
  1112         -      const char *a;
  1113         -
  1114         -      rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
  1115         -      if( rc!=SQLITE_OK ) return rc;
  1116         -      a = sqlite3_column_blob(pStmt, 0);
  1117         -      assert( a );
  1118         -
  1119         -      pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
  1120         -      a += sqlite3Fts3GetVarint(a, &nDoc);
  1121         -      while( a<pEnd ){
  1122         -        a += sqlite3Fts3GetVarint(a, &nByte);
  1123         -      }
  1124         -      if( nDoc==0 || nByte==0 ){
  1125         -        sqlite3_reset(pStmt);
  1126         -        return SQLITE_CORRUPT_VTAB;
  1127         -      }
  1128         -
  1129         -      pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
  1130         -      assert( pCsr->nRowAvg>0 ); 
  1131         -      rc = sqlite3_reset(pStmt);
  1132         -      if( rc!=SQLITE_OK ) return rc;
  1133         -    }
  1134         -
  1135         -    /* Assume that a blob flows over onto overflow pages if it is larger
  1136         -    ** than (pgsz-35) bytes in size (the file-format documentation
  1137         -    ** confirms this).
  1138         -    */
  1139         -    for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
  1140         -      rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
  1141         -      if( rc!=SQLITE_OK ) break;
  1142         -      if( (nBlob+35)>pgsz ){
  1143         -        int nOvfl = (nBlob + 34)/pgsz;
  1144         -        nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
  1145         -      }
  1146         -    }
  1147         -  }
  1148         -
  1149         -  *pnCost += nCost;
         1313  +  int nOvfl = 0;
         1314  +  int ii;
         1315  +  int rc = SQLITE_OK;
         1316  +  int pgsz = p->nPgsz;
         1317  +
         1318  +  assert( p->bHasStat );
         1319  +  assert( pgsz>0 );
         1320  +
         1321  +  for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
         1322  +    Fts3SegReader *pReader = pMsr->apSegment[ii];
         1323  +    if( !fts3SegReaderIsPending(pReader) 
         1324  +     && !fts3SegReaderIsRootOnly(pReader) 
         1325  +    ){
         1326  +      int jj;
         1327  +      for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
         1328  +        int nBlob;
         1329  +        rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
         1330  +        if( rc!=SQLITE_OK ) break;
         1331  +        if( (nBlob+35)>pgsz ){
         1332  +          nOvfl += (nBlob + 34)/pgsz;
         1333  +        }
         1334  +      }
         1335  +    }
         1336  +  }
         1337  +  *pnOvfl = nOvfl;
  1150   1338     return rc;
  1151   1339   }
  1152   1340   
  1153   1341   /*
  1154   1342   ** Free all allocations associated with the iterator passed as the 
  1155   1343   ** second argument.
  1156   1344   */
  1157   1345   void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
  1158   1346     if( pReader && !fts3SegReaderIsPending(pReader) ){
  1159   1347       sqlite3_free(pReader->zTerm);
  1160   1348       if( !fts3SegReaderIsRootOnly(pReader) ){
  1161   1349         sqlite3_free(pReader->aNode);
         1350  +      sqlite3_blob_close(pReader->pBlob);
  1162   1351       }
  1163   1352     }
  1164   1353     sqlite3_free(pReader);
  1165   1354   }
  1166   1355   
  1167   1356   /*
  1168   1357   ** Allocate a new SegReader object.
................................................................................
  1231   1420     }
  1232   1421     return c;
  1233   1422   }
  1234   1423   
  1235   1424   /*
  1236   1425   ** This function is used to allocate an Fts3SegReader that iterates through
  1237   1426   ** a subset of the terms stored in the Fts3Table.pendingTerms array.
         1427  +**
         1428  +** If the isPrefixIter parameter is zero, then the returned SegReader iterates
         1429  +** through each term in the pending-terms table. Or, if isPrefixIter is
         1430  +** non-zero, it iterates through each term and its prefixes. For example, if
         1431  +** the pending terms hash table contains the terms "sqlite", "mysql" and
         1432  +** "firebird", then the iterator visits the following 'terms' (in the order
         1433  +** shown):
         1434  +**
         1435  +**   f fi fir fire fireb firebi firebir firebird
         1436  +**   m my mys mysq mysql
         1437  +**   s sq sql sqli sqlit sqlite
         1438  +**
         1439  +** Whereas if isPrefixIter is zero, the terms visited are:
         1440  +**
         1441  +**   firebird mysql sqlite
  1238   1442   */
  1239   1443   int sqlite3Fts3SegReaderPending(
  1240   1444     Fts3Table *p,                   /* Virtual table handle */
         1445  +  int iIndex,                     /* Index for p->aIndex */
  1241   1446     const char *zTerm,              /* Term to search for */
  1242   1447     int nTerm,                      /* Size of buffer zTerm */
  1243         -  int isPrefix,                   /* True for a term-prefix query */
         1448  +  int bPrefix,                    /* True for a prefix iterator */
  1244   1449     Fts3SegReader **ppReader        /* OUT: SegReader for pending-terms */
  1245   1450   ){
  1246   1451     Fts3SegReader *pReader = 0;     /* Fts3SegReader object to return */
  1247   1452     Fts3HashElem **aElem = 0;       /* Array of term hash entries to scan */
  1248   1453     int nElem = 0;                  /* Size of array at aElem */
  1249   1454     int rc = SQLITE_OK;             /* Return Code */
         1455  +  Fts3Hash *pHash;
  1250   1456   
  1251         -  if( isPrefix ){
         1457  +  pHash = &p->aIndex[iIndex].hPending;
         1458  +  if( bPrefix ){
  1252   1459       int nAlloc = 0;               /* Size of allocated array at aElem */
  1253   1460       Fts3HashElem *pE = 0;         /* Iterator variable */
  1254   1461   
  1255         -    for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
         1462  +    for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
  1256   1463         char *zKey = (char *)fts3HashKey(pE);
  1257   1464         int nKey = fts3HashKeysize(pE);
  1258   1465         if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
  1259   1466           if( nElem==nAlloc ){
  1260   1467             Fts3HashElem **aElem2;
  1261   1468             nAlloc += 16;
  1262   1469             aElem2 = (Fts3HashElem **)sqlite3_realloc(
................................................................................
  1265   1472             if( !aElem2 ){
  1266   1473               rc = SQLITE_NOMEM;
  1267   1474               nElem = 0;
  1268   1475               break;
  1269   1476             }
  1270   1477             aElem = aElem2;
  1271   1478           }
         1479  +
  1272   1480           aElem[nElem++] = pE;
  1273   1481         }
  1274   1482       }
  1275   1483   
  1276   1484       /* If more than one term matches the prefix, sort the Fts3HashElem
  1277   1485       ** objects in term order using qsort(). This uses the same comparison
  1278   1486       ** callback as is used when flushing terms to disk.
  1279   1487       */
  1280   1488       if( nElem>1 ){
  1281   1489         qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
  1282   1490       }
  1283   1491   
  1284   1492     }else{
  1285         -    Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
         1493  +    /* The query is a simple term lookup that matches at most one term in
         1494  +    ** the index. All that is required is a straight hash-lookup. */
         1495  +    Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
  1286   1496       if( pE ){
  1287   1497         aElem = &pE;
  1288   1498         nElem = 1;
  1289   1499       }
  1290   1500     }
  1291   1501   
  1292   1502     if( nElem>0 ){
................................................................................
  1298   1508         memset(pReader, 0, nByte);
  1299   1509         pReader->iIdx = 0x7FFFFFFF;
  1300   1510         pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
  1301   1511         memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
  1302   1512       }
  1303   1513     }
  1304   1514   
  1305         -  if( isPrefix ){
         1515  +  if( bPrefix ){
  1306   1516       sqlite3_free(aElem);
  1307   1517     }
  1308   1518     *ppReader = pReader;
  1309   1519     return rc;
  1310   1520   }
  1311   1521   
  1312   1522   /*
................................................................................
  1358   1568     int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
  1359   1569     if( rc==0 ){
  1360   1570       if( pLhs->iDocid==pRhs->iDocid ){
  1361   1571         rc = pRhs->iIdx - pLhs->iIdx;
  1362   1572       }else{
  1363   1573         rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
  1364   1574       }
         1575  +  }
         1576  +  assert( pLhs->aNode && pRhs->aNode );
         1577  +  return rc;
         1578  +}
         1579  +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
         1580  +  int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
         1581  +  if( rc==0 ){
         1582  +    if( pLhs->iDocid==pRhs->iDocid ){
         1583  +      rc = pRhs->iIdx - pLhs->iIdx;
         1584  +    }else{
         1585  +      rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1;
         1586  +    }
  1365   1587     }
  1366   1588     assert( pLhs->aNode && pRhs->aNode );
  1367   1589     return rc;
  1368   1590   }
  1369   1591   
  1370   1592   /*
  1371   1593   ** Compare the term that the Fts3SegReader object passed as the first argument
................................................................................
  1910   2132       }
  1911   2133       rc = sqlite3_reset(pStmt);
  1912   2134     }
  1913   2135     return rc;
  1914   2136   }
  1915   2137   
  1916   2138   /*
  1917         -** Set *pnSegment to the total number of segments in the database. Set
  1918         -** *pnMax to the largest segment level in the database (segment levels
  1919         -** are stored in the 'level' column of the %_segdir table).
         2139  +** Set *pnMax to the largest segment level in the database for the index
         2140  +** iIndex.
         2141  +**
         2142  +** Segment levels are stored in the 'level' column of the %_segdir table.
  1920   2143   **
  1921   2144   ** Return SQLITE_OK if successful, or an SQLite error code if not.
  1922   2145   */
  1923         -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
         2146  +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
  1924   2147     sqlite3_stmt *pStmt;
  1925   2148     int rc;
         2149  +  assert( iIndex>=0 && iIndex<p->nIndex );
  1926   2150   
  1927         -  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
         2151  +  /* Set pStmt to the compiled version of:
         2152  +  **
         2153  +  **   SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
         2154  +  **
         2155  +  ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
         2156  +  */
         2157  +  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
  1928   2158     if( rc!=SQLITE_OK ) return rc;
         2159  +  sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
         2160  +  sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
  1929   2161     if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1930         -    *pnSegment = sqlite3_column_int(pStmt, 0);
  1931         -    *pnMax = sqlite3_column_int(pStmt, 1);
         2162  +    *pnMax = sqlite3_column_int(pStmt, 0);
  1932   2163     }
  1933   2164     return sqlite3_reset(pStmt);
  1934   2165   }
  1935   2166   
  1936   2167   /*
  1937   2168   ** This function is used after merging multiple segments into a single large
  1938   2169   ** segment to delete the old, now redundant, segment b-trees. Specifically,
................................................................................
  1945   2176   **   2) deletes all %_segdir entries with level iLevel, or all %_segdir
  1946   2177   **      entries regardless of level if (iLevel<0).
  1947   2178   **
  1948   2179   ** SQLITE_OK is returned if successful, otherwise an SQLite error code.
  1949   2180   */
  1950   2181   static int fts3DeleteSegdir(
  1951   2182     Fts3Table *p,                   /* Virtual table handle */
         2183  +  int iIndex,                     /* Index for p->aIndex */
  1952   2184     int iLevel,                     /* Level of %_segdir entries to delete */
  1953   2185     Fts3SegReader **apSegment,      /* Array of SegReader objects */
  1954   2186     int nReader                     /* Size of array apSegment */
  1955   2187   ){
  1956   2188     int rc;                         /* Return Code */
  1957   2189     int i;                          /* Iterator variable */
  1958   2190     sqlite3_stmt *pDelete;          /* SQL statement to delete rows */
................................................................................
  1967   2199         rc = sqlite3_reset(pDelete);
  1968   2200       }
  1969   2201     }
  1970   2202     if( rc!=SQLITE_OK ){
  1971   2203       return rc;
  1972   2204     }
  1973   2205   
         2206  +  assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
  1974   2207     if( iLevel==FTS3_SEGCURSOR_ALL ){
  1975         -    fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
  1976         -  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
  1977         -    sqlite3Fts3PendingTermsClear(p);
         2208  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
         2209  +    if( rc==SQLITE_OK ){
         2210  +      sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
         2211  +      sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
         2212  +    }
  1978   2213     }else{
  1979         -    assert( iLevel>=0 );
  1980         -    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
         2214  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
  1981   2215       if( rc==SQLITE_OK ){
  1982         -      sqlite3_bind_int(pDelete, 1, iLevel);
  1983         -      sqlite3_step(pDelete);
  1984         -      rc = sqlite3_reset(pDelete);
         2216  +      sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
  1985   2217       }
  1986   2218     }
         2219  +
         2220  +  if( rc==SQLITE_OK ){
         2221  +    sqlite3_step(pDelete);
         2222  +    rc = sqlite3_reset(pDelete);
         2223  +  }
  1987   2224   
  1988   2225     return rc;
  1989   2226   }
  1990   2227   
  1991   2228   /*
  1992   2229   ** When this function is called, buffer *ppList (size *pnList bytes) contains 
  1993   2230   ** a position list that may (or may not) feature multiple columns. This
................................................................................
  2027   2264       p += sqlite3Fts3GetVarint32(p, &iCurrent);
  2028   2265     }
  2029   2266   
  2030   2267     *ppList = pList;
  2031   2268     *pnList = nList;
  2032   2269   }
  2033   2270   
  2034         -int sqlite3Fts3SegReaderStart(
         2271  +/*
         2272  +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any
         2273  +** existing data). Grow the buffer if required.
         2274  +**
         2275  +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered
         2276  +** trying to resize the buffer, return SQLITE_NOMEM.
         2277  +*/
         2278  +static int fts3MsrBufferData(
         2279  +  Fts3MultiSegReader *pMsr,       /* Multi-segment-reader handle */
         2280  +  char *pList,
         2281  +  int nList
         2282  +){
         2283  +  if( nList>pMsr->nBuffer ){
         2284  +    char *pNew;
         2285  +    pMsr->nBuffer = nList*2;
         2286  +    pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer);
         2287  +    if( !pNew ) return SQLITE_NOMEM;
         2288  +    pMsr->aBuffer = pNew;
         2289  +  }
         2290  +
         2291  +  memcpy(pMsr->aBuffer, pList, nList);
         2292  +  return SQLITE_OK;
         2293  +}
         2294  +
         2295  +int sqlite3Fts3MsrIncrNext(
         2296  +  Fts3Table *p,                   /* Virtual table handle */
         2297  +  Fts3MultiSegReader *pMsr,       /* Multi-segment-reader handle */
         2298  +  sqlite3_int64 *piDocid,         /* OUT: Docid value */
         2299  +  char **paPoslist,               /* OUT: Pointer to position list */
         2300  +  int *pnPoslist                  /* OUT: Size of position list in bytes */
         2301  +){
         2302  +  int nMerge = pMsr->nAdvance;
         2303  +  Fts3SegReader **apSegment = pMsr->apSegment;
         2304  +  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
         2305  +    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
         2306  +  );
         2307  +
         2308  +  if( nMerge==0 ){
         2309  +    *paPoslist = 0;
         2310  +    return SQLITE_OK;
         2311  +  }
         2312  +
         2313  +  while( 1 ){
         2314  +    Fts3SegReader *pSeg;
         2315  +    pSeg = pMsr->apSegment[0];
         2316  +
         2317  +    if( pSeg->pOffsetList==0 ){
         2318  +      *paPoslist = 0;
         2319  +      break;
         2320  +    }else{
         2321  +      int rc;
         2322  +      char *pList;
         2323  +      int nList;
         2324  +      int j;
         2325  +      sqlite3_int64 iDocid = apSegment[0]->iDocid;
         2326  +
         2327  +      rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
         2328  +      j = 1;
         2329  +      while( rc==SQLITE_OK 
         2330  +        && j<nMerge
         2331  +        && apSegment[j]->pOffsetList
         2332  +        && apSegment[j]->iDocid==iDocid
         2333  +      ){
         2334  +        rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
         2335  +        j++;
         2336  +      }
         2337  +      if( rc!=SQLITE_OK ) return rc;
         2338  +      fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
         2339  +
         2340  +      if( pMsr->iColFilter>=0 ){
         2341  +        fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
         2342  +      }
         2343  +
         2344  +      if( nList>0 ){
         2345  +        if( fts3SegReaderIsPending(apSegment[0]) ){
         2346  +          rc = fts3MsrBufferData(pMsr, pList, nList+1);
         2347  +          if( rc!=SQLITE_OK ) return rc;
         2348  +          *paPoslist = pMsr->aBuffer;
         2349  +          assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 );
         2350  +        }else{
         2351  +          *paPoslist = pList;
         2352  +        }
         2353  +        *piDocid = iDocid;
         2354  +        *pnPoslist = nList;
         2355  +        break;
         2356  +      }
         2357  +    }
         2358  +  }
         2359  +
         2360  +  return SQLITE_OK;
         2361  +}
         2362  +
         2363  +static int fts3SegReaderStart(
  2035   2364     Fts3Table *p,                   /* Virtual table handle */
  2036         -  Fts3SegReaderCursor *pCsr,      /* Cursor object */
  2037         -  Fts3SegFilter *pFilter          /* Restrictions on range of iteration */
         2365  +  Fts3MultiSegReader *pCsr,       /* Cursor object */
         2366  +  const char *zTerm,              /* Term searched for (or NULL) */
         2367  +  int nTerm                       /* Length of zTerm in bytes */
  2038   2368   ){
  2039   2369     int i;
  2040         -
  2041         -  /* Initialize the cursor object */
  2042         -  pCsr->pFilter = pFilter;
         2370  +  int nSeg = pCsr->nSegment;
  2043   2371   
  2044   2372     /* If the Fts3SegFilter defines a specific term (or term prefix) to search 
  2045   2373     ** for, then advance each segment iterator until it points to a term of
  2046   2374     ** equal or greater value than the specified term. This prevents many
  2047   2375     ** unnecessary merge/sort operations for the case where single segment
  2048   2376     ** b-tree leaf nodes contain more than one term.
  2049   2377     */
  2050         -  for(i=0; i<pCsr->nSegment; i++){
  2051         -    int nTerm = pFilter->nTerm;
  2052         -    const char *zTerm = pFilter->zTerm;
         2378  +  for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){
  2053   2379       Fts3SegReader *pSeg = pCsr->apSegment[i];
  2054   2380       do {
  2055         -      int rc = fts3SegReaderNext(p, pSeg);
         2381  +      int rc = fts3SegReaderNext(p, pSeg, 0);
  2056   2382         if( rc!=SQLITE_OK ) return rc;
  2057   2383       }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
  2058   2384     }
  2059         -  fts3SegReaderSort(
  2060         -      pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
         2385  +  fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp);
         2386  +
         2387  +  return SQLITE_OK;
         2388  +}
         2389  +
         2390  +int sqlite3Fts3SegReaderStart(
         2391  +  Fts3Table *p,                   /* Virtual table handle */
         2392  +  Fts3MultiSegReader *pCsr,       /* Cursor object */
         2393  +  Fts3SegFilter *pFilter          /* Restrictions on range of iteration */
         2394  +){
         2395  +  pCsr->pFilter = pFilter;
         2396  +  return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm);
         2397  +}
         2398  +
         2399  +int sqlite3Fts3MsrIncrStart(
         2400  +  Fts3Table *p,                   /* Virtual table handle */
         2401  +  Fts3MultiSegReader *pCsr,       /* Cursor object */
         2402  +  int iCol,                       /* Column to match on. */
         2403  +  const char *zTerm,              /* Term to iterate through a doclist for */
         2404  +  int nTerm                       /* Number of bytes in zTerm */
         2405  +){
         2406  +  int i;
         2407  +  int rc;
         2408  +  int nSegment = pCsr->nSegment;
         2409  +  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
         2410  +    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
         2411  +  );
         2412  +
         2413  +  assert( pCsr->pFilter==0 );
         2414  +  assert( zTerm && nTerm>0 );
         2415  +
         2416  +  /* Advance each segment iterator until it points to the term zTerm/nTerm. */
         2417  +  rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm);
         2418  +  if( rc!=SQLITE_OK ) return rc;
         2419  +
         2420  +  /* Determine how many of the segments actually point to zTerm/nTerm. */
         2421  +  for(i=0; i<nSegment; i++){
         2422  +    Fts3SegReader *pSeg = pCsr->apSegment[i];
         2423  +    if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
         2424  +      break;
         2425  +    }
         2426  +  }
         2427  +  pCsr->nAdvance = i;
         2428  +
         2429  +  /* Advance each of the segments to point to the first docid. */
         2430  +  for(i=0; i<pCsr->nAdvance; i++){
         2431  +    rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
         2432  +    if( rc!=SQLITE_OK ) return rc;
         2433  +  }
         2434  +  fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
         2435  +
         2436  +  assert( iCol<0 || iCol<p->nColumn );
         2437  +  pCsr->iColFilter = iCol;
         2438  +
         2439  +  return SQLITE_OK;
         2440  +}
         2441  +
         2442  +/*
         2443  +** This function is called on a MultiSegReader that has been started using
         2444  +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also
         2445  +** have been made. Calling this function puts the MultiSegReader in such
         2446  +** a state that if the next two calls are:
         2447  +**
         2448  +**   sqlite3Fts3SegReaderStart()
         2449  +**   sqlite3Fts3SegReaderStep()
         2450  +**
         2451  +** then the entire doclist for the term is available in 
         2452  +** MultiSegReader.aDoclist/nDoclist.
         2453  +*/
         2454  +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){
         2455  +  int i;                          /* Used to iterate through segment-readers */
         2456  +
         2457  +  assert( pCsr->zTerm==0 );
         2458  +  assert( pCsr->nTerm==0 );
         2459  +  assert( pCsr->aDoclist==0 );
         2460  +  assert( pCsr->nDoclist==0 );
         2461  +
         2462  +  pCsr->nAdvance = 0;
         2463  +  pCsr->bRestart = 1;
         2464  +  for(i=0; i<pCsr->nSegment; i++){
         2465  +    pCsr->apSegment[i]->pOffsetList = 0;
         2466  +    pCsr->apSegment[i]->nOffsetList = 0;
         2467  +    pCsr->apSegment[i]->iDocid = 0;
         2468  +  }
  2061   2469   
  2062   2470     return SQLITE_OK;
  2063   2471   }
         2472  +
  2064   2473   
  2065   2474   int sqlite3Fts3SegReaderStep(
  2066   2475     Fts3Table *p,                   /* Virtual table handle */
  2067         -  Fts3SegReaderCursor *pCsr       /* Cursor object */
         2476  +  Fts3MultiSegReader *pCsr        /* Cursor object */
  2068   2477   ){
  2069   2478     int rc = SQLITE_OK;
  2070   2479   
  2071   2480     int isIgnoreEmpty =  (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
  2072   2481     int isRequirePos =   (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
  2073   2482     int isColFilter =    (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
  2074   2483     int isPrefix =       (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
  2075   2484     int isScan =         (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
  2076   2485   
  2077   2486     Fts3SegReader **apSegment = pCsr->apSegment;
  2078   2487     int nSegment = pCsr->nSegment;
  2079   2488     Fts3SegFilter *pFilter = pCsr->pFilter;
         2489  +  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
         2490  +    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
         2491  +  );
  2080   2492   
  2081   2493     if( pCsr->nSegment==0 ) return SQLITE_OK;
  2082   2494   
  2083   2495     do {
  2084   2496       int nMerge;
  2085   2497       int i;
  2086   2498     
  2087   2499       /* Advance the first pCsr->nAdvance entries in the apSegment[] array
  2088   2500       ** forward. Then sort the list in order of current term again.  
  2089   2501       */
  2090   2502       for(i=0; i<pCsr->nAdvance; i++){
  2091         -      rc = fts3SegReaderNext(p, apSegment[i]);
         2503  +      rc = fts3SegReaderNext(p, apSegment[i], 0);
  2092   2504         if( rc!=SQLITE_OK ) return rc;
  2093   2505       }
  2094   2506       fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
  2095   2507       pCsr->nAdvance = 0;
  2096   2508   
  2097   2509       /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */
  2098   2510       assert( rc==SQLITE_OK );
................................................................................
  2123   2535           && apSegment[nMerge]->nTerm==pCsr->nTerm 
  2124   2536           && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm)
  2125   2537       ){
  2126   2538         nMerge++;
  2127   2539       }
  2128   2540   
  2129   2541       assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
  2130         -    if( nMerge==1 && !isIgnoreEmpty ){
  2131         -      pCsr->aDoclist = apSegment[0]->aDoclist;
         2542  +    if( nMerge==1 
         2543  +     && !isIgnoreEmpty 
         2544  +     && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
         2545  +    ){
  2132   2546         pCsr->nDoclist = apSegment[0]->nDoclist;
  2133         -      rc = SQLITE_ROW;
         2547  +      if( fts3SegReaderIsPending(apSegment[0]) ){
         2548  +        rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist);
         2549  +        pCsr->aDoclist = pCsr->aBuffer;
         2550  +      }else{
         2551  +        pCsr->aDoclist = apSegment[0]->aDoclist;
         2552  +      }
         2553  +      if( rc==SQLITE_OK ) rc = SQLITE_ROW;
  2134   2554       }else{
  2135   2555         int nDoclist = 0;           /* Size of doclist */
  2136   2556         sqlite3_int64 iPrev = 0;    /* Previous docid stored in doclist */
  2137   2557   
  2138   2558         /* The current term of the first nMerge entries in the array
  2139   2559         ** of Fts3SegReader objects is the same. The doclists must be merged
  2140   2560         ** and a single term returned with the merged doclist.
  2141   2561         */
  2142   2562         for(i=0; i<nMerge; i++){
  2143         -        fts3SegReaderFirstDocid(apSegment[i]);
         2563  +        fts3SegReaderFirstDocid(p, apSegment[i]);
  2144   2564         }
  2145         -      fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
         2565  +      fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
  2146   2566         while( apSegment[0]->pOffsetList ){
  2147   2567           int j;                    /* Number of segments that share a docid */
  2148   2568           char *pList;
  2149   2569           int nList;
  2150   2570           int nByte;
  2151   2571           sqlite3_int64 iDocid = apSegment[0]->iDocid;
  2152         -        fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
         2572  +        fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
  2153   2573           j = 1;
  2154   2574           while( j<nMerge
  2155   2575               && apSegment[j]->pOffsetList
  2156   2576               && apSegment[j]->iDocid==iDocid
  2157   2577           ){
  2158         -          fts3SegReaderNextDocid(apSegment[j], 0, 0);
         2578  +          fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
  2159   2579             j++;
  2160   2580           }
  2161   2581   
  2162   2582           if( isColFilter ){
  2163   2583             fts3ColumnFilter(pFilter->iCol, &pList, &nList);
  2164   2584           }
  2165   2585   
  2166   2586           if( !isIgnoreEmpty || nList>0 ){
  2167         -          nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
         2587  +
         2588  +          /* Calculate the 'docid' delta value to write into the merged 
         2589  +          ** doclist. */
         2590  +          sqlite3_int64 iDelta;
         2591  +          if( p->bDescIdx && nDoclist>0 ){
         2592  +            iDelta = iPrev - iDocid;
         2593  +          }else{
         2594  +            iDelta = iDocid - iPrev;
         2595  +          }
         2596  +          assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
         2597  +          assert( nDoclist>0 || iDelta==iDocid );
         2598  +
         2599  +          nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
  2168   2600             if( nDoclist+nByte>pCsr->nBuffer ){
  2169   2601               char *aNew;
  2170   2602               pCsr->nBuffer = (nDoclist+nByte)*2;
  2171   2603               aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
  2172   2604               if( !aNew ){
  2173   2605                 return SQLITE_NOMEM;
  2174   2606               }
  2175   2607               pCsr->aBuffer = aNew;
  2176   2608             }
  2177         -          nDoclist += sqlite3Fts3PutVarint(
  2178         -              &pCsr->aBuffer[nDoclist], iDocid-iPrev
  2179         -          );
         2609  +          nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
  2180   2610             iPrev = iDocid;
  2181   2611             if( isRequirePos ){
  2182   2612               memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
  2183   2613               nDoclist += nList;
  2184   2614               pCsr->aBuffer[nDoclist++] = '\0';
  2185   2615             }
  2186   2616           }
  2187   2617   
  2188         -        fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
         2618  +        fts3SegReaderSort(apSegment, nMerge, j, xCmp);
  2189   2619         }
  2190   2620         if( nDoclist>0 ){
  2191   2621           pCsr->aDoclist = pCsr->aBuffer;
  2192   2622           pCsr->nDoclist = nDoclist;
  2193   2623           rc = SQLITE_ROW;
  2194   2624         }
  2195   2625       }
  2196   2626       pCsr->nAdvance = nMerge;
  2197   2627     }while( rc==SQLITE_OK );
  2198   2628   
  2199   2629     return rc;
  2200   2630   }
         2631  +
  2201   2632   
  2202   2633   void sqlite3Fts3SegReaderFinish(
  2203         -  Fts3SegReaderCursor *pCsr       /* Cursor object */
         2634  +  Fts3MultiSegReader *pCsr       /* Cursor object */
  2204   2635   ){
  2205   2636     if( pCsr ){
  2206   2637       int i;
  2207   2638       for(i=0; i<pCsr->nSegment; i++){
  2208   2639         sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
  2209   2640       }
  2210   2641       sqlite3_free(pCsr->apSegment);
................................................................................
  2223   2654   ** currently present in the database.
  2224   2655   **
  2225   2656   ** If this function is called with iLevel<0, but there is only one
  2226   2657   ** segment in the database, SQLITE_DONE is returned immediately. 
  2227   2658   ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 
  2228   2659   ** an SQLite error code is returned.
  2229   2660   */
  2230         -static int fts3SegmentMerge(Fts3Table *p, int iLevel){
         2661  +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
  2231   2662     int rc;                         /* Return code */
  2232   2663     int iIdx = 0;                   /* Index of new segment */
  2233         -  int iNewLevel = 0;              /* Level to create new segment at */
         2664  +  int iNewLevel = 0;              /* Level/index to create new segment at */
  2234   2665     SegmentWriter *pWriter = 0;     /* Used to write the new, merged, segment */
  2235   2666     Fts3SegFilter filter;           /* Segment term filter condition */
  2236         -  Fts3SegReaderCursor csr;        /* Cursor to iterate through level(s) */
         2667  +  Fts3MultiSegReader csr;        /* Cursor to iterate through level(s) */
         2668  +  int bIgnoreEmpty = 0;           /* True to ignore empty segments */
  2237   2669   
  2238         -  rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr);
         2670  +  assert( iLevel==FTS3_SEGCURSOR_ALL
         2671  +       || iLevel==FTS3_SEGCURSOR_PENDING
         2672  +       || iLevel>=0
         2673  +  );
         2674  +  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
         2675  +  assert( iIndex>=0 && iIndex<p->nIndex );
         2676  +
         2677  +  rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
  2239   2678     if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
  2240   2679   
  2241   2680     if( iLevel==FTS3_SEGCURSOR_ALL ){
  2242   2681       /* This call is to merge all segments in the database to a single
  2243   2682       ** segment. The level of the new segment is equal to the the numerically 
  2244         -    ** greatest segment level currently present in the database. The index
  2245         -    ** of the new segment is always 0.  */
  2246         -    int nDummy; /* TODO: Remove this */
         2683  +    ** greatest segment level currently present in the database for this
         2684  +    ** index. The idx of the new segment is always 0.  */
  2247   2685       if( csr.nSegment==1 ){
  2248   2686         rc = SQLITE_DONE;
  2249   2687         goto finished;
  2250   2688       }
  2251         -    rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel);
         2689  +    rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
         2690  +    bIgnoreEmpty = 1;
         2691  +
         2692  +  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
         2693  +    iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; 
         2694  +    rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
  2252   2695     }else{
  2253         -    /* This call is to merge all segments at level iLevel. Find the next
         2696  +    /* This call is to merge all segments at level iLevel. find the next
  2254   2697       ** available segment index at level iLevel+1. The call to
  2255   2698       ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 
  2256   2699       ** a single iLevel+2 segment if necessary.  */
  2257         -    iNewLevel = iLevel+1;
  2258         -    rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
         2700  +    rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
         2701  +    iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
  2259   2702     }
  2260   2703     if( rc!=SQLITE_OK ) goto finished;
  2261   2704     assert( csr.nSegment>0 );
  2262         -  assert( iNewLevel>=0 );
         2705  +  assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
         2706  +  assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
  2263   2707   
  2264   2708     memset(&filter, 0, sizeof(Fts3SegFilter));
  2265   2709     filter.flags = FTS3_SEGMENT_REQUIRE_POS;
  2266         -  filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
         2710  +  filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
  2267   2711   
  2268   2712     rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
  2269   2713     while( SQLITE_OK==rc ){
  2270   2714       rc = sqlite3Fts3SegReaderStep(p, &csr);
  2271   2715       if( rc!=SQLITE_ROW ) break;
  2272   2716       rc = fts3SegWriterAdd(p, &pWriter, 1, 
  2273   2717           csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
  2274   2718     }
  2275   2719     if( rc!=SQLITE_OK ) goto finished;
  2276   2720     assert( pWriter );
  2277   2721   
  2278         -  rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment);
  2279         -  if( rc!=SQLITE_OK ) goto finished;
         2722  +  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
         2723  +    rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
         2724  +    if( rc!=SQLITE_OK ) goto finished;
         2725  +  }
  2280   2726     rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
  2281   2727   
  2282   2728    finished:
  2283   2729     fts3SegWriterFree(pWriter);
  2284   2730     sqlite3Fts3SegReaderFinish(&csr);
  2285   2731     return rc;
  2286   2732   }
  2287   2733   
  2288   2734   
  2289   2735   /* 
  2290         -** Flush the contents of pendingTerms to a level 0 segment.
         2736  +** Flush the contents of pendingTerms to level 0 segments.
  2291   2737   */
  2292   2738   int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
  2293         -  return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING);
         2739  +  int rc = SQLITE_OK;
         2740  +  int i;
         2741  +  for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
         2742  +    rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
         2743  +    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
         2744  +  }
         2745  +  sqlite3Fts3PendingTermsClear(p);
         2746  +  return rc;
  2294   2747   }
  2295   2748   
  2296   2749   /*
  2297   2750   ** Encode N integers as varints into a blob.
  2298   2751   */
  2299   2752   static void fts3EncodeIntArray(
  2300   2753     int N,             /* The number of integers to encode */
................................................................................
  2436   2889       return;
  2437   2890     }
  2438   2891     sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
  2439   2892     sqlite3_step(pStmt);
  2440   2893     *pRC = sqlite3_reset(pStmt);
  2441   2894     sqlite3_free(a);
  2442   2895   }
         2896  +
         2897  +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
         2898  +  int i;
         2899  +  int bSeenDone = 0;
         2900  +  int rc = SQLITE_OK;
         2901  +  for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
         2902  +    rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
         2903  +    if( rc==SQLITE_DONE ){
         2904  +      bSeenDone = 1;
         2905  +      rc = SQLITE_OK;
         2906  +    }
         2907  +  }
         2908  +  sqlite3Fts3SegmentsClose(p);
         2909  +  sqlite3Fts3PendingTermsClear(p);
         2910  +
         2911  +  return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
         2912  +}
  2443   2913   
  2444   2914   /*
  2445   2915   ** Handle a 'special' INSERT of the form:
  2446   2916   **
  2447   2917   **   "INSERT INTO tbl(tbl) VALUES(<expr>)"
  2448   2918   **
  2449   2919   ** Argument pVal contains the result of <expr>. Currently the only 
................................................................................
  2453   2923     int rc;                         /* Return Code */
  2454   2924     const char *zVal = (const char *)sqlite3_value_text(pVal);
  2455   2925     int nVal = sqlite3_value_bytes(pVal);
  2456   2926   
  2457   2927     if( !zVal ){
  2458   2928       return SQLITE_NOMEM;
  2459   2929     }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
  2460         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
  2461         -    if( rc==SQLITE_DONE ){
  2462         -      rc = SQLITE_OK;
  2463         -    }else{
  2464         -      sqlite3Fts3PendingTermsClear(p);
  2465         -    }
         2930  +    rc = fts3DoOptimize(p, 0);
  2466   2931   #ifdef SQLITE_TEST
  2467   2932     }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
  2468   2933       p->nNodeSize = atoi(&zVal[9]);
  2469   2934       rc = SQLITE_OK;
  2470   2935     }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
  2471   2936       p->nMaxPendingData = atoi(&zVal[11]);
  2472   2937       rc = SQLITE_OK;
  2473   2938   #endif
  2474   2939     }else{
  2475   2940       rc = SQLITE_ERROR;
  2476   2941     }
  2477   2942   
  2478         -  sqlite3Fts3SegmentsClose(p);
  2479   2943     return rc;
  2480   2944   }
  2481   2945   
  2482         -/*
  2483         -** Return the deferred doclist associated with deferred token pDeferred.
  2484         -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
  2485         -** been called to allocate and populate the doclist.
  2486         -*/
  2487         -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
  2488         -  if( pDeferred->pList ){
  2489         -    *pnByte = pDeferred->pList->nData;
  2490         -    return pDeferred->pList->aData;
  2491         -  }
  2492         -  *pnByte = 0;
  2493         -  return 0;
  2494         -}
  2495         -
  2496         -/*
  2497         -** Helper fucntion for FreeDeferredDoclists(). This function removes all
  2498         -** references to deferred doclists from within the tree of Fts3Expr 
  2499         -** structures headed by 
  2500         -*/
  2501         -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
  2502         -  if( pExpr ){
  2503         -    fts3DeferredDoclistClear(pExpr->pLeft);
  2504         -    fts3DeferredDoclistClear(pExpr->pRight);
  2505         -    if( pExpr->isLoaded ){
  2506         -      sqlite3_free(pExpr->aDoclist);
  2507         -      pExpr->isLoaded = 0;
  2508         -      pExpr->aDoclist = 0;
  2509         -      pExpr->nDoclist = 0;
  2510         -      pExpr->pCurrent = 0;
  2511         -      pExpr->iCurrent = 0;
  2512         -    }
  2513         -  }
  2514         -}
  2515         -
  2516   2946   /*
  2517   2947   ** Delete all cached deferred doclists. Deferred doclists are cached
  2518   2948   ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
  2519   2949   */
  2520   2950   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
  2521   2951     Fts3DeferredToken *pDef;
  2522   2952     for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
  2523         -    sqlite3_free(pDef->pList);
         2953  +    fts3PendingListDelete(pDef->pList);
  2524   2954       pDef->pList = 0;
  2525   2955     }
  2526         -  if( pCsr->pDeferred ){
  2527         -    fts3DeferredDoclistClear(pCsr->pExpr);
  2528         -  }
  2529   2956   }
  2530   2957   
  2531   2958   /*
  2532   2959   ** Free all entries in the pCsr->pDeffered list. Entries are added to 
  2533   2960   ** this list using sqlite3Fts3DeferToken().
  2534   2961   */
  2535   2962   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
  2536   2963     Fts3DeferredToken *pDef;
  2537   2964     Fts3DeferredToken *pNext;
  2538   2965     for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
  2539   2966       pNext = pDef->pNext;
  2540         -    sqlite3_free(pDef->pList);
         2967  +    fts3PendingListDelete(pDef->pList);
  2541   2968       sqlite3_free(pDef);
  2542   2969     }
  2543   2970     pCsr->pDeferred = 0;
  2544   2971   }
  2545   2972   
  2546   2973   /*
  2547   2974   ** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
................................................................................
  2597   3024           rc = fts3PendingListAppendVarint(&pDef->pList, 0);
  2598   3025         }
  2599   3026       }
  2600   3027     }
  2601   3028   
  2602   3029     return rc;
  2603   3030   }
         3031  +
         3032  +int sqlite3Fts3DeferredTokenList(
         3033  +  Fts3DeferredToken *p, 
         3034  +  char **ppData, 
         3035  +  int *pnData
         3036  +){
         3037  +  char *pRet;
         3038  +  int nSkip;
         3039  +  sqlite3_int64 dummy;
         3040  +
         3041  +  *ppData = 0;
         3042  +  *pnData = 0;
         3043  +
         3044  +  if( p->pList==0 ){
         3045  +    return SQLITE_OK;
         3046  +  }
         3047  +
         3048  +  pRet = (char *)sqlite3_malloc(p->pList->nData);
         3049  +  if( !pRet ) return SQLITE_NOMEM;
         3050  +
         3051  +  nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
         3052  +  *pnData = p->pList->nData - nSkip;
         3053  +  *ppData = pRet;
         3054  +  
         3055  +  memcpy(pRet, &p->pList->aData[nSkip], *pnData);
         3056  +  return SQLITE_OK;
         3057  +}
  2604   3058   
  2605   3059   /*
  2606   3060   ** Add an entry for token pToken to the pCsr->pDeferred list.
  2607   3061   */
  2608   3062   int sqlite3Fts3DeferToken(
  2609   3063     Fts3Cursor *pCsr,               /* Fts3 table cursor */
  2610   3064     Fts3PhraseToken *pToken,        /* Token to defer */
................................................................................
  2672   3126     sqlite3_value **apVal,          /* Array of arguments */
  2673   3127     sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
  2674   3128   ){
  2675   3129     Fts3Table *p = (Fts3Table *)pVtab;
  2676   3130     int rc = SQLITE_OK;             /* Return Code */
  2677   3131     int isRemove = 0;               /* True for an UPDATE or DELETE */
  2678   3132     sqlite3_int64 iRemove = 0;      /* Rowid removed by UPDATE or DELETE */
  2679         -  u32 *aSzIns;                    /* Sizes of inserted documents */
         3133  +  u32 *aSzIns = 0;                /* Sizes of inserted documents */
  2680   3134     u32 *aSzDel;                    /* Sizes of deleted documents */
  2681   3135     int nChng = 0;                  /* Net change in number of documents */
  2682   3136     int bInsertDone = 0;
  2683   3137   
  2684   3138     assert( p->pSegments==0 );
  2685   3139   
  2686   3140     /* Check for a "special" INSERT operation. One of the form:
................................................................................
  2687   3141     **
  2688   3142     **   INSERT INTO xyz(xyz) VALUES('command');
  2689   3143     */
  2690   3144     if( nArg>1 
  2691   3145      && sqlite3_value_type(apVal[0])==SQLITE_NULL 
  2692   3146      && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL 
  2693   3147     ){
  2694         -    return fts3SpecialInsert(p, apVal[p->nColumn+2]);
         3148  +    rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
         3149  +    goto update_out;
  2695   3150     }
  2696   3151   
  2697   3152     /* Allocate space to hold the change in document sizes */
  2698   3153     aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
  2699         -  if( aSzIns==0 ) return SQLITE_NOMEM;
         3154  +  if( aSzIns==0 ){
         3155  +    rc = SQLITE_NOMEM;
         3156  +    goto update_out;
         3157  +  }
  2700   3158     aSzDel = &aSzIns[p->nColumn+1];
  2701   3159     memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
  2702   3160   
  2703   3161     /* If this is an INSERT operation, or an UPDATE that modifies the rowid
  2704   3162     ** value, then this operation requires constraint handling.
  2705   3163     **
  2706   3164     ** If the on-conflict mode is REPLACE, this means that the existing row
................................................................................
  2742   3200         }else{
  2743   3201           rc = fts3InsertData(p, apVal, pRowid);
  2744   3202           bInsertDone = 1;
  2745   3203         }
  2746   3204       }
  2747   3205     }
  2748   3206     if( rc!=SQLITE_OK ){
  2749         -    sqlite3_free(aSzIns);
  2750         -    return rc;
         3207  +    goto update_out;
  2751   3208     }
  2752   3209   
  2753   3210     /* If this is a DELETE or UPDATE operation, remove the old record. */
  2754   3211     if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  2755   3212       assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
  2756   3213       rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
  2757   3214       isRemove = 1;
................................................................................
  2776   3233       nChng++;
  2777   3234     }
  2778   3235   
  2779   3236     if( p->bHasStat ){
  2780   3237       fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
  2781   3238     }
  2782   3239   
         3240  + update_out:
  2783   3241     sqlite3_free(aSzIns);
  2784   3242     sqlite3Fts3SegmentsClose(p);
  2785   3243     return rc;
  2786   3244   }
  2787   3245   
  2788   3246   /* 
  2789   3247   ** Flush any data in the pending-terms hash table to disk. If successful,
................................................................................
  2790   3248   ** merge all segments in the database (including the new segment, if 
  2791   3249   ** there was any data to flush) into a single segment. 
  2792   3250   */
  2793   3251   int sqlite3Fts3Optimize(Fts3Table *p){
  2794   3252     int rc;
  2795   3253     rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
  2796   3254     if( rc==SQLITE_OK ){
  2797         -    rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
  2798         -    if( rc==SQLITE_OK ){
  2799         -      rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2800         -      if( rc==SQLITE_OK ){
  2801         -        sqlite3Fts3PendingTermsClear(p);
  2802         -      }
         3255  +    rc = fts3DoOptimize(p, 1);
         3256  +    if( rc==SQLITE_OK || rc==SQLITE_DONE ){
         3257  +      int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
         3258  +      if( rc2!=SQLITE_OK ) rc = rc2;
  2803   3259       }else{
  2804   3260         sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
  2805   3261         sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2806   3262       }
  2807   3263     }
  2808   3264     sqlite3Fts3SegmentsClose(p);
  2809   3265     return rc;
  2810   3266   }
  2811   3267   
  2812   3268   #endif

Changes to main.mk.

   218    218     parse.h \
   219    219     sqlite3.h
   220    220   
   221    221   
   222    222   # Source code to the test files.
   223    223   #
   224    224   TESTSRC = \
          225  +  $(TOP)/ext/fts3/fts3_test.c \
   225    226     $(TOP)/src/test1.c \
   226    227     $(TOP)/src/test2.c \
   227    228     $(TOP)/src/test3.c \
   228    229     $(TOP)/src/test4.c \
   229    230     $(TOP)/src/test5.c \
   230    231     $(TOP)/src/test6.c \
   231    232     $(TOP)/src/test7.c \

Changes to src/alter.c.

   354    354   
   355    355     /* Drop the table and index from the internal schema.  */
   356    356     sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
   357    357   
   358    358     /* Reload the table, index and permanent trigger schemas. */
   359    359     zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName);
   360    360     if( !zWhere ) return;
   361         -  sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
          361  +  sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
   362    362   
   363    363   #ifndef SQLITE_OMIT_TRIGGER
   364    364     /* Now, if the table is not stored in the temp database, reload any temp 
   365    365     ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. 
   366    366     */
   367    367     if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){
   368         -    sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC);
          368  +    sqlite3VdbeAddParseSchemaOp(v, 1, zWhere);
   369    369     }
   370    370   #endif
   371    371   }
   372    372   
   373    373   /*
   374    374   ** Parameter zName is the name of a table that is about to be altered
   375    375   ** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN).

Changes to src/btree.c.

   853    853   ** the page, 1 means the second cell, and so forth) return a pointer
   854    854   ** to the cell content.
   855    855   **
   856    856   ** This routine works only for pages that do not contain overflow cells.
   857    857   */
   858    858   #define findCell(P,I) \
   859    859     ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)])))
          860  +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I)))))
          861  +
   860    862   
   861    863   /*
   862    864   ** This a more complex version of findCell() that works for
   863    865   ** pages that do contain overflow cells.
   864    866   */
   865    867   static u8 *findOverflowCell(MemPage *pPage, int iCell){
   866    868     int i;
................................................................................
  4447   4449     if( pCur->eState==CURSOR_INVALID ){
  4448   4450       *pRes = -1;
  4449   4451       assert( pCur->apPage[pCur->iPage]->nCell==0 );
  4450   4452       return SQLITE_OK;
  4451   4453     }
  4452   4454     assert( pCur->apPage[0]->intKey || pIdxKey );
  4453   4455     for(;;){
  4454         -    int lwr, upr;
         4456  +    int lwr, upr, idx;
  4455   4457       Pgno chldPg;
  4456   4458       MemPage *pPage = pCur->apPage[pCur->iPage];
  4457   4459       int c;
  4458   4460   
  4459   4461       /* pPage->nCell must be greater than zero. If this is the root-page
  4460   4462       ** the cursor would have been INVALID above and this for(;;) loop
  4461   4463       ** not run. If this is not the root-page, then the moveToChild() routine
................................................................................
  4463   4465       ** be the right kind (index or table) of b-tree page. Otherwise
  4464   4466       ** a moveToChild() or moveToRoot() call would have detected corruption.  */
  4465   4467       assert( pPage->nCell>0 );
  4466   4468       assert( pPage->intKey==(pIdxKey==0) );
  4467   4469       lwr = 0;
  4468   4470       upr = pPage->nCell-1;
  4469   4471       if( biasRight ){
  4470         -      pCur->aiIdx[pCur->iPage] = (u16)upr;
         4472  +      pCur->aiIdx[pCur->iPage] = (u16)(idx = upr);
  4471   4473       }else{
  4472         -      pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2);
         4474  +      pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2);
  4473   4475       }
  4474   4476       for(;;){
  4475         -      int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */
  4476   4477         u8 *pCell;                          /* Pointer to current cell in pPage */
  4477   4478   
         4479  +      assert( idx==pCur->aiIdx[pCur->iPage] );
  4478   4480         pCur->info.nSize = 0;
  4479   4481         pCell = findCell(pPage, idx) + pPage->childPtrSize;
  4480   4482         if( pPage->intKey ){
  4481   4483           i64 nCellKey;
  4482   4484           if( pPage->hasData ){
  4483   4485             u32 dummy;
  4484   4486             pCell += getVarint32(pCell, dummy);
................................................................................
  4553   4555           lwr = idx+1;
  4554   4556         }else{
  4555   4557           upr = idx-1;
  4556   4558         }
  4557   4559         if( lwr>upr ){
  4558   4560           break;
  4559   4561         }
  4560         -      pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2);
         4562  +      pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2);
  4561   4563       }
  4562   4564       assert( lwr==upr+1 );
  4563   4565       assert( pPage->isInit );
  4564   4566       if( pPage->leaf ){
  4565   4567         chldPg = 0;
  4566   4568       }else if( lwr>=pPage->nCell ){
  4567   4569         chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]);
................................................................................
  5415   5417     }
  5416   5418     rc = freeSpace(pPage, pc, sz);
  5417   5419     if( rc ){
  5418   5420       *pRC = rc;
  5419   5421       return;
  5420   5422     }
  5421   5423     endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2];
         5424  +  assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 );  /* ptr is always 2-byte aligned */
  5422   5425     while( ptr<endPtr ){
  5423         -    ptr[0] = ptr[2];
  5424         -    ptr[1] = ptr[3];
         5426  +    *(u16*)ptr = *(u16*)&ptr[2];
  5425   5427       ptr += 2;
  5426   5428     }
  5427   5429     pPage->nCell--;
  5428   5430     put2byte(&data[hdr+3], pPage->nCell);
  5429   5431     pPage->nFree += 2;
  5430   5432   }
  5431   5433   
................................................................................
  5458   5460     int idx = 0;      /* Where to write new cell content in data[] */
  5459   5461     int j;            /* Loop counter */
  5460   5462     int end;          /* First byte past the last cell pointer in data[] */
  5461   5463     int ins;          /* Index in data[] where new cell pointer is inserted */
  5462   5464     int cellOffset;   /* Address of first cell pointer in data[] */
  5463   5465     u8 *data;         /* The content of the whole page */
  5464   5466     u8 *ptr;          /* Used for moving information around in data[] */
         5467  +  u8 *endPtr;       /* End of the loop */
  5465   5468   
  5466   5469     int nSkip = (iChild ? 4 : 0);
  5467   5470   
  5468   5471     if( *pRC ) return;
  5469   5472   
  5470   5473     assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
  5471   5474     assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 );
................................................................................
  5508   5511       assert( idx+sz <= (int)pPage->pBt->usableSize );
  5509   5512       pPage->nCell++;
  5510   5513       pPage->nFree -= (u16)(2 + sz);
  5511   5514       memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip);
  5512   5515       if( iChild ){
  5513   5516         put4byte(&data[idx], iChild);
  5514   5517       }
  5515         -    for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){
  5516         -      ptr[0] = ptr[-2];
  5517         -      ptr[1] = ptr[-1];
         5518  +    ptr = &data[end];
         5519  +    endPtr = &data[ins];
         5520  +    assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 );  /* ptr is always 2-byte aligned */
         5521  +    while( ptr>endPtr ){
         5522  +      *(u16*)ptr = *(u16*)&ptr[-2];
         5523  +      ptr -= 2;
  5518   5524       }
  5519   5525       put2byte(&data[ins], idx);
  5520   5526       put2byte(&data[pPage->hdrOffset+3], pPage->nCell);
  5521   5527   #ifndef SQLITE_OMIT_AUTOVACUUM
  5522   5528       if( pPage->pBt->autoVacuum ){
  5523   5529         /* The cell may contain a pointer to an overflow page. If so, write
  5524   5530         ** the entry for the overflow page into the pointer map.
................................................................................
  5555   5561     /* Check that the page has just been zeroed by zeroPage() */
  5556   5562     assert( pPage->nCell==0 );
  5557   5563     assert( get2byteNotZero(&data[hdr+5])==nUsable );
  5558   5564   
  5559   5565     pCellptr = &data[pPage->cellOffset + nCell*2];
  5560   5566     cellbody = nUsable;
  5561   5567     for(i=nCell-1; i>=0; i--){
         5568  +    u16 sz = aSize[i];
  5562   5569       pCellptr -= 2;
  5563         -    cellbody -= aSize[i];
         5570  +    cellbody -= sz;
  5564   5571       put2byte(pCellptr, cellbody);
  5565         -    memcpy(&data[cellbody], apCell[i], aSize[i]);
         5572  +    memcpy(&data[cellbody], apCell[i], sz);
  5566   5573     }
  5567   5574     put2byte(&data[hdr+3], nCell);
  5568   5575     put2byte(&data[hdr+5], cellbody);
  5569   5576     pPage->nFree -= (nCell*2 + nUsable - cellbody);
  5570   5577     pPage->nCell = (u16)nCell;
  5571   5578   }
  5572   5579   
................................................................................
  6012   6019       ** process of being overwritten.  */
  6013   6020       MemPage *pOld = apCopy[i] = (MemPage*)&aSpace1[pBt->pageSize + k*i];
  6014   6021       memcpy(pOld, apOld[i], sizeof(MemPage));
  6015   6022       pOld->aData = (void*)&pOld[1];
  6016   6023       memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize);
  6017   6024   
  6018   6025       limit = pOld->nCell+pOld->nOverflow;
  6019         -    for(j=0; j<limit; j++){
  6020         -      assert( nCell<nMaxCells );
  6021         -      apCell[nCell] = findOverflowCell(pOld, j);
  6022         -      szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
  6023         -      nCell++;
  6024         -    }
         6026  +    if( pOld->nOverflow>0 ){
         6027  +      for(j=0; j<limit; j++){
         6028  +        assert( nCell<nMaxCells );
         6029  +        apCell[nCell] = findOverflowCell(pOld, j);
         6030  +        szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
         6031  +        nCell++;
         6032  +      }
         6033  +    }else{
         6034  +      u8 *aData = pOld->aData;
         6035  +      u16 maskPage = pOld->maskPage;
         6036  +      u16 cellOffset = pOld->cellOffset;
         6037  +      for(j=0; j<limit; j++){
         6038  +        assert( nCell<nMaxCells );
         6039  +        apCell[nCell] = findCellv2(aData, maskPage, cellOffset, j);
         6040  +        szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
         6041  +        nCell++;
         6042  +      }
         6043  +    }       
  6025   6044       if( i<nOld-1 && !leafData){
  6026   6045         u16 sz = (u16)szNew[i];
  6027   6046         u8 *pTemp;
  6028   6047         assert( nCell<nMaxCells );
  6029   6048         szCell[nCell] = sz;
  6030   6049         pTemp = &aSpace1[iSpace1];
  6031   6050         iSpace1 += sz;

Changes to src/build.c.

  1615   1615             pDb->zName
  1616   1616           );
  1617   1617         }
  1618   1618       }
  1619   1619   #endif
  1620   1620   
  1621   1621       /* Reparse everything to update our internal data structures */
  1622         -    sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
  1623         -        sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC);
         1622  +    sqlite3VdbeAddParseSchemaOp(v, iDb,
         1623  +               sqlite3MPrintf(db, "tbl_name='%q'", p->zName));
  1624   1624     }
  1625   1625   
  1626   1626   
  1627   1627     /* Add the table to the in-memory representation of the database.
  1628   1628     */
  1629   1629     if( db->init.busy ){
  1630   1630       Table *pOld;
................................................................................
  2813   2813   
  2814   2814       /* Fill the index with data and reparse the schema. Code an OP_Expire
  2815   2815       ** to invalidate all pre-compiled statements.
  2816   2816       */
  2817   2817       if( pTblName ){
  2818   2818         sqlite3RefillIndex(pParse, pIndex, iMem);
  2819   2819         sqlite3ChangeCookie(pParse, iDb);
  2820         -      sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
  2821         -         sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), 
  2822         -         P4_DYNAMIC);
         2820  +      sqlite3VdbeAddParseSchemaOp(v, iDb,
         2821  +         sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName));
  2823   2822         sqlite3VdbeAddOp1(v, OP_Expire, 0);
  2824   2823       }
  2825   2824     }
  2826   2825   
  2827   2826     /* When adding an index to the list of indices for a table, make
  2828   2827     ** sure all indices labeled OE_Replace come after all those labeled
  2829   2828     ** OE_Ignore.  This is necessary for the correct constraint check

Changes to src/fkey.c.

   382    382         for(i=0; i<nCol; i++){
   383    383           sqlite3VdbeAddOp2(v, OP_Copy, aiCol[i]+1+regData, regTemp+i);
   384    384         }
   385    385     
   386    386         /* If the parent table is the same as the child table, and we are about
   387    387         ** to increment the constraint-counter (i.e. this is an INSERT operation),
   388    388         ** then check if the row being inserted matches itself. If so, do not
   389         -      ** increment the constraint-counter.  */
          389  +      ** increment the constraint-counter. 
          390  +      **
          391  +      ** If any of the parent-key values are NULL, then the row cannot match 
          392  +      ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any
          393  +      ** of the parent-key values are NULL (at this point it is known that
          394  +      ** none of the child key values are).
          395  +      */
   390    396         if( pTab==pFKey->pFrom && nIncr==1 ){
   391    397           int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1;
   392    398           for(i=0; i<nCol; i++){
   393    399             int iChild = aiCol[i]+1+regData;
   394    400             int iParent = pIdx->aiColumn[i]+1+regData;
          401  +          assert( aiCol[i]!=pTab->iPKey );
          402  +          if( pIdx->aiColumn[i]==pTab->iPKey ){
          403  +            /* The parent key is a composite key that includes the IPK column */
          404  +            iParent = regData;
          405  +          }
   395    406             sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent);
          407  +          sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
   396    408           }
   397    409           sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
   398    410         }
   399    411     
   400    412         sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
   401    413         sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT);
   402    414         sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0);

Changes to src/func.c.

   505    505   /*
   506    506   ** For LIKE and GLOB matching on EBCDIC machines, assume that every
   507    507   ** character is exactly one byte in size.  Also, all characters are
   508    508   ** able to participate in upper-case-to-lower-case mappings in EBCDIC
   509    509   ** whereas only characters less than 0x80 do in ASCII.
   510    510   */
   511    511   #if defined(SQLITE_EBCDIC)
   512         -# define sqlite3Utf8Read(A,C)    (*(A++))
   513         -# define GlogUpperToLower(A)     A = sqlite3UpperToLower[A]
          512  +# define sqlite3Utf8Read(A,C)  (*(A++))
          513  +# define GlogUpperToLower(A)   A = sqlite3UpperToLower[A]
   514    514   #else
   515         -# define GlogUpperToLower(A)     if( A<0x80 ){ A = sqlite3UpperToLower[A]; }
          515  +# define GlogUpperToLower(A)   if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; }
   516    516   #endif
   517    517   
   518    518   static const struct compareInfo globInfo = { '*', '?', '[', 0 };
   519    519   /* The correct SQL-92 behavior is for the LIKE operator to ignore
   520    520   ** case.  Thus  'a' LIKE 'A' would be true. */
   521    521   static const struct compareInfo likeInfoNorm = { '%', '_',   0, 1 };
   522    522   /* If SQLITE_CASE_SENSITIVE_LIKE is defined, then the LIKE operator
................................................................................
   551    551   **
   552    552   **         abc[*]xyz        Matches "abc*xyz" only
   553    553   */
   554    554   static int patternCompare(
   555    555     const u8 *zPattern,              /* The glob pattern */
   556    556     const u8 *zString,               /* The string to compare against the glob */
   557    557     const struct compareInfo *pInfo, /* Information about how to do the compare */
   558         -  const int esc                    /* The escape character */
          558  +  u32 esc                          /* The escape character */
   559    559   ){
   560         -  int c, c2;
          560  +  u32 c, c2;
   561    561     int invert;
   562    562     int seen;
   563    563     u8 matchOne = pInfo->matchOne;
   564    564     u8 matchAll = pInfo->matchAll;
   565    565     u8 matchSet = pInfo->matchSet;
   566    566     u8 noCase = pInfo->noCase; 
   567    567     int prevEscape = 0;     /* True if the previous character was 'escape' */
................................................................................
   607    607         }
   608    608         return 0;
   609    609       }else if( !prevEscape && c==matchOne ){
   610    610         if( sqlite3Utf8Read(zString, &zString)==0 ){
   611    611           return 0;
   612    612         }
   613    613       }else if( c==matchSet ){
   614         -      int prior_c = 0;
          614  +      u32 prior_c = 0;
   615    615         assert( esc==0 );    /* This only occurs for GLOB, not LIKE */
   616    616         seen = 0;
   617    617         invert = 0;
   618    618         c = sqlite3Utf8Read(zString, &zString);
   619    619         if( c==0 ) return 0;
   620    620         c2 = sqlite3Utf8Read(zPattern, &zPattern);
   621    621         if( c2=='^' ){
................................................................................
   683    683   */
   684    684   static void likeFunc(
   685    685     sqlite3_context *context, 
   686    686     int argc, 
   687    687     sqlite3_value **argv
   688    688   ){
   689    689     const unsigned char *zA, *zB;
   690         -  int escape = 0;
          690  +  u32 escape = 0;
   691    691     int nPat;
   692    692     sqlite3 *db = sqlite3_context_db_handle(context);
   693    693   
   694    694     zB = sqlite3_value_text(argv[0]);
   695    695     zA = sqlite3_value_text(argv[1]);
   696    696   
   697    697     /* Limit the length of the LIKE or GLOB pattern to avoid problems

Changes to src/main.c.

  1947   1947         if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){
  1948   1948           zVfs = zVal;
  1949   1949         }else{
  1950   1950           struct OpenMode {
  1951   1951             const char *z;
  1952   1952             int mode;
  1953   1953           } *aMode = 0;
  1954         -        char *zModeType;
  1955         -        int mask;
  1956         -        int limit;
         1954  +        char *zModeType = 0;
         1955  +        int mask = 0;
         1956  +        int limit = 0;
  1957   1957   
  1958   1958           if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){
  1959   1959             static struct OpenMode aCacheMode[] = {
  1960   1960               { "shared",  SQLITE_OPEN_SHAREDCACHE },
  1961   1961               { "private", SQLITE_OPEN_PRIVATECACHE },
  1962   1962               { 0, 0 }
  1963   1963             };

Changes to src/pcache1.c.

   570    570     pCache = (PCache1 *)sqlite3_malloc(sz);
   571    571     if( pCache ){
   572    572       memset(pCache, 0, sz);
   573    573       if( separateCache ){
   574    574         pGroup = (PGroup*)&pCache[1];
   575    575         pGroup->mxPinned = 10;
   576    576       }else{
   577         -      pGroup = &pcache1_g.grp;
          577  +      pGroup = &pcache1.grp;
   578    578       }
   579    579       pCache->pGroup = pGroup;
   580    580       pCache->szPage = szPage;
   581    581       pCache->bPurgeable = (bPurgeable ? 1 : 0);
   582    582       if( bPurgeable ){
   583    583         pCache->nMin = 10;
   584    584         pcache1EnterMutex(pGroup);

Changes to src/pragma.c.

     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** This file contains code used to implement the PRAGMA command.
    13     13   */
    14     14   #include "sqliteInt.h"
    15     15   
    16         -/* Ignore this whole file if pragmas are disabled
    17         -*/
    18         -#if !defined(SQLITE_OMIT_PRAGMA)
    19         -
    20     16   /*
    21     17   ** Interpret the given string as a safety level.  Return 0 for OFF,
    22     18   ** 1 for ON or NORMAL and 2 for FULL.  Return 1 for an empty or 
    23     19   ** unrecognized string argument.
    24     20   **
    25     21   ** Note that the values returned are one less that the values that
    26     22   ** should be passed into sqlite3BtreeSetSafetyLevel().  The is done
................................................................................
    48     44   
    49     45   /*
    50     46   ** Interpret the given string as a boolean value.
    51     47   */
    52     48   u8 sqlite3GetBoolean(const char *z){
    53     49     return getSafetyLevel(z)&1;
    54     50   }
           51  +
           52  +/* The sqlite3GetBoolean() function is used by other modules but the
           53  +** remainder of this file is specific to PRAGMA processing.  So omit
           54  +** the rest of the file if PRAGMAs are omitted from the build.
           55  +*/
           56  +#if !defined(SQLITE_OMIT_PRAGMA)
    55     57   
    56     58   /*
    57     59   ** Interpret the given string as a locking mode value.
    58     60   */
    59     61   static int getLockingMode(const char *z){
    60     62     if( z ){
    61     63       if( 0==sqlite3StrICmp(z, "exclusive") ) return PAGER_LOCKINGMODE_EXCLUSIVE;

Changes to src/shell.c.

  2298   2298       
  2299   2299     if( HAS_TIMER && c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0
  2300   2300      && nArg==2
  2301   2301     ){
  2302   2302       enableTimer = booleanValue(azArg[1]);
  2303   2303     }else
  2304   2304     
         2305  +  if( c=='v' && strncmp(azArg[0], "version", n)==0 ){
         2306  +    printf("SQLite %s %s\n",
         2307  +        sqlite3_libversion(), sqlite3_sourceid());
         2308  +  }else
         2309  +
  2305   2310     if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){
  2306   2311       int j;
  2307   2312       assert( nArg<=ArraySize(azArg) );
  2308   2313       for(j=1; j<nArg && j<ArraySize(p->colWidth); j++){
  2309   2314         p->colWidth[j-1] = atoi(azArg[j]);
  2310   2315       }
  2311   2316     }else
................................................................................
  2832   2837       }else if( strcmp(z,"-echo")==0 ){
  2833   2838         data.echoOn = 1;
  2834   2839       }else if( strcmp(z,"-stats")==0 ){
  2835   2840         data.statsOn = 1;
  2836   2841       }else if( strcmp(z,"-bail")==0 ){
  2837   2842         bail_on_error = 1;
  2838   2843       }else if( strcmp(z,"-version")==0 ){
  2839         -      printf("%s\n", sqlite3_libversion());
         2844  +      printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid());
  2840   2845         return 0;
  2841   2846       }else if( strcmp(z,"-interactive")==0 ){
  2842   2847         stdin_is_interactive = 1;
  2843   2848       }else if( strcmp(z,"-batch")==0 ){
  2844   2849         stdin_is_interactive = 0;
  2845   2850       }else if( strcmp(z,"-heap")==0 ){
  2846   2851         i++;
................................................................................
  2877   2882       /* Run commands received from standard input
  2878   2883       */
  2879   2884       if( stdin_is_interactive ){
  2880   2885         char *zHome;
  2881   2886         char *zHistory = 0;
  2882   2887         int nHistory;
  2883   2888         printf(
  2884         -        "SQLite version %s\n"
         2889  +        "SQLite version %s %.19s\n"
  2885   2890           "Enter \".help\" for instructions\n"
  2886   2891           "Enter SQL statements terminated with a \";\"\n",
  2887         -        sqlite3_libversion()
         2892  +        sqlite3_libversion(), sqlite3_sourceid()
  2888   2893         );
  2889   2894         zHome = find_home_dir();
  2890   2895         if( zHome ){
  2891   2896           nHistory = strlen30(zHome) + 20;
  2892   2897           if( (zHistory = malloc(nHistory))!=0 ){
  2893   2898             sqlite3_snprintf(nHistory, zHistory,"%s/.sqlite_history", zHome);
  2894   2899           }

Changes to src/sqliteInt.h.

   678    678   ** Schema objects are automatically deallocated when the last Btree that
   679    679   ** references them is destroyed.   The TEMP Schema is manually freed by
   680    680   ** sqlite3_close().
   681    681   *
   682    682   ** A thread must be holding a mutex on the corresponding Btree in order
   683    683   ** to access Schema content.  This implies that the thread must also be
   684    684   ** holding a mutex on the sqlite3 connection pointer that owns the Btree.
   685         -** For a TEMP Schema, on the connection mutex is required.
          685  +** For a TEMP Schema, only the connection mutex is required.
   686    686   */
   687    687   struct Schema {
   688    688     int schema_cookie;   /* Database schema version number for this file */
   689    689     int iGeneration;     /* Generation counter.  Incremented with each change */
   690    690     Hash tblHash;        /* All tables indexed by name */
   691    691     Hash idxHash;        /* All (named) indices indexed by name */
   692    692     Hash trigHash;       /* All triggers indexed by name */
................................................................................
  2875   2875   int sqlite3FixExprList(DbFixer*, ExprList*);
  2876   2876   int sqlite3FixTriggerStep(DbFixer*, TriggerStep*);
  2877   2877   int sqlite3AtoF(const char *z, double*, int, u8);
  2878   2878   int sqlite3GetInt32(const char *, int*);
  2879   2879   int sqlite3Atoi(const char*);
  2880   2880   int sqlite3Utf16ByteLen(const void *pData, int nChar);
  2881   2881   int sqlite3Utf8CharLen(const char *pData, int nByte);
  2882         -int sqlite3Utf8Read(const u8*, const u8**);
         2882  +u32 sqlite3Utf8Read(const u8*, const u8**);
  2883   2883   
  2884   2884   /*
  2885   2885   ** Routines to read and write variable-length integers.  These used to
  2886   2886   ** be defined locally, but now we use the varint routines in the util.c
  2887   2887   ** file.  Code should use the MACRO forms below, as the Varint32 versions
  2888   2888   ** are coded to assume the single byte case is already handled (which 
  2889   2889   ** the MACRO form does).

Changes to src/tclsqlite.c.

  3580   3580       extern int Sqlitetestrtree_Init(Tcl_Interp*);
  3581   3581       extern int Sqlitequota_Init(Tcl_Interp*);
  3582   3582       extern int Sqlitemultiplex_Init(Tcl_Interp*);
  3583   3583       extern int SqliteSuperlock_Init(Tcl_Interp*);
  3584   3584       extern int SqlitetestSyscall_Init(Tcl_Interp*);
  3585   3585       extern int Sqlitetestfuzzer_Init(Tcl_Interp*);
  3586   3586       extern int Sqlitetestwholenumber_Init(Tcl_Interp*);
         3587  +
         3588  +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
         3589  +    extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
         3590  +#endif
  3587   3591   
  3588   3592   #ifdef SQLITE_ENABLE_ZIPVFS
  3589   3593       extern int Zipvfs_Init(Tcl_Interp*);
  3590   3594       Zipvfs_Init(interp);
  3591   3595   #endif
  3592   3596   
  3593   3597       Sqliteconfig_Init(interp);
................................................................................
  3620   3624       Sqlitetestrtree_Init(interp);
  3621   3625       Sqlitequota_Init(interp);
  3622   3626       Sqlitemultiplex_Init(interp);
  3623   3627       SqliteSuperlock_Init(interp);
  3624   3628       SqlitetestSyscall_Init(interp);
  3625   3629       Sqlitetestfuzzer_Init(interp);
  3626   3630       Sqlitetestwholenumber_Init(interp);
         3631  +
         3632  +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
         3633  +    Sqlitetestfts3_Init(interp);
         3634  +#endif
  3627   3635   
  3628   3636       Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0);
  3629   3637   
  3630   3638   #ifdef SQLITE_SSE
  3631   3639       Sqlitetestsse_Init(interp);
  3632   3640   #endif
  3633   3641     }

Changes to src/test_quota.c.

   319    319       pSubOpen = quotaSubOpen(pConn);
   320    320       rc = pOrigVfs->xOpen(pOrigVfs, zName, pSubOpen, flags, pOutFlags);
   321    321       if( rc==SQLITE_OK ){
   322    322         for(pFile=pGroup->pFiles; pFile && strcmp(pFile->zFilename, zName);
   323    323             pFile=pFile->pNext){}
   324    324         if( pFile==0 ){
   325    325           int nName = strlen(zName);
   326         -        pFile = sqlite3_malloc( sizeof(*pFile) + nName + 1 );
          326  +        pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 );
   327    327           if( pFile==0 ){
   328    328             quotaLeave();
   329    329             pSubOpen->pMethods->xClose(pSubOpen);
   330    330             return SQLITE_NOMEM;
   331    331           }
   332    332           memset(pFile, 0, sizeof(*pFile));
   333    333           pFile->zFilename = (char*)&pFile[1];
................................................................................
   679    679     }
   680    680     if( pGroup==0 ){
   681    681       int nPattern = strlen(zPattern);
   682    682       if( iLimit<=0 ){
   683    683         quotaLeave();
   684    684         return SQLITE_OK;
   685    685       }
   686         -    pGroup = sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 );
          686  +    pGroup = (quotaGroup *)sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 );
   687    687       if( pGroup==0 ){
   688    688         quotaLeave();
   689    689         return SQLITE_NOMEM;
   690    690       }
   691    691       memset(pGroup, 0, sizeof(*pGroup));
   692    692       pGroup->zPattern = (char*)&pGroup[1];
   693    693       memcpy((char *)pGroup->zPattern, zPattern, nPattern+1);

Changes to src/tokenize.c.

   349    349         return i;
   350    350       }
   351    351   #ifndef SQLITE_OMIT_BLOB_LITERAL
   352    352       case 'x': case 'X': {
   353    353         testcase( z[0]=='x' ); testcase( z[0]=='X' );
   354    354         if( z[1]=='\'' ){
   355    355           *tokenType = TK_BLOB;
   356         -        for(i=2; (c=z[i])!=0 && c!='\''; i++){
   357         -          if( !sqlite3Isxdigit(c) ){
   358         -            *tokenType = TK_ILLEGAL;
   359         -          }
          356  +        for(i=2; sqlite3Isxdigit(z[i]); i++){}
          357  +        if( z[i]!='\'' || i%2 ){
          358  +          *tokenType = TK_ILLEGAL;
          359  +          while( z[i] && z[i]!='\'' ){ i++; }
   360    360           }
   361         -        if( i%2 || !c ) *tokenType = TK_ILLEGAL;
   362         -        if( c ) i++;
          361  +        if( z[i] ) i++;
   363    362           return i;
   364    363         }
   365    364         /* Otherwise fall through to the next case */
   366    365       }
   367    366   #endif
   368    367       default: {
   369    368         if( !IdChar(*z) ){

Changes to src/trigger.c.

   297    297       z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n);
   298    298       sqlite3NestedParse(pParse,
   299    299          "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
   300    300          db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName,
   301    301          pTrig->table, z);
   302    302       sqlite3DbFree(db, z);
   303    303       sqlite3ChangeCookie(pParse, iDb);
   304         -    sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf(
   305         -        db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC
   306         -    );
          304  +    sqlite3VdbeAddParseSchemaOp(v, iDb,
          305  +        sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName));
   307    306     }
   308    307   
   309    308     if( db->init.busy ){
   310    309       Trigger *pLink = pTrig;
   311    310       Hash *pHash = &db->aDb[iDb].pSchema->trigHash;
   312    311       assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
   313    312       pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig);

Changes to src/update.c.

   240    240     for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){}
   241    241     if( nIdx>0 ){
   242    242       aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx );
   243    243       if( aRegIdx==0 ) goto update_cleanup;
   244    244     }
   245    245     for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
   246    246       int reg;
   247         -    if( chngRowid ){
          247  +    if( hasFK || chngRowid ){
   248    248         reg = ++pParse->nMem;
   249    249       }else{
   250    250         reg = 0;
   251    251         for(i=0; i<pIdx->nColumn; i++){
   252    252           if( aXRef[pIdx->aiColumn[i]]>=0 ){
   253    253             reg = ++pParse->nMem;
   254    254             break;

Changes to src/utf.c.

   159    159       while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){            \
   160    160         c = (c<<6) + (0x3f & *(zIn++));                      \
   161    161       }                                                      \
   162    162       if( c<0x80                                             \
   163    163           || (c&0xFFFFF800)==0xD800                          \
   164    164           || (c&0xFFFFFFFE)==0xFFFE ){  c = 0xFFFD; }        \
   165    165     }
   166         -int sqlite3Utf8Read(
          166  +u32 sqlite3Utf8Read(
   167    167     const unsigned char *zIn,       /* First byte of UTF-8 character */
   168    168     const unsigned char **pzNext    /* Write first byte past UTF-8 char here */
   169    169   ){
   170    170     unsigned int c;
   171    171   
   172    172     /* Same as READ_UTF8() above but without the zTerm parameter.
   173    173     ** For this routine, we assume the UTF8 string is always zero-terminated.

Changes to src/vdbe.c.

  1395   1395       assert( pOp[-1].p4type==P4_COLLSEQ );
  1396   1396       assert( pOp[-1].opcode==OP_CollSeq );
  1397   1397       ctx.pColl = pOp[-1].p4.pColl;
  1398   1398     }
  1399   1399     db->lastRowid = lastRowid;
  1400   1400     (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */
  1401   1401     lastRowid = db->lastRowid;
         1402  +
         1403  +  /* If any auxiliary data functions have been called by this user function,
         1404  +  ** immediately call the destructor for any non-static values.
         1405  +  */
         1406  +  if( ctx.pVdbeFunc ){
         1407  +    sqlite3VdbeDeleteAuxData(ctx.pVdbeFunc, pOp->p1);
         1408  +    pOp->p4.pVdbeFunc = ctx.pVdbeFunc;
         1409  +    pOp->p4type = P4_VDBEFUNC;
         1410  +  }
         1411  +
  1402   1412     if( db->mallocFailed ){
  1403   1413       /* Even though a malloc() has failed, the implementation of the
  1404   1414       ** user function may have called an sqlite3_result_XXX() function
  1405   1415       ** to return a value. The following call releases any resources
  1406   1416       ** associated with such a value.
  1407   1417       */
  1408   1418       sqlite3VdbeMemRelease(&ctx.s);
  1409   1419       goto no_mem;
  1410   1420     }
  1411   1421   
  1412         -  /* If any auxiliary data functions have been called by this user function,
  1413         -  ** immediately call the destructor for any non-static values.
  1414         -  */
  1415         -  if( ctx.pVdbeFunc ){
  1416         -    sqlite3VdbeDeleteAuxData(ctx.pVdbeFunc, pOp->p1);
  1417         -    pOp->p4.pVdbeFunc = ctx.pVdbeFunc;
  1418         -    pOp->p4type = P4_VDBEFUNC;
  1419         -  }
  1420         -
  1421   1422     /* If the function returned an error, throw an exception */
  1422   1423     if( ctx.isError ){
  1423   1424       sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s));
  1424   1425       rc = ctx.isError;
  1425   1426     }
  1426   1427   
  1427   1428     /* Copy the result of the function into register P3 */
................................................................................
  2581   2582         */
  2582   2583         sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - "
  2583   2584           "SQL statements in progress");
  2584   2585         rc = SQLITE_BUSY;
  2585   2586       }else{
  2586   2587         nName = sqlite3Strlen30(zName);
  2587   2588   
  2588         -#ifndef SQLITE_OMIT_VIRTUAL_TABLE
         2589  +#ifndef SQLITE_OMIT_VIRTUALTABLE
  2589   2590         /* This call is Ok even if this savepoint is actually a transaction
  2590   2591         ** savepoint (and therefore should not prompt xSavepoint()) callbacks.
  2591   2592         ** If this is a transaction savepoint being opened, it is guaranteed
  2592   2593         ** that the db->aVTrans[] array is empty.  */
  2593   2594         assert( db->autoCommit==0 || db->nVTrans==0 );
  2594   2595         rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN,
  2595   2596                                   db->nStatement+db->nSavepoint);

Changes to src/vdbe.h.

   168    168   int sqlite3VdbeAddOp0(Vdbe*,int);
   169    169   int sqlite3VdbeAddOp1(Vdbe*,int,int);
   170    170   int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
   171    171   int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
   172    172   int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
   173    173   int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
   174    174   int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp);
          175  +void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
   175    176   void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
   176    177   void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
   177    178   void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
   178    179   void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
   179    180   void sqlite3VdbeJumpHere(Vdbe*, int addr);
   180    181   void sqlite3VdbeChangeToNoop(Vdbe*, int addr, int N);
   181    182   void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N);

Changes to src/vdbeaux.c.

   152    152     pOp->opcode = (u8)op;
   153    153     pOp->p5 = 0;
   154    154     pOp->p1 = p1;
   155    155     pOp->p2 = p2;
   156    156     pOp->p3 = p3;
   157    157     pOp->p4.p = 0;
   158    158     pOp->p4type = P4_NOTUSED;
   159         -  p->expired = 0;
   160         -  if( op==OP_ParseSchema ){
   161         -    /* Any program that uses the OP_ParseSchema opcode needs to lock
   162         -    ** all btrees. */
   163         -    int j;
   164         -    for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
   165         -  }
   166    159   #ifdef SQLITE_DEBUG
   167    160     pOp->zComment = 0;
   168    161     if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]);
   169    162   #endif
   170    163   #ifdef VDBE_PROFILE
   171    164     pOp->cycles = 0;
   172    165     pOp->cnt = 0;
................................................................................
   196    189     const char *zP4,    /* The P4 operand */
   197    190     int p4type          /* P4 operand type */
   198    191   ){
   199    192     int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
   200    193     sqlite3VdbeChangeP4(p, addr, zP4, p4type);
   201    194     return addr;
   202    195   }
          196  +
          197  +/*
          198  +** Add an OP_ParseSchema opcode.  This routine is broken out from
          199  +** sqlite3VdbeAddOp4() since it needs to also local all btrees.
          200  +**
          201  +** The zWhere string must have been obtained from sqlite3_malloc().
          202  +** This routine will take ownership of the allocated memory.
          203  +*/
          204  +void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){
          205  +  int j;
          206  +  int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0);
          207  +  sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC);
          208  +  for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
          209  +}
   203    210   
   204    211   /*
   205    212   ** Add an opcode that includes the p4 value as an integer.
   206    213   */
   207    214   int sqlite3VdbeAddOp4Int(
   208    215     Vdbe *p,            /* Add the opcode to this VM */
   209    216     int op,             /* The new opcode */

Changes to src/vtab.c.

   379    379       );
   380    380       sqlite3DbFree(db, zStmt);
   381    381       v = sqlite3GetVdbe(pParse);
   382    382       sqlite3ChangeCookie(pParse, iDb);
   383    383   
   384    384       sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
   385    385       zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
   386         -    sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
          386  +    sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
   387    387       sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, 
   388    388                            pTab->zName, sqlite3Strlen30(pTab->zName) + 1);
   389    389     }
   390    390   
   391    391     /* If we are rereading the sqlite_master table create the in-memory
   392    392     ** record of the table. The xConnect() method is not called until
   393    393     ** the first time the virtual table is used in an SQL statement. This

Changes to src/wal.c.

  2148   2148             break;
  2149   2149           }else if( rc!=SQLITE_BUSY ){
  2150   2150             return rc;
  2151   2151           }
  2152   2152         }
  2153   2153       }
  2154   2154       if( mxI==0 ){
  2155         -      assert( rc==SQLITE_BUSY );
         2155  +      assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 );
  2156   2156         return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK;
  2157   2157       }
  2158   2158   
  2159   2159       rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
  2160   2160       if( rc ){
  2161   2161         return rc==SQLITE_BUSY ? WAL_RETRY : rc;
  2162   2162       }

Changes to test/fkey3.test.

    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   
    19     19   ifcapable {!foreignkey||!trigger} {
    20     20     finish_test
    21     21     return
    22     22   }
           23  +
           24  +set testprefix fkey3
    23     25   
    24     26   # Create a table and some data to work with.
    25     27   #
    26     28   do_test fkey3-1.1 {
    27     29     execsql {
    28     30       PRAGMA foreign_keys=ON;
    29     31       CREATE TABLE t1(x INTEGER PRIMARY KEY);
................................................................................
    73     75       INSERT INTO t2 VALUES(100);
    74     76       INSERT INTO t2 VALUES(101);
    75     77       SELECT 1, x FROM t1;
    76     78       SELECT 2, y FROM t2;
    77     79     }
    78     80   } {1 100 1 101 2 100 2 101}
    79     81   
           82  +
           83  +#-------------------------------------------------------------------------
           84  +# The following tests - fkey-3.* - test some edge cases to do with 
           85  +# inserting rows into tables that have foreign keys where the parent
           86  +# table is the same as the child table. Especially cases where the
           87  +# new row being inserted matches itself.
           88  +#
           89  +do_execsql_test 3.1.1 {
           90  +  CREATE TABLE t3(a, b, c, d, 
           91  +    UNIQUE(a, b),
           92  +    FOREIGN KEY(c, d) REFERENCES t3(a, b)
           93  +  );
           94  +  INSERT INTO t3 VALUES(1, 2, 1, 2);
           95  +} {}
           96  +do_catchsql_test 3.1.2 {
           97  +  INSERT INTO t3 VALUES(NULL, 2, 5, 2);
           98  +} {1 {foreign key constraint failed}}
           99  +do_catchsql_test 3.1.3 {
          100  +  INSERT INTO t3 VALUES(NULL, 3, 5, 2);
          101  +} {1 {foreign key constraint failed}}
          102  +
          103  +do_execsql_test 3.2.1 {
          104  +  CREATE TABLE t4(a UNIQUE, b REFERENCES t4(a));
          105  +}
          106  +do_catchsql_test 3.2.2 {
          107  +  INSERT INTO t4 VALUES(NULL, 1);
          108  +} {1 {foreign key constraint failed}}
          109  +
          110  +do_execsql_test 3.3.1 {
          111  +  CREATE TABLE t5(a INTEGER PRIMARY KEY, b REFERENCES t5(a));
          112  +  INSERT INTO t5 VALUES(NULL, 1);
          113  +} {}
          114  +do_catchsql_test 3.3.2 {
          115  +  INSERT INTO t5 VALUES(NULL, 3);
          116  +} {1 {foreign key constraint failed}}
          117  +
          118  +do_execsql_test 3.4.1 {
          119  +  CREATE TABLE t6(a INTEGER PRIMARY KEY, b, c, d,
          120  +    FOREIGN KEY(c, d) REFERENCES t6(a, b)
          121  +  );
          122  +  CREATE UNIQUE INDEX t6i ON t6(b, a);
          123  +}
          124  +do_execsql_test 3.4.2  { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {}
          125  +do_execsql_test 3.4.3  { INSERT INTO t6 VALUES(2, 'a', 2, 'a');    } {}
          126  +do_execsql_test 3.4.4  { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {}
          127  +do_execsql_test 3.4.5  { INSERT INTO t6 VALUES(5, 'a', 2, 'a'); } {}
          128  +do_catchsql_test 3.4.6 { 
          129  +  INSERT INTO t6 VALUES(NULL, 'a', 65, 'a');    
          130  +} {1 {foreign key constraint failed}}
          131  +
          132  +do_execsql_test 3.4.7 {
          133  +  INSERT INTO t6 VALUES(100, 'one', 100, 'one');
          134  +  DELETE FROM t6 WHERE a = 100;
          135  +}
          136  +do_execsql_test 3.4.8 {
          137  +  INSERT INTO t6 VALUES(100, 'one', 100, 'one');
          138  +  UPDATE t6 SET c = 1, d = 'a' WHERE a = 100;
          139  +  DELETE FROM t6 WHERE a = 100;
          140  +}
          141  +
          142  +do_execsql_test 3.5.1 {
          143  +  CREATE TABLE t7(a, b, c, d INTEGER PRIMARY KEY,
          144  +    FOREIGN KEY(c, d) REFERENCES t7(a, b)
          145  +  );
          146  +  CREATE UNIQUE INDEX t7i ON t7(a, b);
          147  +}
          148  +do_execsql_test 3.5.2  { INSERT INTO t7 VALUES('x', 1, 'x', NULL) } {}
          149  +do_execsql_test 3.5.3  { INSERT INTO t7 VALUES('x', 2, 'x', 2) } {}
          150  +do_catchsql_test 3.5.4  { 
          151  +  INSERT INTO t7 VALUES('x', 450, 'x', NULL);
          152  +} {1 {foreign key constraint failed}}
          153  +do_catchsql_test 3.5.5  { 
          154  +  INSERT INTO t7 VALUES('x', 450, 'x', 451);
          155  +} {1 {foreign key constraint failed}}
          156  +
          157  +
          158  +do_execsql_test 3.6.1 {
          159  +  CREATE TABLE t8(a, b, c, d, e, FOREIGN KEY(c, d) REFERENCES t8(a, b));
          160  +  CREATE UNIQUE INDEX t8i1 ON t8(a, b);
          161  +  CREATE UNIQUE INDEX t8i2 ON t8(c);
          162  +  INSERT INTO t8 VALUES(1, 1, 1, 1, 1);
          163  +}
          164  +do_catchsql_test 3.6.2 { 
          165  +  UPDATE t8 SET d = 2; 
          166  +} {1 {foreign key constraint failed}}
          167  +do_execsql_test 3.6.3 { UPDATE t8 SET d = 1; }
          168  +do_execsql_test 3.6.4 { UPDATE t8 SET e = 2; }
          169  +
          170  +do_catchsql_test 3.6.5 {
          171  +  CREATE TABLE TestTable (
          172  +    id INTEGER PRIMARY KEY,
          173  +    name text,
          174  +    source_id integer not null,
          175  +    parent_id integer,
          176  +
          177  +    foreign key(source_id, parent_id) references TestTable(source_id, id)
          178  +  );
          179  +  CREATE UNIQUE INDEX testindex on TestTable(source_id, id);
          180  +  PRAGMA foreign_keys=1;
          181  +  INSERT INTO TestTable VALUES (1, 'parent', 1, null);
          182  +  INSERT INTO TestTable VALUES (2, 'child', 1, 1);
          183  +  UPDATE TestTable SET parent_id=1000 where id=2;
          184  +} {1 {foreign key constraint failed}}
          185  +
    80    186   finish_test

Added test/fts3auto.test.

            1  +# 2011 June 10
            2  +#
            3  +#    May you do good and not evil.
            4  +#    May you find forgiveness for yourself and forgive others.
            5  +#    May you share freely, never taking more than you give.
            6  +#
            7  +#***********************************************************************
            8  +#
            9  +
           10  +set testdir [file dirname $argv0]
           11  +source $testdir/tester.tcl
           12  +
           13  +# If this build does not include FTS3, skip the tests in this file.
           14  +#
           15  +ifcapable !fts3 { finish_test ; return }
           16  +source $testdir/fts3_common.tcl
           17  +source $testdir/malloc_common.tcl
           18  +
           19  +set testprefix fts3auto
           20  +set sfep $sqlite_fts3_enable_parentheses
           21  +set sqlite_fts3_enable_parentheses 1
           22  +
           23  +#--------------------------------------------------------------------------
           24  +# Start of Tcl infrastructure used by tests. The entry points are:
           25  +#
           26  +#   do_fts3query_test
           27  +#   fts3_make_deferrable
           28  +#   fts3_zero_long_segments 
           29  +#
           30  +
           31  +#
           32  +#    do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR
           33  +#
           34  +# This proc runs several test cases on FTS3/4 table $TABLE using match
           35  +# expression $MATCHEXPR. All documents in $TABLE must be formatted so that
           36  +# they can be "tokenized" using the Tcl list commands (llength, lindex etc.).
           37  +# The name and column names used by $TABLE must not require any quoting or
           38  +# escaping when used in SQL statements.
           39  +#
           40  +# $MATCHINFO may be any expression accepted by the FTS4 MATCH operator, 
           41  +# except that the "<column-name>:token" syntax is not supported. Tcl list
           42  +# commands are used to tokenize the expression. Any parenthesis must appear
           43  +# either as separate list elements, or as the first (for opening) or last
           44  +# (for closing) character of a list element. i.e. the expression "(a OR b)c"
           45  +# will not be parsed correctly, but "( a OR b) c" will.
           46  +#
           47  +# Available OPTIONS are:
           48  +#
           49  +#     -deferred TOKENLIST
           50  +#
           51  +# If the "deferred" option is supplied, it is passed a list of tokens that
           52  +# are deferred by FTS and result in the relevant matchinfo() stats being an
           53  +# approximation. 
           54  +#
           55  +set sqlite_fts3_enable_parentheses 1
           56  +proc do_fts3query_test {tn args} {
           57  +
           58  +  set nArg [llength $args]
           59  +  if {$nArg < 2 || ($nArg % 2)} {
           60  +    set cmd do_fts3query_test
           61  +    error "wrong # args: should be \"$cmd ?-deferred LIST? TABLE MATCHEXPR\""
           62  +  }
           63  +  set tbl   [lindex $args [expr $nArg-2]]
           64  +  set match [lindex $args [expr $nArg-1]]
           65  +  set deferred [list]
           66  +
           67  +  foreach {k v} [lrange $args 0 [expr $nArg-3]] {
           68  +    switch -- $k {
           69  +      -deferred {
           70  +        set deferred $v
           71  +      }
           72  +      default {
           73  +        error "bad option \"$k\": must be -deferred"
           74  +      }
           75  +    }
           76  +  }
           77  +
           78  +  get_near_results $tbl $match $deferred aMatchinfo
           79  +
           80  +  set matchinfo_asc [list]
           81  +  foreach docid [lsort -integer -incr [array names aMatchinfo]] {
           82  +    lappend matchinfo_asc $docid $aMatchinfo($docid)
           83  +  }
           84  +  set matchinfo_desc [list]
           85  +  foreach docid [lsort -integer -decr [array names aMatchinfo]] {
           86  +    lappend matchinfo_desc $docid $aMatchinfo($docid)
           87  +  }
           88  +
           89  +  set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)"
           90  +
           91  +  do_execsql_test $tn$title.1 "
           92  +    SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC
           93  +  " [lsort -integer -incr [array names aMatchinfo]] 
           94  +
           95  +  do_execsql_test $tn$title.2 "
           96  +    SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC
           97  +  " [lsort -integer -decr [array names aMatchinfo]] 
           98  +
           99  +  do_execsql_test $tn$title.3 "
          100  +    SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
          101  +    WHERE $tbl MATCH '$match' ORDER BY docid DESC
          102  +  " $matchinfo_desc
          103  +
          104  +  do_execsql_test $tn$title.4 "
          105  +    SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
          106  +    WHERE $tbl MATCH '$match' ORDER BY docid ASC
          107  +  " $matchinfo_asc
          108  +}
          109  +
          110  +#    fts3_make_deferrable TABLE TOKEN
          111  +#
          112  +proc fts3_make_deferrable {tbl token} {
          113  +
          114  +  set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy]
          115  +  set name [sqlite3_column_name $stmt 0]
          116  +  sqlite3_finalize $stmt
          117  +
          118  +  set nRow [db one "SELECT count(*) FROM $tbl"]
          119  +  set pgsz [db one "PRAGMA page_size"]
          120  +  execsql BEGIN
          121  +  for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
          122  +    set doc [string repeat "$token " 100]
          123  +    execsql "INSERT INTO $tbl ($name) VALUES(\$doc)"
          124  +  }
          125  +  execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')"
          126  +  execsql COMMIT
          127  +
          128  +  return [expr $nRow*$pgsz]
          129  +}
          130  +
          131  +#    fts3_zero_long_segments TABLE ?LIMIT?
          132  +#
          133  +proc fts3_zero_long_segments {tbl limit} {
          134  +  execsql " 
          135  +    UPDATE ${tbl}_segments 
          136  +    SET block = zeroblob(length(block)) 
          137  +    WHERE length(block)>$limit
          138  +  "
          139  +  return [db changes]
          140  +}
          141  +
          142  +
          143  +proc mit {blob} {
          144  +  set scan(littleEndian) i*
          145  +  set scan(bigEndian) I*
          146  +  binary scan $blob $scan($::tcl_platform(byteOrder)) r
          147  +  return $r
          148  +}
          149  +db func mit mit
          150  +
          151  +proc fix_phrase_expr {cols expr colfiltervar} {
          152  +  upvar $colfiltervar iColFilter
          153  +
          154  +  set out [list]
          155  +  foreach t $expr {
          156  +    if {[string match *:* $t]} {
          157  +      set col [lindex [split $t :] 0]
          158  +      set t   [lindex [split $t :] 1]
          159  +      set iCol [lsearch $cols $col]
          160  +      if {$iCol<0} { error "unknown column: $col" }
          161  +      if {$iColFilter < 0} {
          162  +        set iColFilter $iCol
          163  +      } elseif {$iColFilter != $iCol} {
          164  +        set iColFilter [llength $cols]
          165  +      }
          166  +    }
          167  +    lappend out $t
          168  +  }
          169  +
          170  +  return $out
          171  +}
          172  +
          173  +proc fix_near_expr {cols expr colfiltervar} { 
          174  +  upvar $colfiltervar iColFilter
          175  + 
          176  +  set iColFilter -1
          177  +
          178  +  set out [list]
          179  +  lappend out [fix_phrase_expr $cols [lindex $expr 0] iColFilter]
          180  +  foreach {a b} [lrange $expr 1 end] {
          181  +    if {[string match -nocase near $a]}   { set a 10 }
          182  +    if {[string match -nocase near/* $a]} { set a [string range $a 5 end] }
          183  +    lappend out $a
          184  +    lappend out [fix_phrase_expr $cols $b iColFilter]
          185  +  }
          186  +  return $out
          187  +}
          188  +
          189  +proc get_single_near_results {tbl expr deferred arrayvar nullvar} {
          190  +  upvar $arrayvar aMatchinfo
          191  +  upvar $nullvar nullentry
          192  +  catch {array unset aMatchinfo}
          193  +
          194  +  set cols [list]
          195  +  set miss [list]
          196  +  db eval "PRAGMA table_info($tbl)" A { lappend cols $A(name) ; lappend miss 0 }
          197  +  set expr [fix_near_expr $cols $expr iColFilter]
          198  +
          199  +  # Calculate the expected results using [fts3_near_match]. The following
          200  +  # loop populates the "hits" and "counts" arrays as follows:
          201  +  # 
          202  +  #   1. For each document in the table that matches the NEAR expression,
          203  +  #      hits($docid) is set to 1. The set of docids that match the expression
          204  +  #      can therefore be found using [array names hits].
          205  +  #
          206  +  #   2. For each column of each document in the table, counts($docid,$iCol)
          207  +  #      is set to the -phrasecountvar output.
          208  +  #
          209  +  set res [list]
          210  +  catch { array unset hits }
          211  +  db eval "SELECT docid, * FROM $tbl" d {
          212  +    set iCol 0
          213  +    foreach col [lrange $d(*) 1 end] {
          214  +      set docid $d(docid)
          215  +      if {$iColFilter<0 || $iCol==$iColFilter} {
          216  +        set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)]
          217  +        if {$hit} { set hits($docid) 1 }
          218  +      } else {
          219  +        set counts($docid,$iCol) $miss
          220  +      }
          221  +      incr iCol
          222  +    }
          223  +  }
          224  +  set nPhrase [expr ([llength $expr]+1)/2]
          225  +  set nCol $iCol
          226  +
          227  +  # This block populates the nHit and nDoc arrays. For each phrase/column
          228  +  # in the query/table, array elements are set as follows:
          229  +  #
          230  +  #   nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in 
          231  +  #                          column $iCol.
          232  +  #
          233  +  #   nDoc($iPhrase,$iCol) - Number of documents with at least one hit for
          234  +  #                          phrase $iPhrase in column $iCol.
          235  +  #
          236  +  for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} {
          237  +    for {set iCol 0} {$iCol < $nCol} {incr iCol} {
          238  +      set nHit($iPhrase,$iCol) 0
          239  +      set nDoc($iPhrase,$iCol) 0
          240  +    }
          241  +  }
          242  +  foreach key [array names counts] {
          243  +    set iCol [lindex [split $key ,] 1]
          244  +    set iPhrase 0
          245  +    foreach c $counts($key) {
          246  +      if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
          247  +      incr nHit($iPhrase,$iCol) $c
          248  +      incr iPhrase
          249  +    }
          250  +  }
          251  +
          252  +  if {[llength $deferred] && [llength $expr]==1} {
          253  +    set phrase [lindex $expr 0]
          254  +    set rewritten [list]
          255  +    set partial 0
          256  +    foreach tok $phrase {
          257  +      if {[lsearch $deferred $tok]>=0} {
          258  +        lappend rewritten *
          259  +      } else {
          260  +        lappend rewritten $tok
          261  +        set partial 1
          262  +      }
          263  +    }
          264  +    if {$partial==0} {
          265  +      set tblsize [db one "SELECT count(*) FROM $tbl"]
          266  +      for {set iCol 0} {$iCol < $nCol} {incr iCol} {
          267  +        set nHit(0,$iCol) $tblsize
          268  +        set nDoc(0,$iCol) $tblsize
          269  +      }
          270  +    } elseif {$rewritten != $phrase} {
          271  +      while {[lindex $rewritten end] == "*"} {
          272  +        set rewritten [lrange $rewritten 0 end-1]
          273  +      }
          274  +      while {[lindex $rewritten 0] == "*"} {
          275  +        set rewritten [lrange $rewritten 1 end]
          276  +      }
          277  +      get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry
          278  +      foreach docid [array names hits] {
          279  +        set aMatchinfo($docid) $aRewrite($docid)
          280  +      }
          281  +      return
          282  +    }
          283  +  }
          284  +
          285  +  # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
          286  +  # contain the output of matchinfo('x') for the document.
          287  +  #
          288  +  foreach docid [array names hits] {
          289  +    set mi [list]
          290  +    for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          291  +      for {set iCol 0} {$iCol<$nCol} {incr iCol} {
          292  +        lappend mi [lindex $counts($docid,$iCol) $iPhrase]
          293  +        lappend mi $nHit($iPhrase,$iCol)
          294  +        lappend mi $nDoc($iPhrase,$iCol)
          295  +      }
          296  +    }
          297  +    set aMatchinfo($docid) $mi
          298  +  }
          299  +
          300  +  # Set up the nullentry output.
          301  +  #
          302  +  set nullentry [list]
          303  +  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          304  +    for {set iCol 0} {$iCol<$nCol} {incr iCol} {
          305  +      lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol)
          306  +    }
          307  +  }
          308  +}
          309  +
          310  +
          311  +proc matching_brackets {expr} {
          312  +  if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { 
          313  +    return 0 
          314  +  }
          315  +
          316  +  set iBracket 1
          317  +  set nExpr [string length $expr]
          318  +  for {set i 1} {$iBracket && $i < $nExpr} {incr i} {
          319  +    set c [string range $expr $i $i]
          320  +    if {$c == "("} {incr iBracket}
          321  +    if {$c == ")"} {incr iBracket -1}
          322  +  }
          323  +
          324  +  return [expr ($iBracket==0 && $i==$nExpr)]
          325  +}
          326  +
          327  +proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} {
          328  +  upvar $arrayvar aMatchinfo
          329  +  if {$nullvar != ""} { upvar $nullvar nullentry }
          330  +
          331  +  set expr [string trim $expr]
          332  +  while { [matching_brackets $expr] } {
          333  +    set expr [string trim [string range $expr 1 end-1]]
          334  +  }
          335  +
          336  +  set prec(NOT) 1
          337  +  set prec(AND) 2
          338  +  set prec(OR)  3
          339  +
          340  +  set currentprec 0
          341  +  set iBracket 0
          342  +  set expr_length [llength $expr]
          343  +  for {set i 0} {$i < $expr_length} {incr i} {
          344  +    set op [lindex $expr $i]
          345  +    if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } {
          346  +      set opidx $i
          347  +      set currentprec $prec($op)
          348  +    } else {
          349  +      for {set j 0} {$j < [string length $op]} {incr j} {
          350  +        set c [string range $op $j $j]
          351  +        if {$c == "("} { incr iBracket +1 }
          352  +        if {$c == ")"} { incr iBracket -1 }
          353  +      }
          354  +    }
          355  +  }
          356  +  if {$iBracket!=0} { error "mismatched brackets in: $expr" }
          357  +
          358  +  if {[info exists opidx]==0} {
          359  +    get_single_near_results $tbl $expr $deferred aMatchinfo nullentry
          360  +  } else {
          361  +    set eLeft  [lrange $expr 0 [expr $opidx-1]]
          362  +    set eRight [lrange $expr [expr $opidx+1] end]
          363  +
          364  +    get_near_results $tbl $eLeft  $deferred aLeft  nullleft
          365  +    get_near_results $tbl $eRight $deferred aRight nullright
          366  +
          367  +    switch -- [lindex $expr $opidx] {
          368  +      "NOT" {
          369  +        foreach hit [array names aLeft] {
          370  +          if {0==[info exists aRight($hit)]} {
          371  +            set aMatchinfo($hit) $aLeft($hit)
          372  +          }
          373  +        }
          374  +        set nullentry $nullleft
          375  +      }
          376  +
          377  +      "AND" {
          378  +        foreach hit [array names aLeft] {
          379  +          if {[info exists aRight($hit)]} {
          380  +            set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
          381  +          }
          382  +        }
          383  +        set nullentry [concat $nullleft $nullright]
          384  +      }
          385  +
          386  +      "OR" {
          387  +        foreach hit [array names aLeft] {
          388  +          if {[info exists aRight($hit)]} {
          389  +            set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
          390  +            unset aRight($hit)
          391  +          } else {
          392  +            set aMatchinfo($hit) [concat $aLeft($hit) $nullright]
          393  +          }
          394  +        }
          395  +        foreach hit [array names aRight] {
          396  +          set aMatchinfo($hit) [concat $nullleft $aRight($hit)]
          397  +        }
          398  +
          399  +        set nullentry [concat $nullleft $nullright]
          400  +      }
          401  +    }
          402  +  }
          403  +}
          404  +
          405  +
          406  +# End of test procs. Actual tests are below this line.
          407  +#--------------------------------------------------------------------------
          408  +
          409  +#--------------------------------------------------------------------------
          410  +# The following test cases - fts3auto-1.* - focus on testing the Tcl 
          411  +# command [fts3_near_match], which is used by other tests in this file.
          412  +#
          413  +proc test_fts3_near_match {tn doc expr res} {
          414  +  fts3_near_match $doc $expr -phrasecountvar p
          415  +  uplevel do_test [list $tn] [list [list set {} $p]] [list $res]
          416  +}
          417  +
          418  +test_fts3_near_match 1.1.1 {a b c a b} a                   {2}
          419  +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c}         {2 2 1}
          420  +test_fts3_near_match 1.1.3 {a b c a b} {"a b"}             {2}
          421  +test_fts3_near_match 1.1.4 {a b c a b} {"b c"}             {1}
          422  +test_fts3_near_match 1.1.5 {a b c a b} {"c c"}             {0}
          423  +
          424  +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f}         {0 0}
          425  +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f}         {1 1}
          426  +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b}         {0 0}
          427  +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b}         {1 1}
          428  +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0}
          429  +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1}
          430  +
          431  +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
          432  +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"}      {0 0 0}
          433  +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}
          434  +
          435  +#--------------------------------------------------------------------------
          436  +# Test cases fts3auto-2.* run some simple tests using the 
          437  +# [do_fts3query_test] proc.
          438  +#
          439  +foreach {tn create} {
          440  +  1    "fts4(a, b)"
          441  +  2    "fts4(a, b, order=DESC)"
          442  +  3    "fts4(a, b, order=ASC)"
          443  +  4    "fts4(a, b, prefix=1)"
          444  +  5    "fts4(a, b, order=DESC, prefix=1)"
          445  +  6    "fts4(a, b, order=ASC, prefix=1)"
          446  +} {
          447  +  do_test 2.$tn.1 {
          448  +    catchsql { DROP TABLE t1 }
          449  +    execsql  "CREATE VIRTUAL TABLE t1 USING $create"
          450  +    for {set i 0} {$i<32} {incr i} {
          451  +      set doc [list]
          452  +      if {$i&0x01} {lappend doc one}
          453  +      if {$i&0x02} {lappend doc two}
          454  +      if {$i&0x04} {lappend doc three}
          455  +      if {$i&0x08} {lappend doc four}
          456  +      if {$i&0x10} {lappend doc five}
          457  +      execsql { INSERT INTO t1 VALUES($doc, null) }
          458  +    }
          459  +  } {}
          460  +
          461  +  foreach {tn2 expr} {
          462  +    1     {one}
          463  +    2     {one NEAR/1 five}
          464  +    3     {t*}
          465  +    4     {t* NEAR/0 five}
          466  +    5     {o* NEAR/1 f*}
          467  +    6     {one NEAR five NEAR two NEAR four NEAR three}
          468  +    7     {one NEAR xyz}
          469  +    8     {one OR two}
          470  +    9     {one AND two}
          471  +    10    {one NOT two}
          472  +    11    {one AND two OR three}
          473  +    12    {three OR one AND two}
          474  +    13    {(three OR one) AND two}
          475  +    14    {(three OR one) AND two NOT (five NOT four)}
          476  +    15    {"one two"}
          477  +    16    {"one two" NOT "three four"}
          478  +  } {
          479  +    do_fts3query_test 2.$tn.2.$tn2 t1 $expr
          480  +  }
          481  +}
          482  +
          483  +#--------------------------------------------------------------------------
          484  +# Some test cases involving deferred tokens.
          485  +#
          486  +
          487  +foreach {tn create} {
          488  +  1    "fts4(x)"
          489  +  2    "fts4(x, order=DESC)"
          490  +} {
          491  +  catchsql { DROP TABLE t1 }
          492  +  execsql  "CREATE VIRTUAL TABLE t1 USING $create"
          493  +  do_execsql_test 3.$tn.1 {
          494  +    INSERT INTO t1(docid, x) VALUES(-2, 'a b c d e f g h i j k');
          495  +    INSERT INTO t1(docid, x) VALUES(-1, 'b c d e f g h i j k a');
          496  +    INSERT INTO t1(docid, x) VALUES(0, 'c d e f g h i j k a b');
          497  +    INSERT INTO t1(docid, x) VALUES(1, 'd e f g h i j k a b c');
          498  +    INSERT INTO t1(docid, x) VALUES(2, 'e f g h i j k a b c d');
          499  +    INSERT INTO t1(docid, x) VALUES(3, 'f g h i j k a b c d e');
          500  +    INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k');
          501  +    INSERT INTO t1(docid, x) VALUES(5, 'a d g j');
          502  +    INSERT INTO t1(docid, x) VALUES(6, 'c a b');
          503  +  }
          504  +
          505  +  set limit [fts3_make_deferrable t1 c]
          506  +
          507  +  do_fts3query_test 3.$tn.2.1 t1 {a OR c}
          508  +
          509  +  do_test 3.$tn.3 { 
          510  +    fts3_zero_long_segments t1 $limit 
          511  +  } {1}
          512  +
          513  +  foreach {tn2 expr def} {
          514  +    1     {a NEAR c}            {}
          515  +    2     {a AND c}             c
          516  +    3     {"a c"}               c
          517  +    4     {"c a"}               c
          518  +    5     {"a c" NEAR/1 g}      {}
          519  +    6     {"a c" NEAR/0 g}      {}
          520  +  } {
          521  +    do_fts3query_test 3.$tn.4.$tn2 -deferred $def t1 $expr
          522  +  }
          523  +}
          524  +
          525  +#--------------------------------------------------------------------------
          526  +# 
          527  +foreach {tn create} {
          528  +  1    "fts4(x, y)"
          529  +  2    "fts4(x, y, order=DESC)"
          530  +  3    "fts4(x, y, order=DESC, prefix=2)"
          531  +} {
          532  +
          533  +  execsql [subst {
          534  +    DROP TABLE t1;
          535  +    CREATE VIRTUAL TABLE t1 USING $create;
          536  +    INSERT INTO t1 VALUES('one two five four five', '');
          537  +    INSERT INTO t1 VALUES('', 'one two five four five');
          538  +    INSERT INTO t1 VALUES('one two', 'five four five');
          539  +  }]
          540  +
          541  +  do_fts3query_test 4.$tn.1.1 t1 {one AND five}
          542  +  do_fts3query_test 4.$tn.1.2 t1 {one NEAR five}
          543  +  do_fts3query_test 4.$tn.1.3 t1 {one NEAR/1 five}
          544  +  do_fts3query_test 4.$tn.1.4 t1 {one NEAR/2 five}
          545  +  do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five}
          546  +
          547  +  do_test 4.$tn.2 { 
          548  +    set limit [fts3_make_deferrable t1 five]
          549  +    execsql { INSERT INTO t1(t1) VALUES('optimize') }
          550  +    expr {[fts3_zero_long_segments t1 $limit]>0}
          551  +  } {1}
          552  +
          553  +  do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five}
          554  +  do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five}
          555  +  do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five}
          556  +  do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five}
          557  +
          558  +  do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five}
          559  +
          560  +  do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
          561  +  do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*}
          562  +  do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*}
          563  +  do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*}
          564  +  do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*}
          565  +}
          566  +
          567  +#--------------------------------------------------------------------------
          568  +# The following test cases - fts3auto-5.* - focus on using prefix indexes.
          569  +#
          570  +set chunkconfig [fts3_configure_incr_load 1 1]
          571  +foreach {tn create pending} {
          572  +  1    "fts4(a, b)"                                  1
          573  +  2    "fts4(a, b, order=ASC, prefix=1)"             1
          574  +  3    "fts4(a, b, order=ASC,  prefix=1,3)"          0
          575  +  4    "fts4(a, b, order=DESC, prefix=2,4)"          0
          576  +  5    "fts4(a, b, order=DESC, prefix=1)"            0
          577  +  6    "fts4(a, b, order=ASC,  prefix=1,3)"          0
          578  +} {
          579  +
          580  +  execsql [subst {
          581  +    DROP TABLE IF EXISTS t1;
          582  +    CREATE VIRTUAL TABLE t1 USING $create;
          583  +  }]
          584  +
          585  +  if {$pending} {execsql BEGIN}
          586  +
          587  +  foreach {a b} {
          588  +    "the song of songs which is solomons"
          589  +    "let him kiss me with the kisses of his mouth for thy love is better than wine"
          590  +    "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee"
          591  +    "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee"
          592  +    "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon"
          593  +    "look not upon me because i am black because the sun hath looked upon me my mothers children were angry with me they made me the keeper of the vineyards but mine own vineyard have i not kept"
          594  +    "tell me o thou whom my soul loveth where thou feedest where thou makest thy flock to rest at noon for why should i be as one that turneth aside by the flocks of thy companions?"
          595  +    "if thou know not o thou fairest among women go thy way forth by the footsteps of the flock and feed thy kids beside the shepherds tents"
          596  +    "i have compared thee o my love to a company of horses in pharaohs chariots"
          597  +    "thy cheeks are comely with rows of jewels thy neck with chains of gold"
          598  +    "we will make thee borders of gold with studs of silver"
          599  +    "while the king sitteth at his table my spikenard sendeth forth the smell thereof"
          600  +    "a bundle of myrrh is my wellbeloved unto me he shall lie all night betwixt my breasts"
          601  +    "my beloved is unto me as a cluster of camphire in the vineyards of en gedi"
          602  +    "behold thou art fair my love behold thou art fair thou hast doves eyes"
          603  +    "behold thou art fair my beloved yea pleasant also our bed is green"
          604  +    "the beams of our house are cedar and our rafters of fir"
          605  +  } {
          606  +    execsql {INSERT INTO t1(a, b) VALUES($a, $b)}
          607  +  }
          608  +
          609  +
          610  +  do_fts3query_test 5.$tn.1.1 t1 {s*}
          611  +  do_fts3query_test 5.$tn.1.2 t1 {so*}
          612  +  do_fts3query_test 5.$tn.1.3 t1 {"s* o*"}
          613  +  do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*}
          614  +  do_fts3query_test 5.$tn.1.5 t1 {a*}
          615  +  do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*}
          616  +  do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"}
          617  +
          618  +  if {$pending} {execsql COMMIT}
          619  +}
          620  +eval fts3_configure_incr_load $chunkconfig
          621  +
          622  +foreach {tn pending create} {
          623  +  1    0 "fts4(a, b, c, d)"
          624  +  2    1 "fts4(a, b, c, d)"
          625  +  3    0 "fts4(a, b, c, d, order=DESC)"
          626  +  4    1 "fts4(a, b, c, d, order=DESC)"
          627  +} {
          628  +  execsql [subst {
          629  +    DROP TABLE IF EXISTS t1;
          630  +    CREATE VIRTUAL TABLE t1 USING $create;
          631  +  }]
          632  +
          633  +
          634  +  if {$pending} { execsql BEGIN }
          635  +
          636  +  foreach {a b c d} {
          637  +    "A B C" "D E F" "G H I" "J K L"
          638  +    "B C D" "E F G" "H I J" "K L A"
          639  +    "C D E" "F G H" "I J K" "L A B"
          640  +    "D E F" "G H I" "J K L" "A B C"
          641  +    "E F G" "H I J" "K L A" "B C D"
          642  +    "F G H" "I J K" "L A B" "C D E"
          643  +  } {
          644  +    execsql { INSERT INTO t1 VALUES($a, $b, $c, $d) }
          645  +  }
          646  +
          647  +  do_fts3query_test 6.$tn.1 t1 {b:G}
          648  +  do_fts3query_test 6.$tn.2 t1 {b:G AND c:I}
          649  +  do_fts3query_test 6.$tn.3 t1 {b:G NEAR c:I}
          650  +  do_fts3query_test 6.$tn.4 t1 {a:C OR b:G OR c:K OR d:C}
          651  +  do_fts3query_test 6.$tn.5 t1 {a:G OR b:G}
          652  +
          653  +  catchsql { COMMIT }
          654  +}
          655  +
          656  +set sqlite_fts3_enable_parentheses $sfep
          657  +finish_test
          658  +

Changes to test/fts3defer.test.

    16     16   ifcapable !fts3 {
    17     17     finish_test
    18     18     return
    19     19   }
    20     20   
    21     21   set sqlite_fts3_enable_parentheses 1
    22     22   
           23  +set fts3_simple_deferred_tokens_only 1
           24  +
    23     25   set ::testprefix fts3defer
    24     26   
    25     27   #--------------------------------------------------------------------------
    26     28   # Test cases fts3defer-1.* are the "warm body" cases. The database contains
    27     29   # one row with 15000 instances of the token "a". This makes the doclist for
    28     30   # "a" so large that FTS3 will avoid loading it in most cases.
    29     31   #
................................................................................
   253    255   
   254    256     do_select_test 1.1 {
   255    257       SELECT rowid FROM t1 WHERE t1 MATCH 'jk xnxhf'
   256    258     } {13 29 40 47 48 52 63 92}
   257    259     do_select_test 1.2 {
   258    260       SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
   259    261     } {100}
   260         -if {$tn==3} breakpoint
   261    262     do_select_test 1.3 {
   262    263       SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
   263    264     } {7 70 98}
   264    265     do_select_test 1.4 {
   265    266       SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl jk'
   266    267     } {3 5 8 10 13 18 20 23 32 37 41 43 55 60 65 67 72 74 76 81 94 96 97}
   267    268     do_select_test 1.5 {
................................................................................
   278    279     } {68 100}
   279    280     do_select_test 1.9 {
   280    281       SELECT rowid FROM t1 WHERE t1 MATCH 'zm ubwrfqnbjf'
   281    282     } {7 70 98}
   282    283     do_select_test 1.10 {
   283    284       SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
   284    285     } {10 13 17 31 35 51 58 88 89 90 93 100}
   285         -  do_select_test 1.11 {
   286         -    SELECT rowid FROM t1 
   287         -    WHERE t1 MATCH '(
   288         -      zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR 
   289         -      zk OR zkhdvkw OR zm OR zsmhnf
   290         -    ) vgsld'
   291         -  } {10 13 17 31 35 51 58 88 89 90 93 100}
          286  +
          287  +  if { $fts3_simple_deferred_tokens_only==0 } {
          288  +    do_select_test 1.11 {
          289  +      SELECT rowid FROM t1 
          290  +      WHERE t1 MATCH '(
          291  +        zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR 
          292  +        zk OR zkhdvkw OR zm OR zsmhnf
          293  +      ) vgsld'
          294  +    } {10 13 17 31 35 51 58 88 89 90 93 100}
          295  +  }
   292    296   
   293    297     do_select_test 2.1 {
   294    298       SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
   295    299     } {3 24 52 53}
   296    300     do_select_test 2.2 {
   297    301       SELECT rowid FROM t1 WHERE t1 MATCH '"zm zf"'
   298    302     } {33 53 75 88 101}
................................................................................
   360    364     # The following block of tests runs normally with FTS3 or FTS4 without the
   361    365     # long doclists zeroed. And with OOM-injection for FTS4 with long doclists
   362    366     # zeroed. Change this by messing with the [set dmt_modes] commands above.
   363    367     #
   364    368     foreach DO_MALLOC_TEST $dmt_modes {
   365    369       
   366    370       # Phrase search.
          371  +    #
   367    372       do_select_test 5.$DO_MALLOC_TEST.1 {
   368    373         SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"'
   369    374       } {8 15 36 64 67 72}
   370    375   
   371    376       # Multiple tokens search.
   372    377       do_select_test 5.$DO_MALLOC_TEST.2 {
   373    378         SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl zm'
................................................................................
   412    417     } {10}
   413    418     do_select_test 6.2.1 {
   414    419       SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk"'
   415    420     } {8}
   416    421     do_select_test 6.2.2 {
   417    422       SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"'
   418    423     } {15 26 92 96}
   419         -  do_select_test 6.2.3 {
   420         -    SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
   421         -  } {8 15 26 92 96}
          424  +  if {$fts3_simple_deferred_tokens_only==0} {
          425  +    do_select_test 6.2.3 {
          426  +      SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
          427  +    } {8 15 26 92 96}
          428  +  }
   422    429   }
   423    430   
   424    431   set testprefix fts3defer
   425    432   
   426    433   do_execsql_test 3.1 {
   427    434     CREATE VIRTUAL TABLE x1 USING fts4(a, b);
   428    435     INSERT INTO x1 VALUES('a b c', 'd e f');

Changes to test/fts3defer2.test.

    44     44     INSERT INTO t1(t1) VALUES('optimize');
    45     45   }
    46     46   do_execsql_test 1.1.4 {
    47     47     SELECT count(*) FROM t1_segments WHERE length(block)>10000;
    48     48     UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000;
    49     49   } {2}
    50     50   
           51  +do_execsql_test 1.2.0 {
           52  +  SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
           53  +} {{a b c d e f a x y}}
           54  +
    51     55   do_execsql_test 1.2.1 {
    52     56     SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
    53     57   } {{a b c d e f a x y}}
    54     58   
    55     59   do_execsql_test 1.2.2 {
    56     60     SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
    57     61     FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
    58     62   } [list                              \
    59     63      {a b c d [e] [f] [a] x y}         \
    60     64      {0 1 8 1 0 0 10 1 0 2 12 1}       \
    61         -   [list 3 1   1 1 1   1 8 8   1 8 8   8 5001 9]
           65  +   [list 3 1   1 1 1   1 1 1   1 1 1   8 5001 9]
    62     66   ]
    63     67   
    64     68   do_execsql_test 1.2.3 {
    65     69     SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
    66     70     FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
    67     71   } [list                                 \
    68     72      {[a] b c d [e] [f] [a] x y}          \
    69     73      {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
    70         -   [list 3 1   1 1 1   1 8 8   2 8 8   8 5001 9]
           74  +   [list 3 1   1 1 1   1 1 1   2 2 1   8 5001 9]
    71     75   ]
    72     76   
    73     77   do_execsql_test 1.3.1 { DROP TABLE t1 }
    74     78   
    75     79   #-----------------------------------------------------------------------------
    76     80   # Test cases fts3defer2-2.* focus specifically on the matchinfo function.
    77     81   # 
................................................................................
    95     99     do_execsql_test 2.2.$tn.1 {
    96    100       SELECT mit(matchinfo(t2, 'pcxnal')) FROM t2 WHERE t2 MATCH 'a b';
    97    101     } [list                                          \
    98    102       [list 2 1  1 54 54  1 3 3  54 372 8]        \
    99    103       [list 2 1  1 54 54  1 3 3  54 372 7]        \
   100    104     ]
   101    105   
   102         -  set sqlite_fts3_enable_parentheses 1
   103    106     do_execsql_test 2.2.$tn.2 {
          107  +    SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g z';
          108  +  } [list                                       \
          109  +    [list 1 2 2  1 54 54]                       \
          110  +  ]
          111  +
          112  +  set sqlite_fts3_enable_parentheses 1
          113  +  do_execsql_test 2.2.$tn.3 {
   104    114       SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)';
   105    115     } [list                                       \
   106    116       [list 1 2 2  1 2 2   1 54 54]               \
   107    117       [list 1 2 2  1 2 2   0 54 54]               \
   108    118     ]
   109    119     set sqlite_fts3_enable_parentheses 0
   110    120   }

Changes to test/fts3matchinfo.test.

    64     64     CREATE VIRTUAL TABLE t3 USING fts3(mtchinfo=fts3);
    65     65     INSERT INTO t3(mtchinfo) VALUES('Beside the lake, beneath the trees');
    66     66     SELECT mtchinfo FROM t3;
    67     67   } {{Beside the lake, beneath the trees}}
    68     68   
    69     69   do_execsql_test 3.2 {
    70     70     CREATE VIRTUAL TABLE xx USING FTS4;
           71  +}
           72  +do_execsql_test 3.3 {
    71     73     SELECT * FROM xx WHERE xx MATCH 'abc';
           74  +}
           75  +do_execsql_test 3.4 {
    72     76     SELECT * FROM xx WHERE xx MATCH 'a b c';
    73     77   }
    74     78   
    75     79   
    76     80   #--------------------------------------------------------------------------
    77     81   # Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
    78     82   #
................................................................................
   236    240   do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
   237    241   do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
   238    242   do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
   239    243   do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }
   240    244   
   241    245   do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
   242    246   
   243         -do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 
   244         -  x {{5 8 2   5 5 5} {3 8 2   3 5 5}}
   245         -  s {2 1} 
          247  +# It used to be that the second 'a' token would be deferred. That doesn't
          248  +# work any longer.
          249  +if 0 {
          250  +  do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 
          251  +    x {{5 8 2   5 5 5} {3 8 2   3 5 5}}
          252  +    s {2 1} 
          253  +  }
   246    254   }
   247    255   
   248    256   do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
   249    257   do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
   250    258   do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
   251    259   do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
   252    260   do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }

Added test/fts3prefix.test.

            1  +# 2011 May 04
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS3 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix fts3prefix
           18  +
           19  +ifcapable !fts3 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +# This proc tests that the prefixes index appears to represent the same content
           25  +# as the terms index.
           26  +#
           27  +proc fts3_terms_and_prefixes {db tbl prefixlengths} {
           28  +
           29  +  set iIndex 0
           30  +  foreach len $prefixlengths {
           31  +    incr iIndex
           32  +    $db eval {
           33  +      DROP TABLE IF EXISTS fts3check1;
           34  +      DROP TABLE IF EXISTS fts3check2;
           35  +    }
           36  +    $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);"
           37  +    $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);"
           38  +
           39  +    $db eval {
           40  +      DROP TABLE IF EXISTS temp.terms;
           41  +      DROP TABLE IF EXISTS temp.prefixes;
           42  +      CREATE TEMP TABLE terms AS SELECT * FROM fts3check1;
           43  +      CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2;
           44  +      CREATE INDEX temp.idx ON prefixes(term);
           45  +      DROP TABLE fts3check1;
           46  +      DROP TABLE fts3check2;
           47  +    }
           48  +
           49  +    set nExpect 0
           50  +    $db eval { SELECT term, docid, col, pos FROM temp.terms } a {
           51  +      if {[string length $a(term)]<$len} continue
           52  +      incr nExpect
           53  +      set prefix [string range $a(term) 0 [expr $len-1]]
           54  +      set r [$db one { 
           55  +        SELECT count(*) FROM temp.prefixes WHERE 
           56  +        term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos)
           57  +      }]
           58  +      if {$r != 1} {
           59  +        error "$t, $a(docid), $a(col), $a(pos)"
           60  +      }
           61  +    }
           62  +
           63  +    set nCount [$db one {SELECT count(*) FROM temp.prefixes}]
           64  +    if {$nCount != $nExpect} {
           65  +      error "prefixes.count(*) is $nCount expected $nExpect"
           66  +    }
           67  +  
           68  +    execsql { DROP TABLE temp.prefixes }
           69  +    execsql { DROP TABLE temp.terms }
           70  +
           71  +    set list [list]
           72  +    $db eval "
           73  +      SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree 
           74  +      FROM ${tbl}_segdir GROUP BY tree
           75  +    " {
           76  +      lappend list [list $total $tree]
           77  +    }
           78  +
           79  +    if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } {
           80  +      error "inconsistent tree structures: $list"
           81  +    }
           82  +  }
           83  +
           84  +  return ""
           85  +}
           86  +proc fts3_tap_test {tn db tbl lens} {
           87  +  uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""]
           88  +}
           89  +
           90  +#-------------------------------------------------------------------------
           91  +# Test cases 1.* are a sanity check. They test that the prefixes index is
           92  +# being constructed correctly for the simplest possible case.
           93  +#
           94  +do_execsql_test 1.1 {
           95  +  CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6');
           96  +
           97  +  CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1);
           98  +  CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2);
           99  +  CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3);
          100  +  CREATE VIRTUAL TABLE terms USING fts4term(t1);
          101  +}
          102  +do_execsql_test 1.2 {
          103  +  INSERT INTO t1 VALUES('sqlite mysql firebird');
          104  +}
          105  +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s}
          106  +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql}
          107  +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite}
          108  +do_execsql_test 1.4 {
          109  +  SELECT term FROM terms;
          110  +} {firebird mysql sqlite}
          111  +
          112  +fts3_tap_test 1.5 db t1 {1 3 6}
          113  +
          114  +#-------------------------------------------------------------------------
          115  +# A slightly more complicated dataset. This test also verifies that DELETE
          116  +# operations do not corrupt the prefixes index.
          117  +#
          118  +do_execsql_test 2.1 {
          119  +  INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules');
          120  +  INSERT INTO t1 VALUES('that allows users to perform full-text searches on');
          121  +  INSERT INTO t1 VALUES('a set of documents. The most common (and');
          122  +  INSERT INTO t1 VALUES('effective) way to describe full-text searches is');
          123  +  INSERT INTO t1 VALUES('"what Google, Yahoo and Altavista do with');
          124  +  INSERT INTO t1 VALUES('documents placed on the World Wide Web". Users');
          125  +  INSERT INTO t1 VALUES('input a term, or series of terms, perhaps');
          126  +  INSERT INTO t1 VALUES('connected by a binary operator or grouped together');
          127  +  INSERT INTO t1 VALUES('into a phrase, and the full-text query system');
          128  +  INSERT INTO t1 VALUES('finds the set of documents that best matches those');
          129  +  INSERT INTO t1 VALUES('terms considering the operators and groupings the');
          130  +  INSERT INTO t1 VALUES('user has specified. This article describes the');
          131  +  INSERT INTO t1 VALUES('deployment and usage of FTS3 and FTS4.');
          132  +  INSERT INTO t1 VALUES('FTS1 and FTS2 are obsolete full-text search');
          133  +  INSERT INTO t1 VALUES('modules for SQLite. There are known issues with');
          134  +  INSERT INTO t1 VALUES('these older modules and their use should be');
          135  +  INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were');
          136  +  INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of');
          137  +  INSERT INTO t1 VALUES('Google. It is now developed and maintained as part');
          138  +  INSERT INTO t1 VALUES('of SQLite. ');
          139  +}
          140  +fts3_tap_test 2.2 db t1 {1 3 6}
          141  +do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; }
          142  +fts3_tap_test 2.4 db t1 {1 3 6}
          143  +
          144  +do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') }
          145  +fts3_tap_test 2.6 db t1 {1 3 6}
          146  +
          147  +do_execsql_test 3.1 {
          148  +  CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3');
          149  +  INSERT INTO t2 VALUES('On 12 September the wind direction turned and');
          150  +  INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the');
          151  +  INSERT INTO t2 VALUES('fleet was forced to take shelter at');
          152  +  INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind');
          153  +  INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet');
          154  +  INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey');
          155  +  INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
          156  +  INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he');
          157  +  INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of');
          158  +  INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland');
          159  +  INSERT INTO t2 VALUES('and waited for Harold''s return from the north.');
          160  +  INSERT INTO t2 VALUES('On 12 September the wind direction turned and');
          161  +  INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the');
          162  +  INSERT INTO t2 VALUES('fleet was forced to take shelter at');
          163  +  INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind');
          164  +  INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet');
          165  +  INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey');
          166  +  INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
          167  +  INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he');
          168  +  INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of');
          169  +  INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland');
          170  +  INSERT INTO t2 VALUES('and waited for Harold''s return from the north.');
          171  +}
          172  +
          173  +fts3_tap_test 3.2 db t2 {1 2 3}
          174  +do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}}
          175  +fts3_tap_test 3.4 db t2 {1 2 3}
          176  +
          177  +
          178  +#-------------------------------------------------------------------------
          179  +# Simple tests for reading the prefix-index.
          180  +#
          181  +do_execsql_test 4.1 {
          182  +  CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4");
          183  +  INSERT INTO t3 VALUES('one two three');
          184  +  INSERT INTO t3 VALUES('four five six');
          185  +  INSERT INTO t3 VALUES('seven eight nine');
          186  +}
          187  +do_execsql_test 4.2 {
          188  +  SELECT * FROM t3 WHERE t3 MATCH 'f*'
          189  +} {{four five six}}
          190  +do_execsql_test 4.3 {
          191  +  SELECT * FROM t3 WHERE t3 MATCH 'four*'
          192  +} {{four five six}}
          193  +do_execsql_test 4.4 {
          194  +  SELECT * FROM t3 WHERE t3 MATCH 's*'
          195  +} {{four five six} {seven eight nine}}
          196  +do_execsql_test 4.5 {
          197  +  SELECT * FROM t3 WHERE t3 MATCH 'sev*'
          198  +} {{seven eight nine}}
          199  +do_execsql_test 4.6 {
          200  +  SELECT * FROM t3 WHERE t3 MATCH 'one*'
          201  +} {{one two three}}
          202  +
          203  +finish_test

Changes to test/fts3rnd.test.

   158    158   
   159    159     #lsort -uniq -integer $ret
   160    160     set ret
   161    161   }
   162    162   
   163    163   # This [proc] is used to test the FTS3 matchinfo() function.
   164    164   # 
   165         -proc simple_token_matchinfo {zToken} {
          165  +proc simple_token_matchinfo {zToken bDesc} {
   166    166   
   167    167     set nDoc(0) 0
   168    168     set nDoc(1) 0
   169    169     set nDoc(2) 0
   170    170     set nHit(0) 0
   171    171     set nHit(1) 0
   172    172     set nHit(2) 0
   173    173   
          174  +  set dir -inc
          175  +  if {$bDesc} { set dir -dec }
   174    176   
   175    177     foreach key [array names ::t1] {
   176    178       set value $::t1($key)
   177    179       set a($key) [list]
   178    180       foreach i {0 1 2} col $value {
   179    181         set hit [llength [lsearch -all $col $zToken]]
   180    182         lappend a($key) $hit
   181    183         incr nHit($i) $hit
   182    184         if {$hit>0} { incr nDoc($i) }
   183    185       }
   184    186     }
   185    187   
   186    188     set ret [list]
   187         -  foreach docid [lsort -integer [array names a]] {
          189  +  foreach docid [lsort -integer $dir [array names a]] {
   188    190       if { [lindex [lsort -integer $a($docid)] end] } {
   189    191         set matchinfo [list 1 3]
   190    192         foreach i {0 1 2} hit $a($docid) {
   191    193           lappend matchinfo $hit $nHit($i) $nDoc($i)
   192    194         }
   193    195         lappend ret $docid $matchinfo
   194    196       }
................................................................................
   258    260   proc mit {blob} {
   259    261     set scan(littleEndian) i*
   260    262     set scan(bigEndian) I*
   261    263     binary scan $blob $scan($::tcl_platform(byteOrder)) r
   262    264     return $r
   263    265   }
   264    266   db func mit mit
   265         -
   266    267   set sqlite_fts3_enable_parentheses 1
   267    268   
   268         -foreach nodesize {50 500 1000 2000} {
          269  +proc do_orderbydocid_test {tn sql res} {
          270  +  uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res]
          271  +  uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \
          272  +    [lsort -int -dec $res]
          273  +  ]
          274  +}
          275  +
          276  +set NUM_TRIALS 100
          277  +
          278  +foreach {nodesize order} {
          279  +  50    DESC
          280  +  50    ASC
          281  +  500   ASC
          282  +  1000  DESC
          283  +  2000  ASC
          284  +} {
   269    285     catch { array unset ::t1 }
          286  +  set testname "$nodesize/$order"
   270    287   
   271    288     # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows.
   272    289     #
   273    290     db transaction {
   274    291       catchsql { DROP TABLE t1 }
   275         -    execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)"
          292  +    execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)"
   276    293       execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')"
   277    294       for {set i 0} {$i < 100} {incr i} { insert_row $i }
   278    295     }
   279    296     
   280         -  for {set iTest 1} {$iTest <= 100} {incr iTest} {
          297  +  for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} {
   281    298       catchsql COMMIT
   282    299   
   283    300       set DO_MALLOC_TEST 0
   284    301       set nRep 10
   285    302       if {$iTest==100 && $nodesize==50} { 
   286    303         set DO_MALLOC_TEST 1 
   287    304         set nRep 2
   288    305       }
          306  +
          307  +    set ::testprefix fts3rnd-1.$testname.$iTest
   289    308     
   290    309       # Delete one row, update one row and insert one row.
   291    310       #
   292    311       set rows [array names ::t1]
   293    312       set nRow [llength $rows]
   294    313       set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]]
   295    314       set iDelete $iUpdate
................................................................................
   303    322       execsql BEGIN
   304    323         insert_row $iInsert
   305    324         update_row $iUpdate
   306    325         delete_row $iDelete
   307    326       if {0==($iTest%2)} { execsql COMMIT }
   308    327   
   309    328       if {0==($iTest%2)} { 
   310         -      do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok 
          329  +      #do_test 0 { fts3_integrity_check t1 } ok 
   311    330       }
   312    331   
   313    332       # Pick 10 terms from the vocabulary. Check that the results of querying
   314    333       # the database for the set of documents containing each of these terms
   315    334       # is the same as the result obtained by scanning the contents of the Tcl 
   316    335       # array for each term.
   317    336       #
   318    337       for {set i 0} {$i < 10} {incr i} {
   319    338         set term [random_term]
   320         -      do_select_test fts3rnd-1.$nodesize.$iTest.1.$i {
          339  +      do_select_test 1.$i.asc {
          340  +        SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
          341  +        ORDER BY docid ASC
          342  +      } [simple_token_matchinfo $term 0]
          343  +      do_select_test 1.$i.desc {
   321    344           SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
   322         -      } [simple_token_matchinfo $term]
          345  +        ORDER BY docid DESC
          346  +      } [simple_token_matchinfo $term 1]
   323    347       }
   324    348   
   325    349       # This time, use the first two characters of each term as a term prefix
   326    350       # to query for. Test that querying the Tcl array produces the same results
   327    351       # as querying the FTS3 table for the prefix.
   328    352       #
   329    353       for {set i 0} {$i < $nRep} {incr i} {
   330    354         set prefix [string range [random_term] 0 end-1]
   331    355         set match "${prefix}*"
   332         -      do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
          356  +      do_orderbydocid_test 2.$i {
   333    357           SELECT docid FROM t1 WHERE t1 MATCH $match
   334    358         } [simple_phrase $match]
   335    359       }
   336    360   
   337    361       # Similar to the above, except for phrase queries.
   338    362       #
   339    363       for {set i 0} {$i < $nRep} {incr i} {
   340    364         set term [list [random_term] [random_term]]
   341    365         set match "\"$term\""
   342         -      do_select_test fts3rnd-1.$nodesize.$iTest.3.$i {
          366  +      do_orderbydocid_test 3.$i {
   343    367           SELECT docid FROM t1 WHERE t1 MATCH $match
   344    368         } [simple_phrase $term]
   345    369       }
   346    370   
   347    371       # Three word phrases.
   348    372       #
   349    373       for {set i 0} {$i < $nRep} {incr i} {
   350    374         set term [list [random_term] [random_term] [random_term]]
   351    375         set match "\"$term\""
   352         -      do_select_test fts3rnd-1.$nodesize.$iTest.4.$i {
          376  +      do_orderbydocid_test 4.$i {
   353    377           SELECT docid FROM t1 WHERE t1 MATCH $match
   354    378         } [simple_phrase $term]
   355    379       }
   356    380   
   357    381       # Three word phrases made up of term-prefixes.
   358    382       #
   359    383       for {set i 0} {$i < $nRep} {incr i} {
   360    384         set    query "[string range [random_term] 0 end-1]* "
   361    385         append query "[string range [random_term] 0 end-1]* "
   362    386         append query "[string range [random_term] 0 end-1]*"
   363    387   
   364    388         set match "\"$query\""
   365         -      do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
          389  +      do_orderbydocid_test 5.$i {
   366    390           SELECT docid FROM t1 WHERE t1 MATCH $match
   367    391         } [simple_phrase $query]
   368    392       }
   369    393   
   370         -    # A NEAR query with terms as the arguments.
          394  +    # A NEAR query with terms as the arguments:
          395  +    #
          396  +    #     ... MATCH '$term1 NEAR $term2' ...
   371    397       #
   372    398       for {set i 0} {$i < $nRep} {incr i} {
   373    399         set terms [list [random_term] [random_term]]
   374    400         set match [join $terms " NEAR "]
   375         -      do_select_test fts3rnd-1.$nodesize.$iTest.6.$i {
          401  +      do_orderbydocid_test 6.$i {
   376    402           SELECT docid FROM t1 WHERE t1 MATCH $match 
   377    403         } [simple_near $terms 10]
   378    404       }
   379    405   
   380    406       # A 3-way NEAR query with terms as the arguments.
   381    407       #
   382    408       for {set i 0} {$i < $nRep} {incr i} {
   383    409         set terms [list [random_term] [random_term] [random_term]]
   384    410         set nNear 11
   385    411         set match [join $terms " NEAR/$nNear "]
   386         -      do_select_test fts3rnd-1.$nodesize.$iTest.7.$i {
          412  +      do_orderbydocid_test 7.$i {
   387    413           SELECT docid FROM t1 WHERE t1 MATCH $match
   388    414         } [simple_near $terms $nNear]
   389    415       }
   390    416       
   391    417       # Set operations on simple term queries.
   392    418       #
   393    419       foreach {tn op proc} {
................................................................................
   395    421         9  NOT setop_not
   396    422         10 AND setop_and
   397    423       } {
   398    424         for {set i 0} {$i < $nRep} {incr i} {
   399    425           set term1 [random_term]
   400    426           set term2 [random_term]
   401    427           set match "$term1 $op $term2"
   402         -        do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
          428  +        do_orderbydocid_test $tn.$i {
   403    429             SELECT docid FROM t1 WHERE t1 MATCH $match
   404    430           } [$proc [simple_phrase $term1] [simple_phrase $term2]]
   405    431         }
   406    432       }
   4