/ Check-in [7a44fa5a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Sync this branch with the latest trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal2
Files: files | file ages | folders
SHA3-256: 7a44fa5a350a3f19b8e9f5196d22535788885f8c0e849572202bf64a055ddc2d
User & Date: dan 2018-12-01 20:14:06
Wiki:wal2
Context
2018-12-03
18:13
Increase a timeout in test file walprotocol2.test. To account for unix builds without HAVE_USLEEP. check-in: 480be916 user: dan tags: wal2
2018-12-01
20:14
Sync this branch with the latest trunk. check-in: 7a44fa5a user: dan tags: wal2
12:34
Version 3.26.0 check-in: bf8c1b2b user: drh tags: trunk, release, version-3.26.0
2017-10-10
20:11
Add new extension "bgckpt" in ext/misc/bgckpt.c. For experimenting with running wal2 mode checkpoints in a background thread. check-in: 63955442 user: dan tags: wal2
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Added .fossil-settings/empty-dirs.

            1  +compat

Added .fossil-settings/ignore-glob.

            1  +compat/*

Changes to Makefile.in.

    18     18   TOP = @abs_srcdir@
    19     19   
    20     20   # C Compiler and options for use in building executables that
    21     21   # will run on the platform that is doing the build.
    22     22   #
    23     23   BCC = @BUILD_CC@ @BUILD_CFLAGS@
    24     24   
    25         -# TCC is the C Compile and options for use in building executables that 
           25  +# TCC is the C Compile and options for use in building executables that
    26     26   # will run on the target platform.  (BCC and TCC are usually the
    27     27   # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
    28     28   # are provide so that these aspects of the build process can be changed
    29     29   # on the "make" command-line.  Ex:  "make CC=clang CFLAGS=-fsanitize=undefined"
    30     30   #
    31     31   CC = @CC@
    32     32   CFLAGS = @CPPFLAGS@ @CFLAGS@
    33     33   TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu
    34     34   TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
           35  +TCC += -I${TOP}/ext/userauth
    35     36   
    36     37   # Define this for the autoconf-based build, so that the code knows it can
    37     38   # include the generated config.h
    38         -# 
           39  +#
    39     40   TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
    40     41   
    41     42   # Define -DNDEBUG to compile without debugging (i.e., for production usage)
    42     43   # Omitting the define will cause extra debugging code to be inserted and
    43     44   # includes extra comments when "EXPLAIN stmt" is used.
    44     45   #
    45     46   TCC += @TARGET_DEBUG@
................................................................................
    62     63   LIBREADLINE = @TARGET_READLINE_LIBS@
    63     64   
    64     65   # Should the database engine be compiled threadsafe
    65     66   #
    66     67   TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@
    67     68   
    68     69   # Any target libraries which libsqlite must be linked against
    69         -# 
           70  +#
    70     71   TLIBS = @LIBS@ $(LIBS)
    71     72   
    72     73   # Flags controlling use of the in memory btree implementation
    73     74   #
    74     75   # SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to
    75     76   # default to file, 2 to default to memory, and 3 to force temporary
    76     77   # tables to always be in memory.
    77     78   #
    78     79   TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
    79     80   
    80     81   # Enable/disable loadable extensions, and other optional features
    81         -# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
    82         -# The same set of OMIT and ENABLE flags should be passed to the 
           82  +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
           83  +# The same set of OMIT and ENABLE flags should be passed to the
    83     84   # LEMON parser generator and the mkkeywordhash tool as well.
    84     85   OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
    85     86   
    86     87   TCC += $(OPT_FEATURE_FLAGS)
    87     88   
    88     89   # Add in any optional parameters specified on the make commane line
    89     90   # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
    90     91   TCC += $(OPTS)
           92  +
           93  +# Add in compile-time options for some libraries used by extensions
           94  +TCC += @HAVE_ZLIB@
    91     95   
    92     96   # Version numbers and release number for the SQLite being compiled.
    93     97   #
    94     98   VERSION = @VERSION@
    95     99   VERSION_NUMBER = @VERSION_NUMBER@
    96    100   RELEASE = @RELEASE@
    97    101   
................................................................................
   119    123   # The suffix used on shared libraries.  Ex:  ".dll", ".so", ".dylib"
   120    124   #
   121    125   SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
   122    126   
   123    127   # If gcov support was enabled by the configure script, add the appropriate
   124    128   # flags here.  It's not always as easy as just having the user add the right
   125    129   # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
   126         -# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.  
   127         -# Supposedly GCC does the right thing if you use --coverage, but in 
          130  +# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.
          131  +# Supposedly GCC does the right thing if you use --coverage, but in
   128    132   # practice it still fails.  See:
   129    133   #
   130    134   # http://www.mail-archive.com/debian-gcc@lists.debian.org/msg26197.html
   131    135   #
   132    136   # for more info.
   133    137   #
   134    138   GCOV_CFLAGS1 = -DSQLITE_COVERAGE_TEST=1 -fprofile-arcs -ftest-coverage
................................................................................
   162    166   
   163    167   USE_AMALGAMATION = @USE_AMALGAMATION@
   164    168   
   165    169   # Object files for the SQLite library (non-amalgamation).
   166    170   #
   167    171   LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
   168    172            backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
   169         -         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
          173  +         callback.lo complete.lo ctime.lo \
          174  +         date.lo dbpage.lo dbstat.lo delete.lo \
   170    175            expr.lo fault.lo fkey.lo \
   171    176            fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
   172    177            fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
   173    178            fts3_tokenize_vtab.lo \
   174    179            fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
   175    180   	 fts5.lo \
   176    181            func.lo global.lo hash.lo \
   177    182            icu.lo insert.lo json1.lo legacy.lo loadext.lo \
   178    183            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   179         -         memjournal.lo \
          184  +         memdb.lo memjournal.lo \
   180    185            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   181    186            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   182    187            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   183    188            random.lo resolve.lo rowset.lo rtree.lo \
   184    189            sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
   185    190            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   186         -         update.lo util.lo vacuum.lo \
          191  +         update.lo userauth.lo upsert.lo util.lo vacuum.lo \
   187    192            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   188    193            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   189         -         utf.lo vtab.lo
          194  +         window.lo utf.lo vtab.lo
   190    195   
   191    196   # Object files for the amalgamation.
   192    197   #
   193    198   LIBOBJS1 = sqlite3.lo
   194    199   
   195    200   # Determine the real value of LIBOBJ based on the 'configure' script
   196    201   #
................................................................................
   211    216     $(TOP)/src/btree.h \
   212    217     $(TOP)/src/btreeInt.h \
   213    218     $(TOP)/src/build.c \
   214    219     $(TOP)/src/callback.c \
   215    220     $(TOP)/src/complete.c \
   216    221     $(TOP)/src/ctime.c \
   217    222     $(TOP)/src/date.c \
          223  +  $(TOP)/src/dbpage.c \
   218    224     $(TOP)/src/dbstat.c \
   219    225     $(TOP)/src/delete.c \
   220    226     $(TOP)/src/expr.c \
   221    227     $(TOP)/src/fault.c \
   222    228     $(TOP)/src/fkey.c \
   223    229     $(TOP)/src/func.c \
   224    230     $(TOP)/src/global.c \
................................................................................
   231    237     $(TOP)/src/main.c \
   232    238     $(TOP)/src/malloc.c \
   233    239     $(TOP)/src/mem0.c \
   234    240     $(TOP)/src/mem1.c \
   235    241     $(TOP)/src/mem2.c \
   236    242     $(TOP)/src/mem3.c \
   237    243     $(TOP)/src/mem5.c \
          244  +  $(TOP)/src/memdb.c \
   238    245     $(TOP)/src/memjournal.c \
   239    246     $(TOP)/src/msvc.h \
   240    247     $(TOP)/src/mutex.c \
   241    248     $(TOP)/src/mutex.h \
   242    249     $(TOP)/src/mutex_noop.c \
   243    250     $(TOP)/src/mutex_unix.c \
   244    251     $(TOP)/src/mutex_w32.c \
................................................................................
   261    268     $(TOP)/src/prepare.c \
   262    269     $(TOP)/src/printf.c \
   263    270     $(TOP)/src/random.c \
   264    271     $(TOP)/src/resolve.c \
   265    272     $(TOP)/src/rowset.c \
   266    273     $(TOP)/src/select.c \
   267    274     $(TOP)/src/status.c \
   268         -  $(TOP)/src/shell.c \
          275  +  $(TOP)/src/shell.c.in \
   269    276     $(TOP)/src/sqlite.h.in \
   270    277     $(TOP)/src/sqlite3ext.h \
   271    278     $(TOP)/src/sqliteInt.h \
   272    279     $(TOP)/src/sqliteLimit.h \
   273    280     $(TOP)/src/table.c \
   274    281     $(TOP)/src/tclsqlite.c \
   275    282     $(TOP)/src/threads.c \
   276    283     $(TOP)/src/tokenize.c \
   277    284     $(TOP)/src/treeview.c \
   278    285     $(TOP)/src/trigger.c \
   279    286     $(TOP)/src/utf.c \
   280    287     $(TOP)/src/update.c \
          288  +  $(TOP)/src/upsert.c \
   281    289     $(TOP)/src/util.c \
   282    290     $(TOP)/src/vacuum.c \
   283    291     $(TOP)/src/vdbe.c \
   284    292     $(TOP)/src/vdbe.h \
   285    293     $(TOP)/src/vdbeapi.c \
   286    294     $(TOP)/src/vdbeaux.c \
   287    295     $(TOP)/src/vdbeblob.c \
................................................................................
   293    301     $(TOP)/src/vxworks.h \
   294    302     $(TOP)/src/wal.c \
   295    303     $(TOP)/src/wal.h \
   296    304     $(TOP)/src/walker.c \
   297    305     $(TOP)/src/where.c \
   298    306     $(TOP)/src/wherecode.c \
   299    307     $(TOP)/src/whereexpr.c \
   300         -  $(TOP)/src/whereInt.h
          308  +  $(TOP)/src/whereInt.h \
          309  +  $(TOP)/src/window.c
   301    310   
   302    311   # Source code for extensions
   303    312   #
   304    313   SRC += \
   305    314     $(TOP)/ext/fts1/fts1.c \
   306    315     $(TOP)/ext/fts1/fts1.h \
   307    316     $(TOP)/ext/fts1/fts1_hash.c \
................................................................................
   338    347     $(TOP)/ext/fts3/fts3_unicode2.c \
   339    348     $(TOP)/ext/fts3/fts3_write.c
   340    349   SRC += \
   341    350     $(TOP)/ext/icu/sqliteicu.h \
   342    351     $(TOP)/ext/icu/icu.c
   343    352   SRC += \
   344    353     $(TOP)/ext/rtree/rtree.h \
   345         -  $(TOP)/ext/rtree/rtree.c
          354  +  $(TOP)/ext/rtree/rtree.c \
          355  +  $(TOP)/ext/rtree/geopoly.c
   346    356   SRC += \
   347    357     $(TOP)/ext/session/sqlite3session.c \
   348    358     $(TOP)/ext/session/sqlite3session.h
   349    359   SRC += \
          360  +  $(TOP)/ext/userauth/userauth.c \
          361  +  $(TOP)/ext/userauth/sqlite3userauth.h
          362  +SRC += \
   350    363     $(TOP)/ext/rbu/sqlite3rbu.h \
   351    364     $(TOP)/ext/rbu/sqlite3rbu.c
   352    365   SRC += \
   353    366     $(TOP)/ext/misc/json1.c \
   354    367     $(TOP)/ext/misc/stmt.c
   355    368   
   356    369   # Generated source code files
................................................................................
   358    371   SRC += \
   359    372     keywordhash.h \
   360    373     opcodes.c \
   361    374     opcodes.h \
   362    375     parse.c \
   363    376     parse.h \
   364    377     config.h \
          378  +  shell.c \
   365    379     sqlite3.h
   366    380   
   367    381   # Source code to the test files.
   368    382   #
   369    383   TESTSRC = \
   370    384     $(TOP)/src/test1.c \
   371    385     $(TOP)/src/test2.c \
................................................................................
   389    403     $(TOP)/src/test_fs.c \
   390    404     $(TOP)/src/test_func.c \
   391    405     $(TOP)/src/test_hexio.c \
   392    406     $(TOP)/src/test_init.c \
   393    407     $(TOP)/src/test_intarray.c \
   394    408     $(TOP)/src/test_journal.c \
   395    409     $(TOP)/src/test_malloc.c \
          410  +  $(TOP)/src/test_md5.c \
   396    411     $(TOP)/src/test_multiplex.c \
   397    412     $(TOP)/src/test_mutex.c \
   398    413     $(TOP)/src/test_onefile.c \
   399    414     $(TOP)/src/test_osinst.c \
   400    415     $(TOP)/src/test_pcache.c \
   401    416     $(TOP)/src/test_quota.c \
   402    417     $(TOP)/src/test_rtree.c \
   403    418     $(TOP)/src/test_schema.c \
   404    419     $(TOP)/src/test_server.c \
   405    420     $(TOP)/src/test_superlock.c \
   406    421     $(TOP)/src/test_syscall.c \
          422  +  $(TOP)/src/test_tclsh.c \
   407    423     $(TOP)/src/test_tclvar.c \
   408    424     $(TOP)/src/test_thread.c \
   409    425     $(TOP)/src/test_vfs.c \
   410    426     $(TOP)/src/test_windirent.c \
          427  +  $(TOP)/src/test_window.c \
   411    428     $(TOP)/src/test_wsd.c       \
   412    429     $(TOP)/ext/fts3/fts3_term.c \
   413    430     $(TOP)/ext/fts3/fts3_test.c  \
   414    431     $(TOP)/ext/session/test_session.c \
   415         -  $(TOP)/ext/rbu/test_rbu.c 
          432  +  $(TOP)/ext/rbu/test_rbu.c
   416    433   
   417    434   # Statically linked extensions
   418    435   #
   419    436   TESTSRC += \
          437  +  $(TOP)/ext/expert/sqlite3expert.c \
          438  +  $(TOP)/ext/expert/test_expert.c \
   420    439     $(TOP)/ext/misc/amatch.c \
   421    440     $(TOP)/ext/misc/carray.c \
   422    441     $(TOP)/ext/misc/closure.c \
   423    442     $(TOP)/ext/misc/csv.c \
   424    443     $(TOP)/ext/misc/eval.c \
          444  +  $(TOP)/ext/misc/explain.c \
   425    445     $(TOP)/ext/misc/fileio.c \
   426    446     $(TOP)/ext/misc/fuzzer.c \
   427    447     $(TOP)/ext/fts5/fts5_tcl.c \
   428    448     $(TOP)/ext/fts5/fts5_test_mi.c \
   429    449     $(TOP)/ext/fts5/fts5_test_tok.c \
   430    450     $(TOP)/ext/misc/ieee754.c \
   431    451     $(TOP)/ext/misc/mmapwarm.c \
   432    452     $(TOP)/ext/misc/nextchar.c \
          453  +  $(TOP)/ext/misc/normalize.c \
   433    454     $(TOP)/ext/misc/percentile.c \
   434    455     $(TOP)/ext/misc/regexp.c \
   435    456     $(TOP)/ext/misc/remember.c \
   436    457     $(TOP)/ext/misc/series.c \
   437    458     $(TOP)/ext/misc/spellfix.c \
   438    459     $(TOP)/ext/misc/totype.c \
   439    460     $(TOP)/ext/misc/unionvtab.c \
   440         -  $(TOP)/ext/misc/wholenumber.c
          461  +  $(TOP)/ext/misc/wholenumber.c \
          462  +  $(TOP)/ext/misc/zipfile.c \
          463  +  $(TOP)/ext/userauth/userauth.c
   441    464   
   442    465   # Source code to the library files needed by the test fixture
   443    466   #
   444    467   TESTSRC2 = \
   445    468     $(TOP)/src/attach.c \
   446    469     $(TOP)/src/backup.c \
   447    470     $(TOP)/src/bitvec.c \
   448    471     $(TOP)/src/btree.c \
   449    472     $(TOP)/src/build.c \
   450    473     $(TOP)/src/ctime.c \
   451    474     $(TOP)/src/date.c \
          475  +  $(TOP)/src/dbpage.c \
   452    476     $(TOP)/src/dbstat.c \
   453    477     $(TOP)/src/expr.c \
   454    478     $(TOP)/src/func.c \
          479  +  $(TOP)/src/global.c \
   455    480     $(TOP)/src/insert.c \
   456    481     $(TOP)/src/wal.c \
   457    482     $(TOP)/src/main.c \
   458    483     $(TOP)/src/mem5.c \
   459    484     $(TOP)/src/os.c \
   460    485     $(TOP)/src/os_unix.c \
   461    486     $(TOP)/src/os_win.c \
................................................................................
   474    499     $(TOP)/src/vdbeaux.c \
   475    500     $(TOP)/src/vdbe.c \
   476    501     $(TOP)/src/vdbemem.c \
   477    502     $(TOP)/src/vdbetrace.c \
   478    503     $(TOP)/src/where.c \
   479    504     $(TOP)/src/wherecode.c \
   480    505     $(TOP)/src/whereexpr.c \
          506  +  $(TOP)/src/window.c \
   481    507     parse.c \
   482    508     $(TOP)/ext/fts3/fts3.c \
   483    509     $(TOP)/ext/fts3/fts3_aux.c \
   484    510     $(TOP)/ext/fts3/fts3_expr.c \
   485    511     $(TOP)/ext/fts3/fts3_term.c \
   486    512     $(TOP)/ext/fts3/fts3_tokenizer.c \
   487    513     $(TOP)/ext/fts3/fts3_write.c \
   488    514     $(TOP)/ext/async/sqlite3async.c \
   489    515     $(TOP)/ext/session/sqlite3session.c \
   490         -  $(TOP)/ext/misc/stmt.c 
          516  +  $(TOP)/ext/misc/stmt.c
   491    517   
   492    518   # Header files used by all library source files.
   493    519   #
   494    520   HDR = \
   495    521      $(TOP)/src/btree.h \
   496    522      $(TOP)/src/btreeInt.h \
   497    523      $(TOP)/src/hash.h \
................................................................................
   530    556     $(TOP)/ext/fts2/fts2_tokenizer.h
   531    557   EXTHDR += \
   532    558     $(TOP)/ext/fts3/fts3.h \
   533    559     $(TOP)/ext/fts3/fts3Int.h \
   534    560     $(TOP)/ext/fts3/fts3_hash.h \
   535    561     $(TOP)/ext/fts3/fts3_tokenizer.h
   536    562   EXTHDR += \
   537         -  $(TOP)/ext/rtree/rtree.h
          563  +  $(TOP)/ext/rtree/rtree.h \
          564  +  $(TOP)/ext/rtree/geopoly.c
   538    565   EXTHDR += \
   539    566     $(TOP)/ext/icu/sqliteicu.h
   540    567   EXTHDR += \
   541    568     $(TOP)/ext/rtree/sqlite3rtree.h
          569  +EXTHDR += \
          570  +  $(TOP)/ext/userauth/sqlite3userauth.h
   542    571   
   543    572   # executables needed for testing
   544    573   #
   545    574   TESTPROGS = \
   546    575     testfixture$(TEXE) \
   547    576     sqlite3$(TEXE) \
   548    577     sqlite3_analyzer$(TEXE) \
   549    578     sqldiff$(TEXE) \
   550         -  dbhash$(TEXE)
          579  +  dbhash$(TEXE) \
          580  +  sqltclsh$(TEXE)
   551    581   
   552    582   # Databases containing fuzzer test cases
   553    583   #
   554    584   FUZZDATA = \
   555    585     $(TOP)/test/fuzzdata1.db \
   556    586     $(TOP)/test/fuzzdata2.db \
   557    587     $(TOP)/test/fuzzdata3.db \
   558    588     $(TOP)/test/fuzzdata4.db \
   559         -  $(TOP)/test/fuzzdata5.db
          589  +  $(TOP)/test/fuzzdata5.db \
          590  +  $(TOP)/test/fuzzdata6.db \
          591  +  $(TOP)/test/fuzzdata7.db
   560    592   
   561    593   # Standard options to testfixture
   562    594   #
   563    595   TESTOPTS = --verbose=file --output=test-out.txt
   564    596   
   565    597   # Extra compiler options for various shell tools
   566    598   #
   567    599   SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
   568         -# SHELL_OPT += -DSQLITE_ENABLE_FTS5
          600  +#SHELL_OPT += -DSQLITE_ENABLE_FTS5
          601  +SHELL_OPT += -DSQLITE_ENABLE_RTREE
   569    602   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   570    603   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   571    604   SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
          605  +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
          606  +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
          607  +SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
          608  +SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
          609  +SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
   572    610   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   573    611   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
   574    612   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
          613  +FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
   575    614   FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
   576         -DBFUZZ_OPT = 
          615  +DBFUZZ_OPT =
   577    616   
   578    617   # This is the default Makefile target.  The objects listed here
   579    618   # are what get build when you type just "make" with no arguments.
   580    619   #
   581    620   all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
   582    621   
   583    622   Makefile: $(TOP)/Makefile.in
................................................................................
   593    632   libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
   594    633   	$(LTLINK) -no-undefined -o $@ tclsqlite.lo \
   595    634   		libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \
   596    635   		-rpath "$(TCLLIBDIR)" \
   597    636   		-version-info "8:6:8" \
   598    637   		-avoid-version
   599    638   
   600         -sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
          639  +sqlite3$(TEXE):	shell.c sqlite3.c
   601    640   	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
   602         -		$(TOP)/src/shell.c sqlite3.c \
          641  +		shell.c sqlite3.c \
   603    642   		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
   604    643   
   605    644   sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
   606    645   	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
   607    646   
   608    647   dbhash$(TEXE):	$(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
   609    648   	$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)
................................................................................
   625    664   fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
   626    665   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
   627    666   
   628    667   ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
   629    668   	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
   630    669                $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
   631    670   
          671  +sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
          672  +	$(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
          673  +
   632    674   dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
   633    675   	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
   634    676   
          677  +DBFUZZ2_OPTS = \
          678  +  -DSQLITE_THREADSAFE=0 \
          679  +  -DSQLITE_OMIT_LOAD_EXTENSION \
          680  +  -DSQLITE_ENABLE_DESERIALIZE \
          681  +  -DSQLITE_DEBUG \
          682  +  -DSQLITE_ENABLE_DBSTAT_VTAB \
          683  +  -DSQLITE_ENABLE_RTREE \
          684  +  -DSQLITE_ENABLE_FTS4 \
          685  +  -DSQLITE_EANBLE_FTS5
          686  +
          687  +dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
          688  +	clang-6.0 -I. -g -O0 -fsanitize=fuzzer,undefined,address -o dbfuzz2 \
          689  +		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
          690  +	mkdir -p dbfuzz2-dir
          691  +	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
          692  +
   635    693   mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
   636    694   	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
   637    695   		$(TLIBS) -rpath "$(libdir)"
   638    696   
   639    697   MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
   640    698   MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20
   641    699   mptest:	mptester$(TEXE)
................................................................................
   664    722   	$(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl $(OPTS) <tsrc/vdbe.c >vdbe.new
   665    723   	mv vdbe.new tsrc/vdbe.c
   666    724   	cp fts5.c fts5.h tsrc
   667    725   	touch .target_source
   668    726   
   669    727   sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
   670    728   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
   671         -	cp tsrc/shell.c tsrc/sqlite3ext.h .
          729  +	cp tsrc/sqlite3ext.h .
   672    730   	cp $(TOP)/ext/session/sqlite3session.h .
   673    731   
   674    732   sqlite3ext.h:	.target_source
   675    733   	cp tsrc/sqlite3ext.h .
   676    734   
   677    735   tclsqlite3.c:	sqlite3.c
   678    736   	echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
................................................................................
   748    806   
   749    807   ctime.lo:	$(TOP)/src/ctime.c $(HDR)
   750    808   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/ctime.c
   751    809   
   752    810   date.lo:	$(TOP)/src/date.c $(HDR)
   753    811   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c
   754    812   
          813  +dbpage.lo:	$(TOP)/src/dbpage.c $(HDR)
          814  +	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c
          815  +
   755    816   dbstat.lo:	$(TOP)/src/dbstat.c $(HDR)
   756    817   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c
   757    818   
   758    819   delete.lo:	$(TOP)/src/delete.c $(HDR)
   759    820   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/delete.c
   760    821   
   761    822   expr.lo:	$(TOP)/src/expr.c $(HDR)
................................................................................
   802    863   
   803    864   mem3.lo:	$(TOP)/src/mem3.c $(HDR)
   804    865   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem3.c
   805    866   
   806    867   mem5.lo:	$(TOP)/src/mem5.c $(HDR)
   807    868   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem5.c
   808    869   
          870  +memdb.lo:	$(TOP)/src/memdb.c $(HDR)
          871  +	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memdb.c
          872  +
   809    873   memjournal.lo:	$(TOP)/src/memjournal.c $(HDR)
   810    874   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memjournal.c
   811    875   
   812    876   mutex.lo:	$(TOP)/src/mutex.c $(HDR)
   813    877   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex.c
   814    878   
   815    879   mutex_noop.lo:	$(TOP)/src/mutex_noop.c $(HDR)
................................................................................
   880    944   
   881    945   trigger.lo:	$(TOP)/src/trigger.c $(HDR)
   882    946   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/trigger.c
   883    947   
   884    948   update.lo:	$(TOP)/src/update.c $(HDR)
   885    949   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c
   886    950   
          951  +upsert.lo:	$(TOP)/src/upsert.c $(HDR)
          952  +	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
          953  +
   887    954   utf.lo:	$(TOP)/src/utf.c $(HDR)
   888    955   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c
   889    956   
   890    957   util.lo:	$(TOP)/src/util.c $(HDR)
   891    958   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/util.c
   892    959   
   893    960   vacuum.lo:	$(TOP)/src/vacuum.c $(HDR)
................................................................................
   928    995   
   929    996   wherecode.lo:	$(TOP)/src/wherecode.c $(HDR)
   930    997   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/wherecode.c
   931    998   
   932    999   whereexpr.lo:	$(TOP)/src/whereexpr.c $(HDR)
   933   1000   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c
   934   1001   
         1002  +window.lo:	$(TOP)/src/window.c $(HDR)
         1003  +	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/window.c
         1004  +
   935   1005   tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
   936   1006   	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c
   937   1007   
   938   1008   tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
   939         -	$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c
         1009  +	$(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c
   940   1010   
   941   1011   tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
   942   1012   	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
   943   1013   
   944   1014   tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la
   945   1015   	$(LTLINK) -o $@ tclsqlite-shell.lo \
   946   1016   		 libsqlite3.la $(LIBTCL)
................................................................................
   966   1036   
   967   1037   sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
   968   1038   	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
   969   1039   
   970   1040   keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
   971   1041   	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
   972   1042   	./mkkeywordhash$(BEXE) >keywordhash.h
         1043  +
         1044  +# Source files that go into making shell.c
         1045  +SHELL_SRC = \
         1046  +	$(TOP)/src/shell.c.in \
         1047  +        $(TOP)/ext/misc/appendvfs.c \
         1048  +	$(TOP)/ext/misc/shathree.c \
         1049  +	$(TOP)/ext/misc/fileio.c \
         1050  +	$(TOP)/ext/misc/completion.c \
         1051  +	$(TOP)/ext/misc/sqlar.c \
         1052  +	$(TOP)/ext/expert/sqlite3expert.c \
         1053  +	$(TOP)/ext/expert/sqlite3expert.h \
         1054  +	$(TOP)/ext/misc/zipfile.c \
         1055  +        $(TOP)/src/test_windirent.c
         1056  +
         1057  +shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
         1058  +	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
         1059  +
   973   1060   
   974   1061   
   975   1062   
   976   1063   # Rules to build the extension objects.
   977   1064   #
   978   1065   icu.lo:	$(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
   979   1066   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
................................................................................
  1034   1121   
  1035   1122   fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
  1036   1123   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
  1037   1124   
  1038   1125   rtree.lo:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
  1039   1126   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
  1040   1127   
  1041         -sqlite3session.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
         1128  +sqlite3session.lo:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
         1129  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
         1130  +
         1131  +userauth.lo:	$(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR)
  1042   1132   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c
  1043   1133   
  1044   1134   json1.lo:	$(TOP)/ext/misc/json1.c
  1045   1135   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
  1046   1136   
  1047   1137   stmt.lo:	$(TOP)/ext/misc/stmt.c
  1048   1138   	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c
................................................................................
  1062   1152      fts5parse.c fts5parse.h \
  1063   1153      $(TOP)/ext/fts5/fts5_storage.c \
  1064   1154      $(TOP)/ext/fts5/fts5_tokenize.c \
  1065   1155      $(TOP)/ext/fts5/fts5_unicode2.c \
  1066   1156      $(TOP)/ext/fts5/fts5_varint.c \
  1067   1157      $(TOP)/ext/fts5/fts5_vocab.c  \
  1068   1158   
  1069         -fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
         1159  +fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
  1070   1160   	cp $(TOP)/ext/fts5/fts5parse.y .
  1071   1161   	rm -f fts5parse.h
  1072   1162   	./lemon$(BEXE) $(OPTS) fts5parse.y
  1073   1163   
  1074   1164   fts5parse.h: fts5parse.c
  1075   1165   
  1076   1166   fts5.c: $(FTS5_SRC)
................................................................................
  1087   1177   # Rules to build the 'testfixture' application.
  1088   1178   #
  1089   1179   # If using the amalgamation, use sqlite3.c directly to build the test
  1090   1180   # fixture.  Otherwise link against libsqlite3.la.  (This distinction is
  1091   1181   # necessary because the test fixture requires non-API symbols which are
  1092   1182   # hidden when the library is built via the amalgamation).
  1093   1183   #
  1094         -TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  1095         -TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
         1184  +TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
         1185  +TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
         1186  +TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
  1096   1187   TESTFIXTURE_FLAGS += -DBUILD_sqlite
  1097   1188   TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  1098   1189   TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
  1099   1190   TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
         1191  +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
  1100   1192   
  1101   1193   TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
  1102   1194   TESTFIXTURE_SRC1 = sqlite3.c
  1103   1195   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
  1104   1196   TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
  1105   1197   
  1106   1198   testfixture$(TEXE):	$(TESTFIXTURE_SRC)
  1107   1199   	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
  1108   1200   		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
  1109   1201   
         1202  +coretestprogs:	$(TESTPROGS)
         1203  +
         1204  +testprogs:	coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
         1205  +
  1110   1206   # A very detailed test running most or all test cases
  1111   1207   fulltest:	$(TESTPROGS) fuzztest
  1112   1208   	./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
  1113   1209   
  1114   1210   # Really really long testing
  1115   1211   soaktest:	$(TESTPROGS)
  1116   1212   	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 $(TESTOPTS)
  1117   1213   
  1118   1214   # Do extra testing but not everything.
  1119   1215   fulltestonly:	$(TESTPROGS) fuzztest
  1120   1216   	./testfixture$(TEXE) $(TOP)/test/full.test
  1121   1217   
  1122   1218   # Fuzz testing
  1123         -fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
         1219  +fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
  1124   1220   	./fuzzcheck$(TEXE) $(FUZZDATA)
         1221  +	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
  1125   1222   
  1126         -fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
         1223  +fastfuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
  1127   1224   	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
         1225  +	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
  1128   1226   
  1129         -valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
         1227  +valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
  1130   1228   	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
         1229  +	valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
  1131   1230   
  1132   1231   # The veryquick.test TCL tests.
  1133   1232   #
  1134   1233   tcltest:	./testfixture$(TEXE)
  1135   1234   	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
  1136   1235   
  1137   1236   # Minimal testing that runs in less than 3 minutes
................................................................................
  1153   1252   # A very fast test that checks basic sanity.  The name comes from
  1154   1253   # the 60s-era electronics testing:  "Turn it on and see if smoke
  1155   1254   # comes out."
  1156   1255   #
  1157   1256   smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
  1158   1257   	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
  1159   1258   
  1160         -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
  1161         -	echo "#define TCLSH 2" > $@
  1162         -	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
  1163         -	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
  1164         -	echo "static const char *tclsh_main_loop(void){" >> $@
  1165         -	echo "static const char *zMainloop = " >> $@
  1166         -	$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
  1167         -	echo "; return zMainloop; }" >> $@
         1259  +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
         1260  +	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
  1168   1261   
  1169   1262   sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
  1170   1263   	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
         1264  +
         1265  +sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
         1266  +	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
         1267  +
         1268  +sqltclsh$(TEXE): sqltclsh.c
         1269  +	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
         1270  +
         1271  +sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
         1272  +	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
         1273  +
         1274  +CHECKER_DEPS =\
         1275  +  $(TOP)/tool/mkccode.tcl \
         1276  +  sqlite3.c \
         1277  +  $(TOP)/src/tclsqlite.c \
         1278  +  $(TOP)/ext/repair/sqlite3_checker.tcl \
         1279  +  $(TOP)/ext/repair/checkindex.c \
         1280  +  $(TOP)/ext/repair/checkfreelist.c \
         1281  +  $(TOP)/ext/misc/btreeinfo.c \
         1282  +  $(TOP)/ext/repair/sqlite3_checker.c.in
         1283  +
         1284  +sqlite3_checker.c:	$(CHECKER_DEPS)
         1285  +	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
         1286  +
         1287  +sqlite3_checker$(TEXE):	sqlite3_checker.c
         1288  +	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
  1171   1289   
  1172   1290   dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
  1173   1291   	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
  1174   1292              $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)
  1175   1293   
  1176   1294   showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
  1177   1295   	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)
................................................................................
  1181   1299   
  1182   1300   showjournal$(TEXE):	$(TOP)/tool/showjournal.c sqlite3.lo
  1183   1301   	$(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS)
  1184   1302   
  1185   1303   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1186   1304   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1187   1305   
         1306  +showshm$(TEXE):	$(TOP)/tool/showshm.c
         1307  +	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
         1308  +
  1188   1309   changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
  1189   1310   	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
  1190   1311   
         1312  +changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
         1313  +	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
         1314  +
  1191   1315   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo
  1192   1316   	$(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS)
  1193   1317   
         1318  +atrc$(TEXX): $(TOP)/test/atrc.c sqlite3.lo
         1319  +	$(LTLINK) -o $@ $(TOP)/test/atrc.c sqlite3.lo $(TLIBS)
         1320  +
  1194   1321   LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
  1195   1322   	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
  1196   1323   
  1197   1324   wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
  1198   1325   	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)
  1199   1326   
  1200   1327   speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
................................................................................
  1201   1328   	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)
  1202   1329   
  1203   1330   KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
  1204   1331   
  1205   1332   kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
  1206   1333   	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
  1207   1334   
  1208         -rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
         1335  +rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
  1209   1336   	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
  1210   1337   
  1211   1338   loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
  1212   1339   	$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
  1213   1340   
  1214   1341   # This target will fail if the SQLite amalgamation contains any exported
  1215   1342   # symbols that do not begin with "sqlite3_". It is run as part of the
................................................................................
  1229   1356   
  1230   1357   snapshot-tarball: sqlite3.c
  1231   1358   	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
  1232   1359   
  1233   1360   # The next two rules are used to support the "threadtest" target. Building
  1234   1361   # threadtest runs a few thread-safety tests that are implemented in C. This
  1235   1362   # target is invoked by the releasetest.tcl script.
  1236         -# 
         1363  +#
  1237   1364   THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
  1238   1365                     $(TOP)/test/tt3_checkpoint.c \
  1239   1366                     $(TOP)/test/tt3_index.c      \
  1240   1367                     $(TOP)/test/tt3_vacuum.c      \
  1241   1368                     $(TOP)/test/tt3_stress.c      \
  1242   1369                     $(TOP)/test/tt3_lookaside1.c
  1243   1370   
  1244   1371   threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
  1245   1372   	$(LTLINK) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.lo -o $@ $(TLIBS)
  1246   1373   
  1247   1374   threadtest: threadtest3$(TEXE)
  1248   1375   	./threadtest3$(TEXE)
  1249   1376   
  1250         -releasetest:	
         1377  +releasetest:
  1251   1378   	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl
  1252   1379   
  1253   1380   # Standard install and cleanup targets
  1254   1381   #
  1255   1382   lib_install:	libsqlite3.la
  1256   1383   	$(INSTALL) -d $(DESTDIR)$(libdir)
  1257   1384   	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
  1258         -	
         1385  +
  1259   1386   install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
  1260   1387   	$(INSTALL) -d $(DESTDIR)$(bindir)
  1261   1388   	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
  1262   1389   	$(INSTALL) -d $(DESTDIR)$(includedir)
  1263   1390   	$(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir)
  1264   1391   	$(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir)
  1265   1392   	$(INSTALL) -d $(DESTDIR)$(pkgconfigdir)
................................................................................
  1269   1396   	echo 'package ifneeded sqlite3 $(RELEASE) [list load $(TCLLIBDIR)/libtclsqlite3$(SHLIB_SUFFIX) sqlite3]' > $@
  1270   1397   tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
  1271   1398   	$(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)
  1272   1399   	$(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)
  1273   1400   	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
  1274   1401   	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)
  1275   1402   
  1276         -clean:	
         1403  +clean:
  1277   1404   	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
  1278   1405   	rm -f sqlite3.h opcodes.*
  1279   1406   	rm -rf .libs .deps
  1280   1407   	rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
  1281   1408   	rm -f mkkeywordhash$(BEXE) keywordhash.h
  1282   1409   	rm -f *.da *.bb *.bbg gmon.out
  1283   1410   	rm -rf tsrc .target_source

Changes to Makefile.msc.

    88     88   # be used for debugging with Visual Studio.
    89     89   #
    90     90   !IFNDEF SPLIT_AMALGAMATION
    91     91   SPLIT_AMALGAMATION = 0
    92     92   !ENDIF
    93     93   
    94     94   # <<mark>>
           95  +# Set this non-0 to have this makefile assume the Tcl shell executable
           96  +# (tclsh*.exe) is available in the PATH.  By default, this is disabled
           97  +# for compatibility with older build environments.  This setting only
           98  +# applies if TCLSH_CMD is not set manually.
           99  +#
          100  +!IFNDEF USE_TCLSH_IN_PATH
          101  +USE_TCLSH_IN_PATH = 0
          102  +!ENDIF
          103  +
          104  +# Set this non-0 to use zlib, possibly compiling it from source code.
          105  +#
          106  +!IFNDEF USE_ZLIB
          107  +USE_ZLIB = 0
          108  +!ENDIF
          109  +
          110  +# Set this non-0 to build zlib from source code.  This is enabled by
          111  +# default and in that case it will be assumed that the ZLIBDIR macro
          112  +# points to the top-level source code directory for zlib.
          113  +#
          114  +!IFNDEF BUILD_ZLIB
          115  +BUILD_ZLIB = 1
          116  +!ENDIF
          117  +
    95    118   # Set this non-0 to use the International Components for Unicode (ICU).
    96    119   #
    97    120   !IFNDEF USE_ICU
    98    121   USE_ICU = 0
    99    122   !ENDIF
   100    123   # <</mark>>
   101    124   
................................................................................
   312    335   # These are the "standard" SQLite compilation options used when compiling for
   313    336   # the Windows platform.
   314    337   #
   315    338   !IFNDEF OPT_FEATURE_FLAGS
   316    339   !IF $(MINIMAL_AMALGAMATION)==0
   317    340   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
   318    341   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
          342  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
          343  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
          344  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
          345  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
          346  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
          347  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
          348  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
   319    349   !ENDIF
   320    350   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   321    351   !ENDIF
   322    352   
   323    353   # Should the session extension be enabled?  If so, add compilation options
   324    354   # to enable it.
   325    355   #
................................................................................
   593    623   
   594    624   # This is the source code that the shell executable should be compiled
   595    625   # with.
   596    626   #
   597    627   !IFNDEF SHELL_CORE_SRC
   598    628   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   599    629   SHELL_CORE_SRC =
          630  +# <<mark>>
          631  +!ELSEIF $(USE_AMALGAMATION)==0
          632  +SHELL_CORE_SRC =
          633  +# <</mark>>
   600    634   !ELSE
   601    635   SHELL_CORE_SRC = $(SQLITE3C)
   602    636   !ENDIF
   603    637   !ENDIF
   604    638   
   605    639   # This is the core library that the shell executable should depend on.
   606    640   #
   607    641   !IFNDEF SHELL_CORE_DEP
   608    642   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   609    643   SHELL_CORE_DEP = $(SQLITE3DLL)
          644  +# <<mark>>
          645  +!ELSEIF $(USE_AMALGAMATION)==0
          646  +SHELL_CORE_DEP = libsqlite3.lib
          647  +# <</mark>>
   610    648   !ELSE
   611    649   SHELL_CORE_DEP =
   612    650   !ENDIF
   613    651   !ENDIF
   614    652   
          653  +# <<mark>>
          654  +# If zlib support is enabled, add the dependencies for it.
          655  +#
          656  +!IF $(USE_ZLIB)!=0 && $(BUILD_ZLIB)!=0
          657  +SHELL_CORE_DEP = zlib $(SHELL_CORE_DEP)
          658  +TESTFIXTURE_DEP = zlib $(TESTFIXTURE_DEP)
          659  +!ENDIF
          660  +# <</mark>>
          661  +
   615    662   # This is the core library that the shell executable should link with.
   616    663   #
   617    664   !IFNDEF SHELL_CORE_LIB
   618    665   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   619    666   SHELL_CORE_LIB = $(SQLITE3LIB)
          667  +# <<mark>>
          668  +!ELSEIF $(USE_AMALGAMATION)==0
          669  +SHELL_CORE_LIB = libsqlite3.lib
          670  +# <</mark>>
   620    671   !ELSE
   621    672   SHELL_CORE_LIB =
   622    673   !ENDIF
   623    674   !ENDIF
   624    675   
   625    676   # These are additional linker options used for the shell executable.
   626    677   #
................................................................................
   798    849   # <<mark>>
   799    850   # The locations of the Tcl header and library files.  Also, the library that
   800    851   # non-stubs enabled programs using Tcl must link against.  These variables
   801    852   # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
   802    853   # prior to running nmake in order to match the actual installed location and
   803    854   # version on this machine.
   804    855   #
          856  +!IFNDEF TCLDIR
          857  +TCLDIR = $(TOP)\compat\tcl
          858  +!ENDIF
          859  +
   805    860   !IFNDEF TCLINCDIR
   806         -TCLINCDIR = c:\tcl\include
          861  +TCLINCDIR = $(TCLDIR)\include
   807    862   !ENDIF
   808    863   
   809    864   !IFNDEF TCLLIBDIR
   810         -TCLLIBDIR = c:\tcl\lib
          865  +TCLLIBDIR = $(TCLDIR)\lib
   811    866   !ENDIF
   812    867   
   813    868   !IFNDEF LIBTCL
   814    869   LIBTCL = tcl86.lib
   815    870   !ENDIF
   816    871   
   817    872   !IFNDEF LIBTCLSTUB
   818    873   LIBTCLSTUB = tclstub86.lib
   819    874   !ENDIF
   820    875   
   821    876   !IFNDEF LIBTCLPATH
   822         -LIBTCLPATH = c:\tcl\bin
          877  +LIBTCLPATH = $(TCLDIR)\bin
          878  +!ENDIF
          879  +
          880  +# The locations of the zlib header and library files.  These variables
          881  +# (ZLIBINCDIR, ZLIBLIBDIR, and ZLIBLIB) may be overridden via the environment
          882  +# prior to running nmake in order to match the actual installed (or source
          883  +# code) location on this machine.
          884  +#
          885  +!IFNDEF ZLIBDIR
          886  +ZLIBDIR = $(TOP)\compat\zlib
          887  +!ENDIF
          888  +
          889  +!IFNDEF ZLIBINCDIR
          890  +ZLIBINCDIR = $(ZLIBDIR)
          891  +!ENDIF
          892  +
          893  +!IFNDEF ZLIBLIBDIR
          894  +ZLIBLIBDIR = $(ZLIBDIR)
          895  +!ENDIF
          896  +
          897  +!IFNDEF ZLIBLIB
          898  +!IF $(DYNAMIC_SHELL)!=0
          899  +ZLIBLIB = zdll.lib
          900  +!ELSE
          901  +ZLIBLIB = zlib.lib
          902  +!ENDIF
   823    903   !ENDIF
   824    904   
   825    905   # The locations of the ICU header and library files.  These variables
   826    906   # (ICUINCDIR, ICULIBDIR, and LIBICU) may be overridden via the environment
   827    907   # prior to running nmake in order to match the actual installed location on
   828    908   # this machine.
   829    909   #
          910  +!IFNDEF ICUDIR
          911  +ICUDIR = $(TOP)\compat\icu
          912  +!ENDIF
          913  +
   830    914   !IFNDEF ICUINCDIR
   831         -ICUINCDIR = c:\icu\include
          915  +ICUINCDIR = $(ICUDIR)\include
   832    916   !ENDIF
   833    917   
   834    918   !IFNDEF ICULIBDIR
   835         -ICULIBDIR = c:\icu\lib
          919  +ICULIBDIR = $(ICUDIR)\lib
   836    920   !ENDIF
   837    921   
   838    922   !IFNDEF LIBICU
   839    923   LIBICU = icuuc.lib icuin.lib
   840    924   !ENDIF
   841    925   
   842    926   # This is the command to use for tclsh - normally just "tclsh", but we may
   843    927   # know the specific version we want to use.  This variable (TCLSH_CMD) may be
   844    928   # overridden via the environment prior to running nmake in order to select a
   845    929   # specific Tcl shell to use.
   846    930   #
   847    931   !IFNDEF TCLSH_CMD
          932  +!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe")
   848    933   TCLSH_CMD = tclsh
          934  +!ELSE
          935  +TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
          936  +!ENDIF
   849    937   !ENDIF
   850    938   # <</mark>>
   851    939   
   852    940   # Compiler options needed for programs that use the readline() library.
   853    941   #
   854    942   !IFNDEF READLINE_FLAGS
   855    943   READLINE_FLAGS = -DHAVE_READLINE=0
................................................................................
   947   1035   #
   948   1036   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   949   1037   TCC = $(TCC) -Zi
   950   1038   BCC = $(BCC) -Zi
   951   1039   !ENDIF
   952   1040   
   953   1041   # <<mark>>
         1042  +# If zlib support is enabled, add the compiler options for it.
         1043  +#
         1044  +!IF $(USE_ZLIB)!=0
         1045  +TCC = $(TCC) -DSQLITE_HAVE_ZLIB=1
         1046  +RCC = $(RCC) -DSQLITE_HAVE_ZLIB=1
         1047  +TCC = $(TCC) -I$(ZLIBINCDIR)
         1048  +RCC = $(RCC) -I$(ZLIBINCDIR)
         1049  +!ENDIF
         1050  +
   954   1051   # If ICU support is enabled, add the compiler options for it.
   955   1052   #
   956   1053   !IF $(USE_ICU)!=0
   957   1054   TCC = $(TCC) -DSQLITE_ENABLE_ICU=1
   958   1055   RCC = $(RCC) -DSQLITE_ENABLE_ICU=1
   959   1056   TCC = $(TCC) -I$(TOP)\ext\icu
   960   1057   RCC = $(RCC) -I$(TOP)\ext\icu
................................................................................
   970   1067   LTRCOMPILE = $(RCC) -r
   971   1068   LTLIB = lib.exe
   972   1069   LTLINK = $(TCC) -Fe$@
   973   1070   
   974   1071   # If requested, link to the RPCRT4 library.
   975   1072   #
   976   1073   !IF $(USE_RPCRT4_LIB)!=0
   977         -LTLINK = $(LTLINK) rpcrt4.lib
         1074  +LTLIBS = $(LTLIBS) rpcrt4.lib
   978   1075   !ENDIF
   979   1076   
   980   1077   # If a platform was set, force the linker to target that.
   981   1078   # Note that the vcvars*.bat family of batch files typically
   982   1079   # set this for you.  Otherwise, the linker will attempt
   983   1080   # to deduce the binary type based on the object files.
   984   1081   !IFDEF PLATFORM
................................................................................
  1067   1164   LDFLAGS = $(LDOPTS)
  1068   1165   !ENDIF
  1069   1166   
  1070   1167   # <<mark>>
  1071   1168   # Start with the Tcl related linker options.
  1072   1169   #
  1073   1170   !IF $(NO_TCL)==0
  1074         -LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
  1075         -LTLIBS = $(LIBTCL)
         1171  +TCLLIBPATHS = $(TCLLIBPATHS) /LIBPATH:$(TCLLIBDIR)
         1172  +TCLLIBS = $(TCLLIBS) $(LIBTCL)
         1173  +!ENDIF
         1174  +
         1175  +# If zlib support is enabled, add the linker options for it.
         1176  +#
         1177  +!IF $(USE_ZLIB)!=0
         1178  +LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ZLIBLIBDIR)
         1179  +LTLIBS = $(LTLIBS) $(ZLIBLIB)
  1076   1180   !ENDIF
  1077   1181   
  1078   1182   # If ICU support is enabled, add the linker options for it.
  1079   1183   #
  1080   1184   !IF $(USE_ICU)!=0
  1081   1185   LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR)
  1082   1186   LTLIBS = $(LTLIBS) $(LIBICU)
................................................................................
  1087   1191   ###############################################################################
  1088   1192   
  1089   1193   # <<mark>>
  1090   1194   # Object files for the SQLite library (non-amalgamation).
  1091   1195   #
  1092   1196   LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
  1093   1197            backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
  1094         -         callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \
         1198  +         callback.lo complete.lo ctime.lo \
         1199  +         date.lo dbpage.lo dbstat.lo delete.lo \
  1095   1200            expr.lo fault.lo fkey.lo \
  1096   1201            fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
  1097   1202            fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
  1098   1203            fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
  1099   1204            fts5.lo \
  1100   1205            func.lo global.lo hash.lo \
  1101         -         icu.lo insert.lo legacy.lo loadext.lo \
         1206  +         icu.lo insert.lo json1.lo legacy.lo loadext.lo \
  1102   1207            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
  1103         -         memjournal.lo \
         1208  +         memdb.lo memjournal.lo \
  1104   1209            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
  1105   1210            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
  1106   1211            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
  1107   1212            random.lo resolve.lo rowset.lo rtree.lo \
  1108         -         sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
         1213  +         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
  1109   1214            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
  1110         -         update.lo util.lo vacuum.lo \
         1215  +         update.lo upsert.lo util.lo vacuum.lo \
  1111   1216            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
  1112   1217            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
  1113         -         utf.lo vtab.lo
         1218  +         window.lo utf.lo vtab.lo
  1114   1219   # <</mark>>
  1115   1220   
  1116   1221   # Object files for the amalgamation.
  1117   1222   #
  1118   1223   LIBOBJS1 = sqlite3.lo
  1119   1224   
  1120   1225   # Determine the real value of LIBOBJ based on the 'configure' script
................................................................................
  1150   1255     $(TOP)\src\btmutex.c \
  1151   1256     $(TOP)\src\btree.c \
  1152   1257     $(TOP)\src\build.c \
  1153   1258     $(TOP)\src\callback.c \
  1154   1259     $(TOP)\src\complete.c \
  1155   1260     $(TOP)\src\ctime.c \
  1156   1261     $(TOP)\src\date.c \
         1262  +  $(TOP)\src\dbpage.c \
  1157   1263     $(TOP)\src\dbstat.c \
  1158   1264     $(TOP)\src\delete.c \
  1159   1265     $(TOP)\src\expr.c \
  1160   1266     $(TOP)\src\fault.c \
  1161   1267     $(TOP)\src\fkey.c \
  1162   1268     $(TOP)\src\func.c \
  1163   1269     $(TOP)\src\global.c \
................................................................................
  1168   1274     $(TOP)\src\main.c \
  1169   1275     $(TOP)\src\malloc.c \
  1170   1276     $(TOP)\src\mem0.c \
  1171   1277     $(TOP)\src\mem1.c \
  1172   1278     $(TOP)\src\mem2.c \
  1173   1279     $(TOP)\src\mem3.c \
  1174   1280     $(TOP)\src\mem5.c \
         1281  +  $(TOP)\src\memdb.c \
  1175   1282     $(TOP)\src\memjournal.c \
  1176   1283     $(TOP)\src\mutex.c \
  1177   1284     $(TOP)\src\mutex_noop.c \
  1178   1285     $(TOP)\src\mutex_unix.c \
  1179   1286     $(TOP)\src\mutex_w32.c \
  1180   1287     $(TOP)\src\notify.c \
  1181   1288     $(TOP)\src\os.c \
................................................................................
  1200   1307     $(TOP)\src\threads.c \
  1201   1308     $(TOP)\src\tclsqlite.c \
  1202   1309     $(TOP)\src\tokenize.c \
  1203   1310     $(TOP)\src\treeview.c \
  1204   1311     $(TOP)\src\trigger.c \
  1205   1312     $(TOP)\src\utf.c \
  1206   1313     $(TOP)\src\update.c \
         1314  +  $(TOP)\src\upsert.c \
  1207   1315     $(TOP)\src\util.c \
  1208   1316     $(TOP)\src\vacuum.c \
  1209   1317     $(TOP)\src\vdbe.c \
  1210   1318     $(TOP)\src\vdbeapi.c \
  1211   1319     $(TOP)\src\vdbeaux.c \
  1212   1320     $(TOP)\src\vdbeblob.c \
  1213   1321     $(TOP)\src\vdbemem.c \
................................................................................
  1214   1322     $(TOP)\src\vdbesort.c \
  1215   1323     $(TOP)\src\vdbetrace.c \
  1216   1324     $(TOP)\src\vtab.c \
  1217   1325     $(TOP)\src\wal.c \
  1218   1326     $(TOP)\src\walker.c \
  1219   1327     $(TOP)\src\where.c \
  1220   1328     $(TOP)\src\wherecode.c \
  1221         -  $(TOP)\src\whereexpr.c
  1222         -
  1223         -# Shell source code files.
  1224         -#
  1225         -SRC02 = \
  1226         -  $(TOP)\src\shell.c
         1329  +  $(TOP)\src\whereexpr.c \
         1330  +  $(TOP)\src\window.c
  1227   1331   
  1228   1332   # Core miscellaneous files.
  1229   1333   #
  1230   1334   SRC03 = \
  1231   1335     $(TOP)\src\parse.y
  1232   1336   
  1233   1337   # Core header files, part 1.
................................................................................
  1312   1416   SRC09 = \
  1313   1417     $(TOP)\ext\fts3\fts3.h \
  1314   1418     $(TOP)\ext\fts3\fts3Int.h \
  1315   1419     $(TOP)\ext\fts3\fts3_hash.h \
  1316   1420     $(TOP)\ext\fts3\fts3_tokenizer.h \
  1317   1421     $(TOP)\ext\icu\sqliteicu.h \
  1318   1422     $(TOP)\ext\rtree\rtree.h \
         1423  +  $(TOP)\ext\rtree\geopoly.c \
  1319   1424     $(TOP)\ext\rbu\sqlite3rbu.h \
  1320   1425     $(TOP)\ext\session\sqlite3session.h
  1321   1426   
  1322   1427   # Generated source code files
  1323   1428   #
  1324   1429   SRC10 = \
  1325   1430     opcodes.c \
................................................................................
  1327   1432   
  1328   1433   # Generated header files
  1329   1434   #
  1330   1435   SRC11 = \
  1331   1436     keywordhash.h \
  1332   1437     opcodes.h \
  1333   1438     parse.h \
         1439  +  shell.c \
  1334   1440     $(SQLITE3H)
  1335   1441   
  1336   1442   # Generated Tcl header files
  1337   1443   #
  1338   1444   !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
  1339   1445   SRC12 = \
  1340   1446     $(SQLITETCLH) \
................................................................................
  1341   1447     $(SQLITETCLDECLSH)
  1342   1448   !ELSE
  1343   1449   SRC12 =
  1344   1450   !ENDIF
  1345   1451   
  1346   1452   # All source code files.
  1347   1453   #
  1348         -SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
         1454  +SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11)
  1349   1455   
  1350   1456   # Source code to the test files.
  1351   1457   #
  1352   1458   TESTSRC = \
  1353   1459     $(TOP)\src\test1.c \
  1354   1460     $(TOP)\src\test2.c \
  1355   1461     $(TOP)\src\test3.c \
................................................................................
  1372   1478     $(TOP)\src\test_fs.c \
  1373   1479     $(TOP)\src\test_func.c \
  1374   1480     $(TOP)\src\test_hexio.c \
  1375   1481     $(TOP)\src\test_init.c \
  1376   1482     $(TOP)\src\test_intarray.c \
  1377   1483     $(TOP)\src\test_journal.c \
  1378   1484     $(TOP)\src\test_malloc.c \
         1485  +  $(TOP)\src\test_md5.c \
  1379   1486     $(TOP)\src\test_multiplex.c \
  1380   1487     $(TOP)\src\test_mutex.c \
  1381   1488     $(TOP)\src\test_onefile.c \
  1382   1489     $(TOP)\src\test_osinst.c \
  1383   1490     $(TOP)\src\test_pcache.c \
  1384   1491     $(TOP)\src\test_quota.c \
  1385   1492     $(TOP)\src\test_rtree.c \
  1386   1493     $(TOP)\src\test_schema.c \
  1387   1494     $(TOP)\src\test_server.c \
  1388   1495     $(TOP)\src\test_superlock.c \
  1389   1496     $(TOP)\src\test_syscall.c \
         1497  +  $(TOP)\src\test_tclsh.c \
  1390   1498     $(TOP)\src\test_tclvar.c \
  1391   1499     $(TOP)\src\test_thread.c \
  1392   1500     $(TOP)\src\test_vfs.c \
  1393   1501     $(TOP)\src\test_windirent.c \
         1502  +  $(TOP)\src\test_window.c \
  1394   1503     $(TOP)\src\test_wsd.c \
  1395   1504     $(TOP)\ext\fts3\fts3_term.c \
  1396   1505     $(TOP)\ext\fts3\fts3_test.c \
  1397   1506     $(TOP)\ext\rbu\test_rbu.c \
  1398   1507     $(TOP)\ext\session\test_session.c
  1399   1508   
  1400   1509   # Statically linked extensions.
  1401   1510   #
  1402   1511   TESTEXT = \
         1512  +  $(TOP)\ext\expert\sqlite3expert.c \
         1513  +  $(TOP)\ext\expert\test_expert.c \
  1403   1514     $(TOP)\ext\misc\amatch.c \
  1404   1515     $(TOP)\ext\misc\carray.c \
  1405   1516     $(TOP)\ext\misc\closure.c \
  1406   1517     $(TOP)\ext\misc\csv.c \
  1407   1518     $(TOP)\ext\misc\eval.c \
         1519  +  $(TOP)\ext\misc\explain.c \
  1408   1520     $(TOP)\ext\misc\fileio.c \
  1409   1521     $(TOP)\ext\misc\fuzzer.c \
  1410   1522     $(TOP)\ext\fts5\fts5_tcl.c \
  1411   1523     $(TOP)\ext\fts5\fts5_test_mi.c \
  1412   1524     $(TOP)\ext\fts5\fts5_test_tok.c \
  1413   1525     $(TOP)\ext\misc\ieee754.c \
  1414   1526     $(TOP)\ext\misc\mmapwarm.c \
  1415   1527     $(TOP)\ext\misc\nextchar.c \
         1528  +  $(TOP)\ext\misc\normalize.c \
  1416   1529     $(TOP)\ext\misc\percentile.c \
  1417   1530     $(TOP)\ext\misc\regexp.c \
  1418   1531     $(TOP)\ext\misc\remember.c \
  1419   1532     $(TOP)\ext\misc\series.c \
  1420   1533     $(TOP)\ext\misc\spellfix.c \
  1421   1534     $(TOP)\ext\misc\totype.c \
  1422   1535     $(TOP)\ext\misc\unionvtab.c \
  1423   1536     $(TOP)\ext\misc\wholenumber.c
         1537  +
         1538  +# If use of zlib is enabled, add the "zipfile.c" source file.
         1539  +#
         1540  +!IF $(USE_ZLIB)!=0
         1541  +TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
         1542  +!ENDIF
  1424   1543   
  1425   1544   # Source code to the library files needed by the test fixture
  1426   1545   # (non-amalgamation)
  1427   1546   #
  1428   1547   TESTSRC2 = \
  1429   1548     $(SRC00) \
  1430   1549     $(SRC01) \
................................................................................
  1473   1592     $(TOP)\ext\fts2\fts2_tokenizer.h
  1474   1593   EXTHDR = $(EXTHDR) \
  1475   1594     $(TOP)\ext\fts3\fts3.h \
  1476   1595     $(TOP)\ext\fts3\fts3Int.h \
  1477   1596     $(TOP)\ext\fts3\fts3_hash.h \
  1478   1597     $(TOP)\ext\fts3\fts3_tokenizer.h
  1479   1598   EXTHDR = $(EXTHDR) \
  1480         -  $(TOP)\ext\rtree\rtree.h
         1599  +  $(TOP)\ext\rtree\rtree.h \
         1600  +  $(TOP)\ext\rtree\geopoly.c
  1481   1601   EXTHDR = $(EXTHDR) \
  1482   1602     $(TOP)\ext\icu\sqliteicu.h
  1483   1603   EXTHDR = $(EXTHDR) \
  1484   1604     $(TOP)\ext\rtree\sqlite3rtree.h
  1485   1605   EXTHDR = $(EXTHDR) \
  1486   1606     $(TOP)\ext\session\sqlite3session.h
  1487   1607   
  1488   1608   # executables needed for testing
  1489   1609   #
  1490   1610   TESTPROGS = \
  1491   1611     testfixture.exe \
  1492   1612     $(SQLITE3EXE) \
  1493   1613     sqlite3_analyzer.exe \
         1614  +  sqlite3_checker.exe \
  1494   1615     sqldiff.exe \
  1495         -  dbhash.exe
         1616  +  dbhash.exe \
         1617  +  sqltclsh.exe
  1496   1618   
  1497   1619   # Databases containing fuzzer test cases
  1498   1620   #
  1499   1621   FUZZDATA = \
  1500   1622     $(TOP)\test\fuzzdata1.db \
  1501   1623     $(TOP)\test\fuzzdata2.db \
  1502   1624     $(TOP)\test\fuzzdata3.db \
  1503   1625     $(TOP)\test\fuzzdata4.db \
  1504         -  $(TOP)\test\fuzzdata5.db
         1626  +  $(TOP)\test\fuzzdata5.db \
         1627  +  $(TOP)\test\fuzzdata6.db \
         1628  +  $(TOP)\test\fuzzdata7.db
  1505   1629   # <</mark>>
  1506   1630   
  1507   1631   # Additional compiler options for the shell.  These are only effective
  1508   1632   # when the shell is not being dynamically linked.
  1509   1633   #
  1510   1634   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1511         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
         1635  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
         1636  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
         1637  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
         1638  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
  1512   1639   !ENDIF
  1513   1640   
  1514   1641   # <<mark>>
  1515   1642   # Extra compiler options for various test tools.
  1516   1643   #
  1517         -MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
         1644  +MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
  1518   1645   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
  1519         -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
         1646  +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
  1520   1647   FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
  1521   1648   OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
  1522   1649   DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
  1523   1650   KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
  1524         -DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
  1525   1651   ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0
  1526   1652   
  1527   1653   # Standard options to testfixture.
  1528   1654   #
  1529   1655   TESTOPTS = --verbose=file --output=test-out.txt
  1530   1656   
  1531   1657   # Extra targets for the "all" target that require Tcl.
................................................................................
  1536   1662   ALL_TCL_TARGETS =
  1537   1663   !ENDIF
  1538   1664   # <</mark>>
  1539   1665   
  1540   1666   # This is the default Makefile target.  The objects listed here
  1541   1667   # are what get build when you type just "make" with no arguments.
  1542   1668   #
  1543         -all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
         1669  +core:	dll libsqlite3.lib shell
         1670  +
         1671  +# Targets that require the Tcl library.
         1672  +#
         1673  +tcl:	$(ALL_TCL_TARGETS)
         1674  +
         1675  +# This Makefile target builds all of the standard binaries.
         1676  +#
         1677  +all:	core tcl
  1544   1678   
  1545   1679   # Dynamic link library section.
  1546   1680   #
  1547   1681   dll:	$(SQLITE3DLL)
  1548   1682   
  1549   1683   # Shell executable.
  1550   1684   #
................................................................................
  1561   1695   $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
  1562   1696   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1563   1697   
  1564   1698   # <<block2>>
  1565   1699   sqlite3.def:	libsqlite3.lib
  1566   1700   	echo EXPORTS > sqlite3.def
  1567   1701   	dumpbin /all libsqlite3.lib \
  1568         -		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
         1702  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@]*)(?:@\d+)?$$" \1 \
  1569   1703   		| sort >> sqlite3.def
  1570   1704   # <</block2>>
  1571   1705   
  1572         -$(SQLITE3EXE):	$(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
  1573         -	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
         1706  +$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
         1707  +	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
  1574   1708   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1575   1709   
  1576   1710   # <<mark>>
  1577   1711   sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
  1578   1712   	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1579   1713   
  1580   1714   dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
  1581   1715   	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1582   1716   
  1583   1717   scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
  1584         -	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1718  +	$(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1585   1719   
  1586   1720   srcck1.exe:	$(TOP)\tool\srcck1.c
  1587   1721   	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
  1588   1722   
  1589         -sourcetest:	srcck1.exe sqlite3.c
  1590         -	srcck1.exe sqlite3.c
         1723  +sourcetest:	srcck1.exe $(SQLITE3C)
         1724  +	srcck1.exe $(SQLITE3C)
  1591   1725   
  1592   1726   fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
  1593   1727   	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1594   1728   
  1595   1729   dbfuzz.exe:	$(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
  1596   1730   	$(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1597   1731   
  1598   1732   fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
  1599   1733   	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1600   1734   
  1601   1735   ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
  1602   1736   	$(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1603   1737   
         1738  +sessionfuzz.exe:	zlib $(TOP)\test\sessionfuzz.c $(SQLITE3C) $(SQLITE3H)
         1739  +	$(LTLINK) $(NO_WARN) -I$(ZLIBINCDIR) $(TOP)\test\sessionfuzz.c /link $(LDFLAGS) $(LTLINKOPTS) /LIBPATH:$(ZLIBLIBDIR) $(ZLIBLIB)
         1740  +
  1604   1741   mptester.exe:	$(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H)
  1605   1742   	$(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1606   1743   
  1607   1744   MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20
  1608   1745   MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20
  1609   1746   
  1610   1747   mptest:	mptester.exe
................................................................................
  1625   1762   # all that automatic generation.
  1626   1763   #
  1627   1764   .target_source:	$(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP)
  1628   1765   	-rmdir /Q/S tsrc 2>NUL
  1629   1766   	-mkdir tsrc
  1630   1767   	for %i in ($(SRC00)) do copy /Y %i tsrc
  1631   1768   	for %i in ($(SRC01)) do copy /Y %i tsrc
  1632         -	for %i in ($(SRC02)) do copy /Y %i tsrc
  1633   1769   	for %i in ($(SRC03)) do copy /Y %i tsrc
  1634   1770   	for %i in ($(SRC04)) do copy /Y %i tsrc
  1635   1771   	for %i in ($(SRC05)) do copy /Y %i tsrc
  1636   1772   	for %i in ($(SRC06)) do copy /Y %i tsrc
  1637   1773   	for %i in ($(SRC07)) do copy /Y %i tsrc
  1638   1774   	for %i in ($(SRC08)) do copy /Y %i tsrc
  1639   1775   	for %i in ($(SRC09)) do copy /Y %i tsrc
................................................................................
  1645   1781   	del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL
  1646   1782   	$(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new
  1647   1783   	move vdbe.new tsrc\vdbe.c
  1648   1784   	echo > .target_source
  1649   1785   
  1650   1786   sqlite3.c:	.target_source sqlite3ext.h $(MKSQLITE3C_TOOL)
  1651   1787   	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
  1652         -	copy tsrc\shell.c .
  1653   1788   	copy $(TOP)\ext\session\sqlite3session.h .
  1654   1789   
  1655   1790   sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
  1656   1791   	$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
  1657   1792   # <</mark>>
  1658   1793   
  1659   1794   # Rule to build the amalgamation
................................................................................
  1743   1878   
  1744   1879   ctime.lo:	$(TOP)\src\ctime.c $(HDR)
  1745   1880   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\ctime.c
  1746   1881   
  1747   1882   date.lo:	$(TOP)\src\date.c $(HDR)
  1748   1883   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c
  1749   1884   
  1750         -dbstat.lo:	$(TOP)\src\date.c $(HDR)
         1885  +dbpage.lo:	$(TOP)\src\dbpage.c $(HDR)
         1886  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbpage.c
         1887  +
         1888  +dbstat.lo:	$(TOP)\src\dbstat.c $(HDR)
  1751   1889   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\dbstat.c
  1752   1890   
  1753   1891   delete.lo:	$(TOP)\src\delete.c $(HDR)
  1754   1892   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\delete.c
  1755   1893   
  1756   1894   expr.lo:	$(TOP)\src\expr.c $(HDR)
  1757   1895   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\expr.c
................................................................................
  1797   1935   
  1798   1936   mem3.lo:	$(TOP)\src\mem3.c $(HDR)
  1799   1937   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\mem3.c
  1800   1938   
  1801   1939   mem5.lo:	$(TOP)\src\mem5.c $(HDR)
  1802   1940   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\mem5.c
  1803   1941   
         1942  +memdb.lo:	$(TOP)\src\memdb.c $(HDR)
         1943  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memdb.c
         1944  +
  1804   1945   memjournal.lo:	$(TOP)\src\memjournal.c $(HDR)
  1805   1946   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\memjournal.c
  1806   1947   
  1807   1948   mutex.lo:	$(TOP)\src\mutex.c $(HDR)
  1808   1949   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\mutex.c
  1809   1950   
  1810   1951   mutex_noop.lo:	$(TOP)\src\mutex_noop.c $(HDR)
................................................................................
  1875   2016   
  1876   2017   trigger.lo:	$(TOP)\src\trigger.c $(HDR)
  1877   2018   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\trigger.c
  1878   2019   
  1879   2020   update.lo:	$(TOP)\src\update.c $(HDR)
  1880   2021   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\update.c
  1881   2022   
         2023  +upsert.lo:	$(TOP)\src\upsert.c $(HDR)
         2024  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\upsert.c
         2025  +
  1882   2026   utf.lo:	$(TOP)\src\utf.c $(HDR)
  1883   2027   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\utf.c
  1884   2028   
  1885   2029   util.lo:	$(TOP)\src\util.c $(HDR)
  1886   2030   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\util.c
  1887   2031   
  1888   2032   vacuum.lo:	$(TOP)\src\vacuum.c $(HDR)
................................................................................
  1923   2067   
  1924   2068   wherecode.lo:	$(TOP)\src\wherecode.c $(HDR)
  1925   2069   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\wherecode.c
  1926   2070   
  1927   2071   whereexpr.lo:	$(TOP)\src\whereexpr.c $(HDR)
  1928   2072   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c
  1929   2073   
         2074  +window.lo:	$(TOP)\src\window.c $(HDR)
         2075  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\window.c
         2076  +
  1930   2077   tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1931   2078   	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1932   2079   
  1933   2080   tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1934         -	$(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
         2081  +	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1935   2082   
  1936   2083   tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
  1937         -	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2084  +	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  1938   2085   
  1939   2086   # Rules to build opcodes.c and opcodes.h
  1940   2087   #
  1941   2088   opcodes.c:	opcodes.h $(TOP)\tool\mkopcodec.tcl
  1942   2089   	$(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c
  1943   2090   
  1944   2091   opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\tool\mkopcodeh.tcl
................................................................................
  1970   2117   mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
  1971   2118   	$(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \
  1972   2119   		$(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS)
  1973   2120   
  1974   2121   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
  1975   2122   	.\mkkeywordhash.exe > keywordhash.h
  1976   2123   
         2124  +# Source files that go into making shell.c
         2125  +SHELL_SRC = \
         2126  +	$(TOP)\src\shell.c.in \
         2127  +	$(TOP)\ext\misc\appendvfs.c \
         2128  +	$(TOP)\ext\misc\shathree.c \
         2129  +	$(TOP)\ext\misc\fileio.c \
         2130  +	$(TOP)\ext\misc\completion.c \
         2131  +	$(TOP)\ext\expert\sqlite3expert.c \
         2132  +	$(TOP)\ext\expert\sqlite3expert.h \
         2133  +	$(TOP)\src\test_windirent.c
  1977   2134   
         2135  +# If use of zlib is enabled, add the "zipfile.c" source file.
         2136  +#
         2137  +!IF $(USE_ZLIB)!=0
         2138  +SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
         2139  +SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
         2140  +!ENDIF
         2141  +
         2142  +shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
         2143  +	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
         2144  +
         2145  +zlib:
         2146  +	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd
  1978   2147   
  1979   2148   # Rules to build the extension objects.
  1980   2149   #
  1981   2150   icu.lo:	$(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
  1982   2151   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c
  1983   2152   
  1984   2153   fts2.lo:	$(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR)
................................................................................
  2034   2203   
  2035   2204   fts3_unicode2.lo:	$(TOP)\ext\fts3\fts3_unicode2.c $(HDR) $(EXTHDR)
  2036   2205   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_unicode2.c
  2037   2206   
  2038   2207   fts3_write.lo:	$(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR)
  2039   2208   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c
  2040   2209   
         2210  +json1.lo:	$(TOP)\ext\misc\json1.c $(HDR) $(EXTHDR)
         2211  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\json1.c
         2212  +
         2213  +stmt.lo:	$(TOP)\ext\misc\stmt.c $(HDR) $(EXTHDR)
         2214  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\misc\stmt.c
         2215  +
  2041   2216   rtree.lo:	$(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR)
  2042   2217   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c
  2043   2218   
  2044   2219   sqlite3session.lo:	$(TOP)\ext\session\sqlite3session.c $(HDR) $(EXTHDR)
  2045   2220   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c
  2046   2221   
  2047   2222   # FTS5 things
................................................................................
  2059   2234      fts5parse.c fts5parse.h \
  2060   2235      $(TOP)\ext\fts5\fts5_storage.c \
  2061   2236      $(TOP)\ext\fts5\fts5_tokenize.c \
  2062   2237      $(TOP)\ext\fts5\fts5_unicode2.c \
  2063   2238      $(TOP)\ext\fts5\fts5_varint.c \
  2064   2239      $(TOP)\ext\fts5\fts5_vocab.c
  2065   2240   
         2241  +LSM1_SRC = \
         2242  +   $(TOP)\ext\lsm1\lsm.h \
         2243  +   $(TOP)\ext\lsm1\lsmInt.h \
         2244  +   $(TOP)\ext\lsm1\lsm_ckpt.c \
         2245  +   $(TOP)\ext\lsm1\lsm_file.c \
         2246  +   $(TOP)\ext\lsm1\lsm_log.c \
         2247  +   $(TOP)\ext\lsm1\lsm_main.c \
         2248  +   $(TOP)\ext\lsm1\lsm_mem.c \
         2249  +   $(TOP)\ext\lsm1\lsm_mutex.c \
         2250  +   $(TOP)\ext\lsm1\lsm_shared.c \
         2251  +   $(TOP)\ext\lsm1\lsm_sorted.c \
         2252  +   $(TOP)\ext\lsm1\lsm_str.c \
         2253  +   $(TOP)\ext\lsm1\lsm_tree.c \
         2254  +   $(TOP)\ext\lsm1\lsm_unix.c \
         2255  +   $(TOP)\ext\lsm1\lsm_varint.c \
         2256  +   $(TOP)\ext\lsm1\lsm_vtab.c \
         2257  +   $(TOP)\ext\lsm1\lsm_win32.c
         2258  +
  2066   2259   fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
  2067   2260   	copy $(TOP)\ext\fts5\fts5parse.y .
  2068   2261   	del /Q fts5parse.h 2>NUL
  2069   2262   	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y
  2070   2263   
  2071   2264   fts5parse.h:	fts5parse.c
  2072   2265   
  2073   2266   fts5.c:	$(FTS5_SRC)
  2074   2267   	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
  2075   2268   	copy $(TOP)\ext\fts5\fts5.h .
  2076   2269   
         2270  +lsm1.c:	$(LSM1_SRC)
         2271  +	$(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl
         2272  +	copy $(TOP)\ext\lsm1\lsm.h .
         2273  +
  2077   2274   fts5.lo:	fts5.c $(HDR) $(EXTHDR)
  2078   2275   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
  2079   2276   
  2080   2277   fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
  2081   2278   	$(LTCOMPILE) $(NO_WARN) -c fts5.c
  2082   2279   
  2083   2280   fts5.dll:	fts5_ext.lo
................................................................................
  2089   2286   # Rules to build the 'testfixture' application.
  2090   2287   #
  2091   2288   # If using the amalgamation, use sqlite3.c directly to build the test
  2092   2289   # fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
  2093   2290   # necessary because the test fixture requires non-API symbols which are
  2094   2291   # hidden when the library is built via the amalgamation).
  2095   2292   #
  2096         -TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
         2293  +TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  2097   2294   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  2098   2295   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
  2099   2296   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  2100   2297   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
  2101         -TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
         2298  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
         2299  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
         2300  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
  2102   2301   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
  2103   2302   
  2104   2303   TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  2105   2304   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  2106   2305   !IF $(USE_AMALGAMATION)==0
  2107   2306   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  2108   2307   !ELSE
................................................................................
  2125   2324   	type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \
  2126   2325   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \
  2127   2326   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \
  2128   2327   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \
  2129   2328   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
  2130   2329   !ENDIF
  2131   2330   
  2132         -testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
         2331  +testfixture.exe:	$(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
  2133   2332   	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
  2134   2333   		-DBUILD_sqlite -I$(TCLINCDIR) \
  2135   2334   		$(TESTFIXTURE_SRC) \
  2136         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2335  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  2137   2336   
  2138   2337   extensiontest:	testfixture.exe testloadext.dll
  2139   2338   	@set PATH=$(LIBTCLPATH);$(PATH)
  2140   2339   	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
  2141   2340   
         2341  +coretestprogs:	$(TESTPROGS)
         2342  +
         2343  +testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
         2344  +
  2142   2345   fulltest:	$(TESTPROGS) fuzztest
  2143   2346   	@set PATH=$(LIBTCLPATH);$(PATH)
  2144   2347   	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
  2145   2348   
  2146   2349   soaktest:	$(TESTPROGS)
  2147   2350   	@set PATH=$(LIBTCLPATH);$(PATH)
  2148   2351   	.\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS)
................................................................................
  2174   2377   	@set PATH=$(LIBTCLPATH);$(PATH)
  2175   2378   	.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)
  2176   2379   
  2177   2380   smoketest:	$(TESTPROGS)
  2178   2381   	@set PATH=$(LIBTCLPATH);$(PATH)
  2179   2382   	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
  2180   2383   
  2181         -sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP)
  2182         -	echo #define TCLSH 2 > $@
  2183         -	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
  2184         -	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
  2185         -	echo static const char *tclsh_main_loop(void){ >> $@
  2186         -	echo static const char *zMainloop = >> $@
  2187         -	$(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@
  2188         -	echo ; return zMainloop; } >> $@
         2384  +sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
         2385  +	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
  2189   2386   
  2190   2387   sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
  2191   2388   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
  2192         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2389  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  2193   2390   
  2194         -dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
         2391  +sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
         2392  +	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
         2393  +
         2394  +sqltclsh.exe: sqltclsh.c  $(SHELL_CORE_DEP) $(LIBRESOBJS)
         2395  +	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
         2396  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
         2397  +
         2398  +sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
         2399  +	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
         2400  +
         2401  +CHECKER_DEPS =\
         2402  +  $(TOP)/tool/mkccode.tcl \
         2403  +  sqlite3.c \
         2404  +  $(TOP)/src/tclsqlite.c \
         2405  +  $(TOP)/ext/repair/sqlite3_checker.tcl \
         2406  +  $(TOP)/ext/repair/checkindex.c \
         2407  +  $(TOP)/ext/repair/checkfreelist.c \
         2408  +  $(TOP)/ext/misc/btreeinfo.c \
         2409  +  $(TOP)/ext/repair/sqlite3_checker.c.in
         2410  +
         2411  +sqlite3_checker.c:	$(CHECKER_DEPS)
         2412  +	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
         2413  +
         2414  +sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
         2415  +	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
         2416  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
         2417  +
         2418  +dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
  2195   2419   	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
  2196   2420   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
  2197   2421   
  2198         -testloadext.lo:	$(TOP)\src\test_loadext.c
         2422  +testloadext.lo:	$(TOP)\src\test_loadext.c $(SQLITE3H)
  2199   2423   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
  2200   2424   
  2201   2425   testloadext.dll:	testloadext.lo
  2202   2426   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo
  2203   2427   
  2204   2428   showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
  2205   2429   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
................................................................................
  2213   2437   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2214   2438   		$(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2215   2439   
  2216   2440   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  2217   2441   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2218   2442   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2219   2443   
         2444  +showshm.exe:	$(TOP)\tool\showshm.c
         2445  +	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
         2446  +
  2220   2447   changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
  2221   2448   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2222   2449   		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
  2223   2450   		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2224   2451   
         2452  +changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
         2453  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2454  +		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
         2455  +		$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2456  +
  2225   2457   fts3view.exe:	$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H)
  2226   2458   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2227   2459   		$(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2228   2460   
  2229   2461   rollback-test.exe:	$(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H)
  2230   2462   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2231   2463   		$(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2232   2464   
         2465  +atrc.exe:	$(TOP)\test\atrc.c $(SQLITE3C) $(SQLITE3H)
         2466  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2467  +		$(TOP)\test\atrc.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2468  +
  2233   2469   LogEst.exe:	$(TOP)\tool\logest.c $(SQLITE3H)
  2234   2470   	$(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS)
  2235   2471   
  2236   2472   wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H)
  2237   2473   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2238   2474   		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2239   2475   
................................................................................
  2241   2477   	$(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \
  2242   2478   		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2243   2479   
  2244   2480   kvtest.exe:	$(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H)
  2245   2481   	$(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \
  2246   2482   		$(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2247   2483   
  2248         -dbselftest.exe:	$(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H)
  2249         -	$(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C)
  2250         -
  2251   2484   rbu.exe:	$(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H)
  2252   2485   	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \
  2253   2486   		$(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2254   2487   
  2255   2488   LSMDIR=$(TOP)\ext\lsm1
  2256   2489   !INCLUDE $(LSMDIR)\Makefile.msc
  2257   2490   
................................................................................
  2260   2493   # <</mark>>
  2261   2494   
  2262   2495   clean:
  2263   2496   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  2264   2497   	del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL
  2265   2498   	del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL
  2266   2499   # <<mark>>
  2267         -	del /Q sqlite3.c sqlite3.h 2>NUL
  2268   2500   	del /Q opcodes.c opcodes.h 2>NUL
  2269   2501   	del /Q lemon.* lempar.c parse.* 2>NUL
  2270   2502   	del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL
  2271   2503   	del /Q notasharedlib.* 2>NUL
  2272   2504   	-rmdir /Q/S .deps 2>NUL
  2273   2505   	-rmdir /Q/S .libs 2>NUL
  2274   2506   	-rmdir /Q/S tsrc 2>NUL
................................................................................
  2277   2509   	del /Q lsm.dll lsmtest.exe 2>NUL
  2278   2510   	del /Q testloadext.dll 2>NUL
  2279   2511   	del /Q testfixture.exe test.db 2>NUL
  2280   2512   	del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL
  2281   2513   	del /Q changeset.exe 2>NUL
  2282   2514   	del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL
  2283   2515   	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2284         -	del /Q sqlite3.c sqlite3-*.c 2>NUL
         2516  +	del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL
  2285   2517   	del /Q sqlite3rc.h 2>NUL
  2286   2518   	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
  2287   2519   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  2288   2520   	del /Q sqlite-*-output.vsix 2>NUL
  2289   2521   	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
         2522  +	del /Q sqltclsh.* 2>NUL
         2523  +	del /Q dbfuzz.exe sessionfuzz.exe 2>NUL
         2524  +	del /Q kvtest.exe ossshell.exe scrub.exe 2>NUL
         2525  +	del /Q showshm.exe sqlite3_checker.* sqlite3_expert.exe 2>NUL
  2290   2526   	del /Q fts5.* fts5parse.* 2>NUL
         2527  +	del /Q lsm.h lsm1.c 2>NUL
  2291   2528   # <</mark>>

Changes to README.md.

     1      1   <h1 align="center">SQLite Source Repository</h1>
     2      2   
     3         -This repository contains the complete source code for the SQLite database
     4         -engine.  Some test scripts are also include.  However, many other test scripts
            3  +This repository contains the complete source code for the 
            4  +[SQLite database engine](https://sqlite.org/).  Some test scripts 
            5  +are also included.  However, many other test scripts
     5      6   and most of the documentation are managed separately.
     6      7   
     7         -If you are reading this on a Git mirror someplace, you are doing it wrong.
     8         -The [official repository](https://www.sqlite.org/src/) is better.  Go there
     9         -now.
            8  +SQLite [does not use Git](https://sqlite.org/whynotgit.html).
            9  +If you are reading this on GitHub, then you are looking at an
           10  +unofficial mirror. See <https://sqlite.org/src> for the official
           11  +repository.
    10     12   
    11     13   ## Obtaining The Code
    12     14   
    13     15   SQLite sources are managed using the
    14     16   [Fossil](https://www.fossil-scm.org/), a distributed version control system
    15     17   that was specifically designed to support SQLite development.
    16     18   If you do not want to use Fossil, you can download tarballs or ZIP
    17         -archives as follows:
           19  +archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
    18     20   
    19         -  *  Lastest trunk check-in:
    20         -     <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or
    21         -     <https://www.sqlite.org/src/zip/sqlite.zip>.
           21  +  *  Lastest trunk check-in as
           22  +     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
           23  +     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
           24  +     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).
    22     25   
    23         -  *  Latest release:
    24         -     <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or
    25         -     <https://www.sqlite.org/src/zip/sqlite.zip?r=release>.
           26  +  *  Latest release as
           27  +     [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release),
           28  +     [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or
           29  +     [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release).
    26     30   
    27     31     *  For other check-ins, substitute an appropriate branch name or
    28     32        tag or hash prefix for "release" in the URLs of the previous
    29     33        bullet.  Or browse the [timeline](https://www.sqlite.org/src/timeline)
    30     34        to locate the check-in desired, click on its information page link,
    31     35        then click on the "Tarball" or "ZIP Archive" links on the information
    32     36        page.
................................................................................
   100    104   to the "sqlite3.dll" command line above.  When debugging into the SQLite
   101    105   code, adding the "DEBUG=1" argument to one of the above command lines is
   102    106   recommended.
   103    107   
   104    108   SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
   105    109   is required by the makefiles (including those for MSVC).  SQLite contains
   106    110   a lot of generated code and Tcl is used to do much of that code generation.
   107         -The makefiles also require AWK.
   108    111   
   109    112   ## Source Code Tour
   110    113   
   111    114   Most of the core source files are in the **src/** subdirectory.  The
   112    115   **src/** folder also contains files used to build the "testfixture" test
   113    116   harness. The names of the source files used by "testfixture" all begin
   114    117   with "test".
   115    118   The **src/** also contains the "shell.c" file
   116    119   which is the main program for the "sqlite3.exe"
   117    120   [command-line shell](https://sqlite.org/cli.html) and
   118    121   the "tclsqlite.c" file which implements the
   119         -[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite.
          122  +[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
   120    123   (Historical note:  SQLite began as a Tcl
   121    124   extension and only later escaped to the wild as an independent library.)
   122    125   
   123    126   Test scripts and programs are found in the **test/** subdirectory.
   124    127   Addtional test code is found in other source repositories.
   125    128   See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
   126    129   additional information.
................................................................................
   159    162   the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
   160    163   by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
   161    164   for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
   162    165   template for generating its parser.
   163    166   
   164    167   Lemon also generates the **parse.h** header file, at the same time it
   165    168   generates parse.c. But the parse.h header file is
   166         -modified further (to add additional symbols) using the ./addopcodes.awk
   167         -AWK script.
          169  +modified further (to add additional symbols) using the ./addopcodes.tcl
          170  +Tcl script.
   168    171   
   169    172   The **opcodes.h** header file contains macros that define the numbers
   170    173   corresponding to opcodes in the "VDBE" virtual machine.  The opcodes.h
   171    174   file is generated by the scanning the src/vdbe.c source file.  The
   172         -AWK script at ./mkopcodeh.awk does this scan and generates opcodes.h.
   173         -A second AWK script, ./mkopcodec.awk, then scans opcodes.h to generate
          175  +Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
          176  +A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
   174    177   the **opcodes.c** source file, which contains a reverse mapping from
   175    178   opcode-number to opcode-name that is used for EXPLAIN output.
   176    179   
   177    180   The **keywordhash.h** header file contains the definition of a hash table
   178    181   that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into
   179    182   the numeric codes used by the parse.c parser.  The keywordhash.h file is
   180    183   generated by a C-language program at tool mkkeywordhash.c.
................................................................................
   203    206   tool/mksqlite3c.tcl script is run to copy them all together in just the
   204    207   right order while resolving internal "#include" references.
   205    208   
   206    209   The amalgamation source file is more than 200K lines long.  Some symbolic
   207    210   debuggers (most notably MSVC) are unable to deal with files longer than 64K
   208    211   lines.  To work around this, a separate Tcl script, tool/split-sqlite3c.tcl,
   209    212   can be run on the amalgamation to break it up into a single small C file
   210         -called **sqlite3-all.c** that does #include on about five other files
   211         -named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**.  In this way,
          213  +called **sqlite3-all.c** that does #include on about seven other files
          214  +named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**.  In this way,
   212    215   all of the source code is contained within a single translation unit so
   213    216   that the compiler can do extra cross-procedure optimization, but no
   214    217   individual source file exceeds 32K lines in length.
   215    218   
   216    219   ## How It All Fits Together
   217    220   
   218    221   SQLite is modular in design.
................................................................................
   233    236   Key files:
   234    237   
   235    238     *  **sqlite.h.in** - This file defines the public interface to the SQLite
   236    239        library.  Readers will need to be familiar with this interface before
   237    240        trying to understand how the library works internally.
   238    241   
   239    242     *  **sqliteInt.h** - this header file defines many of the data objects
   240         -     used internally by SQLite.
          243  +     used internally by SQLite.  In addition to "sqliteInt.h", some
          244  +     subsystems have their own header files.
   241    245   
   242    246     *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
   243    247        to parse SQL statements, and the actions that are taken at each step
   244    248        in the parsing process.
   245    249   
   246    250     *  **vdbe.c** - This file implements the virtual machine that runs
   247    251        prepared statements.  There are various helper files whose names
   248    252        begin with "vdbe".  The VDBE has access to the vdbeInt.h header file
   249    253        which defines internal data objects.  The rest of SQLite interacts
   250    254        with the VDBE through an interface defined by vdbe.h.
   251    255   
   252         -  *  **where.c** - This file analyzes the WHERE clause and generates
          256  +  *  **where.c** - This file (together with its helper files named
          257  +     by "where*.c") analyzes the WHERE clause and generates
   253    258        virtual machine code to run queries efficiently.  This file is
   254    259        sometimes called the "query optimizer".  It has its own private
   255    260        header file, whereInt.h, that defines data objects used internally.
   256    261   
   257    262     *  **btree.c** - This file contains the implementation of the B-Tree
   258         -     storage engine used by SQLite.
          263  +     storage engine used by SQLite.  The interface to the rest of the system
          264  +     is defined by "btree.h".  The "btreeInt.h" header defines objects
          265  +     used internally by btree.c and not published to the rest of the system.
   259    266   
   260    267     *  **pager.c** - This file contains the "pager" implementation, the
   261         -     module that implements transactions.
          268  +     module that implements transactions.  The "pager.h" header file
          269  +     defines the interface between pager.c and the rest of the system.
   262    270   
   263    271     *  **os_unix.c** and **os_win.c** - These two files implement the interface
   264    272        between SQLite and the underlying operating system using the run-time
   265    273        pluggable VFS interface.
   266    274   
   267         -  *  **shell.c** - This file is not part of the core SQLite library.  This
          275  +  *  **shell.c.in** - This file is not part of the core SQLite library.  This
   268    276        is the file that, when linked against sqlite3.a, generates the
   269         -     "sqlite3.exe" command-line shell.
          277  +     "sqlite3.exe" command-line shell.  The "shell.c.in" file is transformed
          278  +     into "shell.c" as part of the build process.
   270    279   
   271    280     *  **tclsqlite.c** - This file implements the Tcl bindings for SQLite.  It
   272    281        is not part of the core SQLite library.  But as most of the tests in this
   273    282        repository are written in Tcl, the Tcl language bindings are important.
          283  +
          284  +  *  **test*.c** - Files in the src/ folder that begin with "test" go into
          285  +     building the "testfixture.exe" program.  The testfixture.exe program is
          286  +     an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
          287  +     test/ folder to validate the core SQLite code.  The testfixture program
          288  +     (and some other test programs too) is build and run when you type
          289  +     "make test".
          290  +
          291  +  *  **ext/misc/json1.c** - This file implements the various JSON functions
          292  +     that are build into SQLite.
   274    293   
   275    294   There are many other source files.  Each has a succinct header comment that
   276    295   describes its purpose and role within the larger system.
   277    296   
   278    297   
   279    298   ## Contacts
   280    299   
   281    300   The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/)
   282    301   with geographically distributed backups at
   283    302   [http://www2.sqlite.org/](http://www2.sqlite.org) and
   284    303   [http://www3.sqlite.org/](http://www3.sqlite.org).

Changes to VERSION.

     1         -3.21.0
            1  +3.26.0

Changes to autoconf/Makefile.am.

     1      1   
     2         -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
     3         -
            2  +AM_CFLAGS = @BUILD_CFLAGS@ 
     4      3   lib_LTLIBRARIES = libsqlite3.la
     5      4   libsqlite3_la_SOURCES = sqlite3.c
     6      5   libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
     7      6   
     8      7   bin_PROGRAMS = sqlite3
     9      8   sqlite3_SOURCES = shell.c sqlite3.h
    10      9   EXTRA_sqlite3_SOURCES = sqlite3.c
    11     10   sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
    12     11   sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
    13         -sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
           12  +sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
    14     13   
    15     14   include_HEADERS = sqlite3.h sqlite3ext.h
    16     15   
    17         -EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
           16  +EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs Makefile.fallback
    18     17   pkgconfigdir = ${libdir}/pkgconfig
    19     18   pkgconfig_DATA = sqlite3.pc
    20     19   
    21     20   man_MANS = sqlite3.1

Added autoconf/Makefile.fallback.

            1  +#!/usr/bin/make
            2  +#
            3  +# If the configure script does not work, then this Makefile is available
            4  +# as a backup.  Manually configure the variables below.
            5  +#
            6  +# Note:  This makefile works out-of-the-box on MacOS 10.2 (Jaguar)
            7  +#
            8  +CC = gcc
            9  +CFLAGS = -O0 -I.
           10  +LIBS = -lz
           11  +COPTS += -D_BSD_SOURCE
           12  +COPTS += -DSQLITE_ENABLE_LOCKING_STYLE=0
           13  +COPTS += -DSQLITE_THREADSAFE=0
           14  +COPTS += -DSQLITE_OMIT_LOAD_EXTENSION
           15  +COPTS += -DSQLITE_WITHOUT_ZONEMALLOC
           16  +COPTS += -DSQLITE_ENABLE_RTREE
           17  +
           18  +sqlite3:	shell.c sqlite3.c
           19  +	$(CC) $(CFLAGS) $(COPTS) -o sqlite3 shell.c sqlite3.c $(LIBS)

Changes to autoconf/Makefile.msc.

   273    273   # These are the "standard" SQLite compilation options used when compiling for
   274    274   # the Windows platform.
   275    275   #
   276    276   !IFNDEF OPT_FEATURE_FLAGS
   277    277   !IF $(MINIMAL_AMALGAMATION)==0
   278    278   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
   279    279   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
          280  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
          281  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
          282  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
          283  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
          284  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
          285  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
   280    286   !ENDIF
   281    287   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   282    288   !ENDIF
   283    289   
   284    290   # Should the session extension be enabled?  If so, add compilation options
   285    291   # to enable it.
   286    292   #
................................................................................
   556    562   !IFNDEF SHELL_CORE_DEP
   557    563   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   558    564   SHELL_CORE_DEP = $(SQLITE3DLL)
   559    565   !ELSE
   560    566   SHELL_CORE_DEP =
   561    567   !ENDIF
   562    568   !ENDIF
          569  +
   563    570   
   564    571   # This is the core library that the shell executable should link with.
   565    572   #
   566    573   !IFNDEF SHELL_CORE_LIB
   567    574   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   568    575   SHELL_CORE_LIB = $(SQLITE3LIB)
   569    576   !ELSE
................................................................................
   804    811   LTRCOMPILE = $(RCC) -r
   805    812   LTLIB = lib.exe
   806    813   LTLINK = $(TCC) -Fe$@
   807    814   
   808    815   # If requested, link to the RPCRT4 library.
   809    816   #
   810    817   !IF $(USE_RPCRT4_LIB)!=0
   811         -LTLINK = $(LTLINK) rpcrt4.lib
          818  +LTLIBS = $(LTLIBS) rpcrt4.lib
   812    819   !ENDIF
   813    820   
   814    821   # If a platform was set, force the linker to target that.
   815    822   # Note that the vcvars*.bat family of batch files typically
   816    823   # set this for you.  Otherwise, the linker will attempt
   817    824   # to deduce the binary type based on the object files.
   818    825   !IFDEF PLATFORM
................................................................................
   923    930   !ENDIF
   924    931   
   925    932   
   926    933   # Additional compiler options for the shell.  These are only effective
   927    934   # when the shell is not being dynamically linked.
   928    935   #
   929    936   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   930         -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
          937  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
          938  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
          939  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
   931    940   !ENDIF
   932    941   
   933    942   
   934    943   # This is the default Makefile target.  The objects listed here
   935    944   # are what get build when you type just "make" with no arguments.
   936    945   #
   937         -all:	dll shell
          946  +core:	dll shell
          947  +
          948  +# Targets that require the Tcl library.
          949  +#
          950  +tcl:	$(ALL_TCL_TARGETS)
          951  +
          952  +# This Makefile target builds all of the standard binaries.
          953  +#
          954  +all:	core tcl
   938    955   
   939    956   # Dynamic link library section.
   940    957   #
   941    958   dll:	$(SQLITE3DLL)
   942    959   
   943    960   # Shell executable.
   944    961   #
................................................................................
   950    967   
   951    968   Replace.exe:
   952    969   	$(CSC) /target:exe $(TOP)\Replace.cs
   953    970   
   954    971   sqlite3.def:	Replace.exe $(LIBOBJ)
   955    972   	echo EXPORTS > sqlite3.def
   956    973   	dumpbin /all $(LIBOBJ) \
   957         -		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
          974  +		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
   958    975   		| sort >> sqlite3.def
   959    976   
   960         -$(SQLITE3EXE):	$(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
   961         -	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
          977  +$(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
          978  +	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
   962    979   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
   963    980   
   964    981   
   965    982   # Rule to build the amalgamation
   966    983   #
   967    984   sqlite3.lo:	$(SQLITE3C)
   968    985   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)
................................................................................
   969    986   
   970    987   
   971    988   # Rule to build the Win32 resources object file.
   972    989   #
   973    990   !IF $(USE_RC)!=0
   974    991   _HASHCHAR=^#
   975    992   !IF ![echo !IFNDEF VERSION > rcver.vc] && \
   976         -    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
          993  +    ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| "%SystemRoot%\System32\find.exe" "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
   977    994       ![echo !ENDIF >> rcver.vc]
   978    995   !INCLUDE rcver.vc
   979    996   !ENDIF
   980    997   
   981    998   RESOURCE_VERSION = $(VERSION:^#=)
   982    999   RESOURCE_VERSION = $(RESOURCE_VERSION:define=)
   983   1000   RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=)

Changes to autoconf/configure.ac.

     8      8   #   --enable-static-shell
     9      9   #   --enable-dynamic-extensions
    10     10   #
    11     11   
    12     12   AC_PREREQ(2.61)
    13     13   AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
    14     14   AC_CONFIG_SRCDIR([sqlite3.c])
           15  +AC_CONFIG_AUX_DIR([.])
    15     16   
    16     17   # Use automake.
    17     18   AM_INIT_AUTOMAKE([foreign])
    18     19   
    19     20   AC_SYS_LARGEFILE
    20     21   
    21     22   # Check for required programs.
................................................................................
    24     25   AC_PROG_MKDIR_P
    25     26   
    26     27   # Check for library functions that SQLite can optionally use.
    27     28   AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
    28     29   AC_FUNC_STRERROR_R
    29     30   
    30     31   AC_CONFIG_FILES([Makefile sqlite3.pc])
           32  +BUILD_CFLAGS=
    31     33   AC_SUBST(BUILD_CFLAGS)
    32     34   
    33     35   #-------------------------------------------------------------------------
    34     36   # Two options to enable readline compatible libraries: 
    35     37   #
    36     38   #   --enable-editline
    37     39   #   --enable-readline
................................................................................
    81     83   
    82     84   #-----------------------------------------------------------------------
    83     85   #   --enable-threadsafe
    84     86   #
    85     87   AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
    86     88     [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
    87     89     [], [enable_threadsafe=yes])
    88         -THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
    89     90   if test x"$enable_threadsafe" != "xno"; then
    90         -  THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
           91  +  BUILD_CFLAGS="$BUILD_CFLAGS -D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
    91     92     AC_SEARCH_LIBS(pthread_create, pthread)
    92     93     AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
    93     94   fi
    94         -AC_SUBST(THREADSAFE_FLAGS)
    95     95   #-----------------------------------------------------------------------
    96     96   
    97     97   #-----------------------------------------------------------------------
    98     98   #   --enable-dynamic-extensions
    99     99   #
   100    100   AC_ARG_ENABLE(dynamic-extensions, [AS_HELP_STRING(
   101    101     [--enable-dynamic-extensions], [support loadable extensions [default=yes]])], 
   102    102     [], [enable_dynamic_extensions=yes])
   103    103   if test x"$enable_dynamic_extensions" != "xno"; then
   104    104     AC_SEARCH_LIBS(dlopen, dl)
   105    105   else
   106         -  DYNAMIC_EXTENSION_FLAGS=-DSQLITE_OMIT_LOAD_EXTENSION=1
          106  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_OMIT_LOAD_EXTENSION=1"
   107    107   fi
   108    108   AC_MSG_CHECKING([for whether to support dynamic extensions])
   109    109   AC_MSG_RESULT($enable_dynamic_extensions)
   110         -AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
          110  +#-----------------------------------------------------------------------
          111  +
          112  +#-----------------------------------------------------------------------
          113  +#   --enable-fts4
          114  +#
          115  +AC_ARG_ENABLE(fts4, [AS_HELP_STRING(
          116  +  [--enable-fts4], [include fts4 support [default=yes]])], 
          117  +  [], [enable_fts4=yes])
          118  +if test x"$enable_fts4" = "xyes"; then
          119  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS4"
          120  +fi
          121  +#-----------------------------------------------------------------------
          122  +
          123  +#-----------------------------------------------------------------------
          124  +#   --enable-fts3
          125  +#
          126  +AC_ARG_ENABLE(fts3, [AS_HELP_STRING(
          127  +  [--enable-fts3], [include fts3 support [default=no]])], 
          128  +  [], [])
          129  +if test x"$enable_fts3" = "xyes" -a x"$enable_fts4" = "xno"; then
          130  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS3"
          131  +fi
   111    132   #-----------------------------------------------------------------------
   112    133   
   113    134   #-----------------------------------------------------------------------
   114    135   #   --enable-fts5
   115    136   #
   116    137   AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
   117         -  [--enable-fts5], [include fts5 support [default=no]])], 
   118         -  [], [enable_fts5=no])
          138  +  [--enable-fts5], [include fts5 support [default=yes]])], 
          139  +  [], [enable_fts5=yes])
   119    140   if test x"$enable_fts5" = "xyes"; then
   120    141     AC_SEARCH_LIBS(log, m)
   121         -  FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
          142  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS5"
   122    143   fi
   123         -AC_SUBST(FTS5_FLAGS)
   124    144   #-----------------------------------------------------------------------
   125    145   
   126    146   #-----------------------------------------------------------------------
   127    147   #   --enable-json1
   128    148   #
   129    149   AC_ARG_ENABLE(json1, [AS_HELP_STRING(
   130         -  [--enable-json1], [include json1 support [default=no]])], 
   131         -  [], [enable_json1=no])
          150  +  [--enable-json1], [include json1 support [default=yes]])], 
          151  +  [],[enable_json1=yes])
   132    152   if test x"$enable_json1" = "xyes"; then
   133         -  JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
          153  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1"
   134    154   fi
   135         -AC_SUBST(JSON1_FLAGS)
          155  +#-----------------------------------------------------------------------
          156  +
          157  +#-----------------------------------------------------------------------
          158  +#   --enable-rtree
          159  +#
          160  +AC_ARG_ENABLE(rtree, [AS_HELP_STRING(
          161  +  [--enable-rtree], [include rtree support [default=yes]])], 
          162  +  [], [enable_rtree=yes])
          163  +if test x"$enable_rtree" = "xyes"; then
          164  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE"
          165  +fi
   136    166   #-----------------------------------------------------------------------
   137    167   
   138    168   #-----------------------------------------------------------------------
   139    169   #   --enable-session
   140    170   #
   141    171   AC_ARG_ENABLE(session, [AS_HELP_STRING(
   142    172     [--enable-session], [enable the session extension [default=no]])], 
   143         -  [], [enable_session=no])
          173  +  [], [])
   144    174   if test x"$enable_session" = "xyes"; then
   145         -  SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
          175  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
   146    176   fi
   147         -AC_SUBST(SESSION_FLAGS)
          177  +#-----------------------------------------------------------------------
          178  +
          179  +#-----------------------------------------------------------------------
          180  +#   --enable-debug
          181  +#
          182  +AC_ARG_ENABLE(debug, [AS_HELP_STRING(
          183  +  [--enable-debug], [build with debugging features enabled [default=no]])], 
          184  +  [], [])
          185  +if test x"$enable_debug" = "xyes"; then
          186  +  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_DEBUG -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
          187  +  CFLAGS="-g -O0"
          188  +fi
   148    189   #-----------------------------------------------------------------------
   149    190   
   150    191   #-----------------------------------------------------------------------
   151    192   #   --enable-static-shell
   152    193   #
   153    194   AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
   154    195     [--enable-static-shell], 
................................................................................
   159    200   else
   160    201     EXTRA_SHELL_OBJ=libsqlite3.la
   161    202   fi
   162    203   AC_SUBST(EXTRA_SHELL_OBJ)
   163    204   #-----------------------------------------------------------------------
   164    205   
   165    206   AC_CHECK_FUNCS(posix_fallocate)
          207  +AC_CHECK_HEADERS(zlib.h,[
          208  +  AC_SEARCH_LIBS(deflate,z,[BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_HAVE_ZLIB"])
          209  +])
          210  +
          211  +AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"])
          212  +AC_SUBST(SHELL_CFLAGS)
   166    213   
   167    214   #-----------------------------------------------------------------------
   168    215   # UPDATE: Maybe it's better if users just set CFLAGS before invoking
   169    216   # configure. This option doesn't really add much...
   170    217   #
   171    218   #   --enable-tempstore
   172    219   #

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.21.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.21.0'
   730         -PACKAGE_STRING='sqlite 3.21.0'
          729  +PACKAGE_VERSION='3.26.0'
          730  +PACKAGE_STRING='sqlite 3.26.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
   768    768   #endif"
   769    769   
   770    770   ac_subst_vars='LTLIBOBJS
   771    771   LIBOBJS
   772    772   BUILD_CFLAGS
   773    773   USE_GCOV
   774    774   OPT_FEATURE_FLAGS
          775  +HAVE_ZLIB
   775    776   USE_AMALGAMATION
   776    777   TARGET_DEBUG
   777    778   TARGET_HAVE_EDITLINE
   778    779   TARGET_HAVE_READLINE
   779    780   TARGET_READLINE_INC
   780    781   TARGET_READLINE_LIBS
   781    782   HAVE_TCL
................................................................................
   906    907   enable_memsys5
   907    908   enable_memsys3
   908    909   enable_fts3
   909    910   enable_fts4
   910    911   enable_fts5
   911    912   enable_json1
   912    913   enable_update_limit
          914  +enable_geopoly
   913    915   enable_rtree
   914    916   enable_session
   915    917   enable_gcov
   916    918   '
   917    919         ac_precious_vars='build_alias
   918    920   host_alias
   919    921   target_alias
................................................................................
  1460   1462   #
  1461   1463   # Report the --help message.
  1462   1464   #
  1463   1465   if test "$ac_init_help" = "long"; then
  1464   1466     # Omit some internal or obsolete options to make the list less imposing.
  1465   1467     # This message is too long to be a string in the A/UX 3.1 sh.
  1466   1468     cat <<_ACEOF
  1467         -\`configure' configures sqlite 3.21.0 to adapt to many kinds of systems.
         1469  +\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.
  1468   1470   
  1469   1471   Usage: $0 [OPTION]... [VAR=VALUE]...
  1470   1472   
  1471   1473   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1472   1474   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1473   1475   
  1474   1476   Defaults for the options are specified in brackets.
................................................................................
  1525   1527     --build=BUILD     configure for building on BUILD [guessed]
  1526   1528     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1527   1529   _ACEOF
  1528   1530   fi
  1529   1531   
  1530   1532   if test -n "$ac_init_help"; then
  1531   1533     case $ac_init_help in
  1532         -     short | recursive ) echo "Configuration of sqlite 3.21.0:";;
         1534  +     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
  1533   1535      esac
  1534   1536     cat <<\_ACEOF
  1535   1537   
  1536   1538   Optional Features:
  1537   1539     --disable-option-checking  ignore unrecognized --enable/--with options
  1538   1540     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1539   1541     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1558   1560     --enable-memsys5        Enable MEMSYS5
  1559   1561     --enable-memsys3        Enable MEMSYS3
  1560   1562     --enable-fts3           Enable the FTS3 extension
  1561   1563     --enable-fts4           Enable the FTS4 extension
  1562   1564     --enable-fts5           Enable the FTS5 extension
  1563   1565     --enable-json1          Enable the JSON1 extension
  1564   1566     --enable-update-limit   Enable the UPDATE/DELETE LIMIT clause
         1567  +  --enable-geopoly        Enable the GEOPOLY extension
  1565   1568     --enable-rtree          Enable the RTREE extension
  1566   1569     --enable-session        Enable the SESSION extension
  1567   1570     --enable-gcov           Enable coverage testing using gcov
  1568   1571   
  1569   1572   Optional Packages:
  1570   1573     --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
  1571   1574     --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
................................................................................
  1650   1653       cd "$ac_pwd" || { ac_status=$?; break; }
  1651   1654     done
  1652   1655   fi
  1653   1656   
  1654   1657   test -n "$ac_init_help" && exit $ac_status
  1655   1658   if $ac_init_version; then
  1656   1659     cat <<\_ACEOF
  1657         -sqlite configure 3.21.0
         1660  +sqlite configure 3.26.0
  1658   1661   generated by GNU Autoconf 2.69
  1659   1662   
  1660   1663   Copyright (C) 2012 Free Software Foundation, Inc.
  1661   1664   This configure script is free software; the Free Software Foundation
  1662   1665   gives unlimited permission to copy, distribute and modify it.
  1663   1666   _ACEOF
  1664   1667     exit
................................................................................
  2069   2072     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2070   2073   
  2071   2074   } # ac_fn_c_check_header_mongrel
  2072   2075   cat >config.log <<_ACEOF
  2073   2076   This file contains any messages produced by compilers while
  2074   2077   running configure, to aid debugging if configure makes a mistake.
  2075   2078   
  2076         -It was created by sqlite $as_me 3.21.0, which was
         2079  +It was created by sqlite $as_me 3.26.0, which was
  2077   2080   generated by GNU Autoconf 2.69.  Invocation command line was
  2078   2081   
  2079   2082     $ $0 $@
  2080   2083   
  2081   2084   _ACEOF
  2082   2085   exec 5>>config.log
  2083   2086   {
................................................................................
  3927   3930   { $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5
  3928   3931   $as_echo_n "checking the name lister ($NM) interface... " >&6; }
  3929   3932   if ${lt_cv_nm_interface+:} false; then :
  3930   3933     $as_echo_n "(cached) " >&6
  3931   3934   else
  3932   3935     lt_cv_nm_interface="BSD nm"
  3933   3936     echo "int some_variable = 0;" > conftest.$ac_ext
  3934         -  (eval echo "\"\$as_me:3934: $ac_compile\"" >&5)
         3937  +  (eval echo "\"\$as_me:3937: $ac_compile\"" >&5)
  3935   3938     (eval "$ac_compile" 2>conftest.err)
  3936   3939     cat conftest.err >&5
  3937         -  (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
         3940  +  (eval echo "\"\$as_me:3940: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  3938   3941     (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  3939   3942     cat conftest.err >&5
  3940         -  (eval echo "\"\$as_me:3940: output\"" >&5)
         3943  +  (eval echo "\"\$as_me:3943: output\"" >&5)
  3941   3944     cat conftest.out >&5
  3942   3945     if $GREP 'External.*some_variable' conftest.out > /dev/null; then
  3943   3946       lt_cv_nm_interface="MS dumpbin"
  3944   3947     fi
  3945   3948     rm -f conftest*
  3946   3949   fi
  3947   3950   { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5
................................................................................
  5139   5142   	;;
  5140   5143       esac
  5141   5144     fi
  5142   5145     rm -rf conftest*
  5143   5146     ;;
  5144   5147   *-*-irix6*)
  5145   5148     # Find out which ABI we are using.
  5146         -  echo '#line 5146 "configure"' > conftest.$ac_ext
         5149  +  echo '#line 5149 "configure"' > conftest.$ac_ext
  5147   5150     if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  5148   5151     (eval $ac_compile) 2>&5
  5149   5152     ac_status=$?
  5150   5153     $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
  5151   5154     test $ac_status = 0; }; then
  5152   5155       if test "$lt_cv_prog_gnu_ld" = yes; then
  5153   5156         case `/usr/bin/file conftest.$ac_objext` in
................................................................................
  6664   6667      # Note that $ac_compile itself does not contain backslashes and begins
  6665   6668      # with a dollar sign (not a hyphen), so the echo should work correctly.
  6666   6669      # The option is referenced via a variable to avoid confusing sed.
  6667   6670      lt_compile=`echo "$ac_compile" | $SED \
  6668   6671      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  6669   6672      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  6670   6673      -e 's:$: $lt_compiler_flag:'`
  6671         -   (eval echo "\"\$as_me:6671: $lt_compile\"" >&5)
         6674  +   (eval echo "\"\$as_me:6674: $lt_compile\"" >&5)
  6672   6675      (eval "$lt_compile" 2>conftest.err)
  6673   6676      ac_status=$?
  6674   6677      cat conftest.err >&5
  6675         -   echo "$as_me:6675: \$? = $ac_status" >&5
         6678  +   echo "$as_me:6678: \$? = $ac_status" >&5
  6676   6679      if (exit $ac_status) && test -s "$ac_outfile"; then
  6677   6680        # The compiler can only warn and ignore the option if not recognized
  6678   6681        # So say no if there are warnings other than the usual output.
  6679   6682        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  6680   6683        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  6681   6684        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  6682   6685          lt_cv_prog_compiler_rtti_exceptions=yes
................................................................................
  7003   7006      # Note that $ac_compile itself does not contain backslashes and begins
  7004   7007      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7005   7008      # The option is referenced via a variable to avoid confusing sed.
  7006   7009      lt_compile=`echo "$ac_compile" | $SED \
  7007   7010      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7008   7011      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7009   7012      -e 's:$: $lt_compiler_flag:'`
  7010         -   (eval echo "\"\$as_me:7010: $lt_compile\"" >&5)
         7013  +   (eval echo "\"\$as_me:7013: $lt_compile\"" >&5)
  7011   7014      (eval "$lt_compile" 2>conftest.err)
  7012   7015      ac_status=$?
  7013   7016      cat conftest.err >&5
  7014         -   echo "$as_me:7014: \$? = $ac_status" >&5
         7017  +   echo "$as_me:7017: \$? = $ac_status" >&5
  7015   7018      if (exit $ac_status) && test -s "$ac_outfile"; then
  7016   7019        # The compiler can only warn and ignore the option if not recognized
  7017   7020        # So say no if there are warnings other than the usual output.
  7018   7021        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  7019   7022        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  7020   7023        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  7021   7024          lt_cv_prog_compiler_pic_works=yes
................................................................................
  7108   7111      # (2) before a word containing "conftest.", or (3) at the end.
  7109   7112      # Note that $ac_compile itself does not contain backslashes and begins
  7110   7113      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7111   7114      lt_compile=`echo "$ac_compile" | $SED \
  7112   7115      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7113   7116      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7114   7117      -e 's:$: $lt_compiler_flag:'`
  7115         -   (eval echo "\"\$as_me:7115: $lt_compile\"" >&5)
         7118  +   (eval echo "\"\$as_me:7118: $lt_compile\"" >&5)
  7116   7119      (eval "$lt_compile" 2>out/conftest.err)
  7117   7120      ac_status=$?
  7118   7121      cat out/conftest.err >&5
  7119         -   echo "$as_me:7119: \$? = $ac_status" >&5
         7122  +   echo "$as_me:7122: \$? = $ac_status" >&5
  7120   7123      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7121   7124      then
  7122   7125        # The compiler can only warn and ignore the option if not recognized
  7123   7126        # So say no if there are warnings
  7124   7127        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7125   7128        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7126   7129        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  7163   7166      # (2) before a word containing "conftest.", or (3) at the end.
  7164   7167      # Note that $ac_compile itself does not contain backslashes and begins
  7165   7168      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7166   7169      lt_compile=`echo "$ac_compile" | $SED \
  7167   7170      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7168   7171      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7169   7172      -e 's:$: $lt_compiler_flag:'`
  7170         -   (eval echo "\"\$as_me:7170: $lt_compile\"" >&5)
         7173  +   (eval echo "\"\$as_me:7173: $lt_compile\"" >&5)
  7171   7174      (eval "$lt_compile" 2>out/conftest.err)
  7172   7175      ac_status=$?
  7173   7176      cat out/conftest.err >&5
  7174         -   echo "$as_me:7174: \$? = $ac_status" >&5
         7177  +   echo "$as_me:7177: \$? = $ac_status" >&5
  7175   7178      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7176   7179      then
  7177   7180        # The compiler can only warn and ignore the option if not recognized
  7178   7181        # So say no if there are warnings
  7179   7182        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7180   7183        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7181   7184        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  9543   9546   else
  9544   9547     	  if test "$cross_compiling" = yes; then :
  9545   9548     lt_cv_dlopen_self=cross
  9546   9549   else
  9547   9550     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9548   9551     lt_status=$lt_dlunknown
  9549   9552     cat > conftest.$ac_ext <<_LT_EOF
  9550         -#line 9550 "configure"
         9553  +#line 9553 "configure"
  9551   9554   #include "confdefs.h"
  9552   9555   
  9553   9556   #if HAVE_DLFCN_H
  9554   9557   #include <dlfcn.h>
  9555   9558   #endif
  9556   9559   
  9557   9560   #include <stdio.h>
................................................................................
  9639   9642   else
  9640   9643     	  if test "$cross_compiling" = yes; then :
  9641   9644     lt_cv_dlopen_self_static=cross
  9642   9645   else
  9643   9646     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9644   9647     lt_status=$lt_dlunknown
  9645   9648     cat > conftest.$ac_ext <<_LT_EOF
  9646         -#line 9646 "configure"
         9649  +#line 9649 "configure"
  9647   9650   #include "confdefs.h"
  9648   9651   
  9649   9652   #if HAVE_DLFCN_H
  9650   9653   #include <dlfcn.h>
  9651   9654   #endif
  9652   9655   
  9653   9656   #include <stdio.h>
................................................................................
 10450  10453   
 10451  10454   ##########
 10452  10455   # Do we want to support multithreaded use of sqlite
 10453  10456   #
 10454  10457   # Check whether --enable-threadsafe was given.
 10455  10458   if test "${enable_threadsafe+set}" = set; then :
 10456  10459     enableval=$enable_threadsafe;
 10457         -else
 10458         -  enable_threadsafe=yes
 10459  10460   fi
 10460  10461   
 10461  10462   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support threadsafe operation" >&5
 10462  10463   $as_echo_n "checking whether to support threadsafe operation... " >&6; }
 10463  10464   if test "$enable_threadsafe" = "no"; then
 10464  10465     SQLITE_THREADSAFE=0
 10465  10466     { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
................................................................................
 11244  11245   fi
 11245  11246   
 11246  11247   
 11247  11248   #########
 11248  11249   # check for debug enabled
 11249  11250   # Check whether --enable-debug was given.
 11250  11251   if test "${enable_debug+set}" = set; then :
 11251         -  enableval=$enable_debug; use_debug=$enableval
 11252         -else
 11253         -  use_debug=no
        11252  +  enableval=$enable_debug;
 11254  11253   fi
 11255  11254   
 11256         -if test "${use_debug}" = "yes" ; then
        11255  +if test "${enable_debug}" = "yes" ; then
 11257  11256     TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
 11258  11257   else
 11259  11258     TARGET_DEBUG="-DNDEBUG"
 11260  11259   fi
 11261  11260   
 11262  11261   
 11263  11262   #########
 11264  11263   # See whether we should use the amalgamation to build
 11265  11264   # Check whether --enable-amalgamation was given.
 11266  11265   if test "${enable_amalgamation+set}" = set; then :
 11267         -  enableval=$enable_amalgamation; use_amalgamation=$enableval
 11268         -else
 11269         -  use_amalgamation=yes
        11266  +  enableval=$enable_amalgamation;
 11270  11267   fi
 11271  11268   
 11272         -if test "${use_amalgamation}" != "yes" ; then
        11269  +if test "${enable_amalgamation}" == "no" ; then
 11273  11270     USE_AMALGAMATION=0
 11274  11271   fi
        11272  +
        11273  +
        11274  +#########
        11275  +# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
        11276  +for ac_header in zlib.h
        11277  +do :
        11278  +  ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default"
        11279  +if test "x$ac_cv_header_zlib_h" = xyes; then :
        11280  +  cat >>confdefs.h <<_ACEOF
        11281  +#define HAVE_ZLIB_H 1
        11282  +_ACEOF
        11283  +
        11284  +fi
        11285  +
        11286  +done
        11287  +
        11288  +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5
        11289  +$as_echo_n "checking for library containing deflate... " >&6; }
        11290  +if ${ac_cv_search_deflate+:} false; then :
        11291  +  $as_echo_n "(cached) " >&6
        11292  +else
        11293  +  ac_func_search_save_LIBS=$LIBS
        11294  +cat confdefs.h - <<_ACEOF >conftest.$ac_ext
        11295  +/* end confdefs.h.  */
        11296  +
        11297  +/* Override any GCC internal prototype to avoid an error.
        11298  +   Use char because int might match the return type of a GCC
        11299  +   builtin and then its argument prototype would still apply.  */
        11300  +#ifdef __cplusplus
        11301  +extern "C"
        11302  +#endif
        11303  +char deflate ();
        11304  +int
        11305  +main ()
        11306  +{
        11307  +return deflate ();
        11308  +  ;
        11309  +  return 0;
        11310  +}
        11311  +_ACEOF
        11312  +for ac_lib in '' z; do
        11313  +  if test -z "$ac_lib"; then
        11314  +    ac_res="none required"
        11315  +  else
        11316  +    ac_res=-l$ac_lib
        11317  +    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
        11318  +  fi
        11319  +  if ac_fn_c_try_link "$LINENO"; then :
        11320  +  ac_cv_search_deflate=$ac_res
        11321  +fi
        11322  +rm -f core conftest.err conftest.$ac_objext \
        11323  +    conftest$ac_exeext
        11324  +  if ${ac_cv_search_deflate+:} false; then :
        11325  +  break
        11326  +fi
        11327  +done
        11328  +if ${ac_cv_search_deflate+:} false; then :
        11329  +
        11330  +else
        11331  +  ac_cv_search_deflate=no
        11332  +fi
        11333  +rm conftest.$ac_ext
        11334  +LIBS=$ac_func_search_save_LIBS
        11335  +fi
        11336  +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5
        11337  +$as_echo "$ac_cv_search_deflate" >&6; }
        11338  +ac_res=$ac_cv_search_deflate
        11339  +if test "$ac_res" != no; then :
        11340  +  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
        11341  +  HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"
        11342  +else
        11343  +  HAVE_ZLIB=""
        11344  +fi
        11345  +
 11275  11346   
 11276  11347   
 11277  11348   #########
 11278  11349   # See whether we should allow loadable extensions
 11279  11350   # Check whether --enable-load-extension was given.
 11280  11351   if test "${enable_load_extension+set}" = set; then :
 11281         -  enableval=$enable_load_extension; use_loadextension=$enableval
        11352  +  enableval=$enable_load_extension;
 11282  11353   else
 11283         -  use_loadextension=yes
        11354  +  enable_load_extension=yes
 11284  11355   fi
 11285  11356   
 11286         -if test "${use_loadextension}" = "yes" ; then
        11357  +if test "${enable_load_extension}" = "yes" ; then
 11287  11358     OPT_FEATURE_FLAGS=""
 11288  11359     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5
 11289  11360   $as_echo_n "checking for library containing dlopen... " >&6; }
 11290  11361   if ${ac_cv_search_dlopen+:} false; then :
 11291  11362     $as_echo_n "(cached) " >&6
 11292  11363   else
 11293  11364     ac_func_search_save_LIBS=$LIBS
................................................................................
 11346  11417   fi
 11347  11418   
 11348  11419   ##########
 11349  11420   # Do we want to support memsys3 and/or memsys5
 11350  11421   #
 11351  11422   # Check whether --enable-memsys5 was given.
 11352  11423   if test "${enable_memsys5+set}" = set; then :
 11353         -  enableval=$enable_memsys5; enable_memsys5=yes
 11354         -else
 11355         -  enable_memsys5=no
        11424  +  enableval=$enable_memsys5;
 11356  11425   fi
 11357  11426   
 11358  11427   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5
 11359  11428   $as_echo_n "checking whether to support MEMSYS5... " >&6; }
 11360  11429   if test "${enable_memsys5}" = "yes"; then
 11361  11430     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
 11362  11431     { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
................................................................................
 11363  11432   $as_echo "yes" >&6; }
 11364  11433   else
 11365  11434     { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 11366  11435   $as_echo "no" >&6; }
 11367  11436   fi
 11368  11437   # Check whether --enable-memsys3 was given.
 11369  11438   if test "${enable_memsys3+set}" = set; then :
 11370         -  enableval=$enable_memsys3; enable_memsys3=yes
 11371         -else
 11372         -  enable_memsys3=no
        11439  +  enableval=$enable_memsys3;
 11373  11440   fi
 11374  11441   
 11375  11442   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5
 11376  11443   $as_echo_n "checking whether to support MEMSYS3... " >&6; }
 11377  11444   if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
 11378  11445     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
 11379  11446     { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
................................................................................
 11383  11450   $as_echo "no" >&6; }
 11384  11451   fi
 11385  11452   
 11386  11453   #########
 11387  11454   # See whether we should enable Full Text Search extensions
 11388  11455   # Check whether --enable-fts3 was given.
 11389  11456   if test "${enable_fts3+set}" = set; then :
 11390         -  enableval=$enable_fts3; enable_fts3=yes
 11391         -else
 11392         -  enable_fts3=no
        11457  +  enableval=$enable_fts3;
 11393  11458   fi
 11394  11459   
 11395  11460   if test "${enable_fts3}" = "yes" ; then
 11396  11461     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
 11397  11462   fi
 11398  11463   # Check whether --enable-fts4 was given.
 11399  11464   if test "${enable_fts4+set}" = set; then :
 11400         -  enableval=$enable_fts4; enable_fts4=yes
 11401         -else
 11402         -  enable_fts4=no
        11465  +  enableval=$enable_fts4;
 11403  11466   fi
 11404  11467   
 11405  11468   if test "${enable_fts4}" = "yes" ; then
 11406  11469     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
 11407  11470     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 11408  11471   $as_echo_n "checking for library containing log... " >&6; }
 11409  11472   if ${ac_cv_search_log+:} false; then :
................................................................................
 11459  11522     test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
 11460  11523   
 11461  11524   fi
 11462  11525   
 11463  11526   fi
 11464  11527   # Check whether --enable-fts5 was given.
 11465  11528   if test "${enable_fts5+set}" = set; then :
 11466         -  enableval=$enable_fts5; enable_fts5=yes
 11467         -else
 11468         -  enable_fts5=no
        11529  +  enableval=$enable_fts5;
 11469  11530   fi
 11470  11531   
 11471  11532   if test "${enable_fts5}" = "yes" ; then
 11472  11533     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
 11473  11534     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
 11474  11535   $as_echo_n "checking for library containing log... " >&6; }
 11475  11536   if ${ac_cv_search_log+:} false; then :
................................................................................
 11528  11589   
 11529  11590   fi
 11530  11591   
 11531  11592   #########
 11532  11593   # See whether we should enable JSON1
 11533  11594   # Check whether --enable-json1 was given.
 11534  11595   if test "${enable_json1+set}" = set; then :
 11535         -  enableval=$enable_json1; enable_json1=yes
 11536         -else
 11537         -  enable_json1=no
        11596  +  enableval=$enable_json1;
 11538  11597   fi
 11539  11598   
 11540  11599   if test "${enable_json1}" = "yes" ; then
 11541  11600     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
 11542  11601   fi
 11543  11602   
 11544  11603   #########
 11545  11604   # See whether we should enable the LIMIT clause on UPDATE and DELETE
 11546  11605   # statements.
 11547  11606   # Check whether --enable-update-limit was given.
 11548  11607   if test "${enable_update_limit+set}" = set; then :
 11549         -  enableval=$enable_update_limit; enable_udlimit=yes
 11550         -else
 11551         -  enable_udlimit=no
        11608  +  enableval=$enable_update_limit;
 11552  11609   fi
 11553  11610   
 11554  11611   if test "${enable_udlimit}" = "yes" ; then
 11555  11612     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
 11556  11613   fi
        11614  +
        11615  +#########
        11616  +# See whether we should enable GEOPOLY
        11617  +# Check whether --enable-geopoly was given.
        11618  +if test "${enable_geopoly+set}" = set; then :
        11619  +  enableval=$enable_geopoly; enable_geopoly=yes
        11620  +else
        11621  +  enable_geopoly=no
        11622  +fi
        11623  +
        11624  +if test "${enable_geopoly}" = "yes" ; then
        11625  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
        11626  +  enable_rtree=yes
        11627  +fi
 11557  11628   
 11558  11629   #########
 11559  11630   # See whether we should enable RTREE
 11560  11631   # Check whether --enable-rtree was given.
 11561  11632   if test "${enable_rtree+set}" = set; then :
 11562         -  enableval=$enable_rtree; enable_rtree=yes
 11563         -else
 11564         -  enable_rtree=no
        11633  +  enableval=$enable_rtree;
 11565  11634   fi
 11566  11635   
 11567  11636   if test "${enable_rtree}" = "yes" ; then
 11568  11637     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
 11569  11638   fi
 11570  11639   
 11571  11640   #########
 11572  11641   # See whether we should enable the SESSION extension
 11573  11642   # Check whether --enable-session was given.
 11574  11643   if test "${enable_session+set}" = set; then :
 11575         -  enableval=$enable_session; enable_session=yes
 11576         -else
 11577         -  enable_session=no
        11644  +  enableval=$enable_session;
 11578  11645   fi
 11579  11646   
 11580  11647   if test "${enable_session}" = "yes" ; then
 11581  11648     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
 11582  11649     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
 11583  11650   fi
 11584  11651   
................................................................................
 11633  11700   BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
 11634  11701   
 11635  11702   
 11636  11703   #########
 11637  11704   # See whether we should use GCOV
 11638  11705   # Check whether --enable-gcov was given.
 11639  11706   if test "${enable_gcov+set}" = set; then :
 11640         -  enableval=$enable_gcov; use_gcov=$enableval
 11641         -else
 11642         -  use_gcov=no
        11707  +  enableval=$enable_gcov;
 11643  11708   fi
 11644  11709   
 11645  11710   if test "${use_gcov}" = "yes" ; then
 11646  11711     USE_GCOV=1
 11647  11712   else
 11648  11713     USE_GCOV=0
 11649  11714   fi
................................................................................
 12163  12228   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12164  12229   
 12165  12230   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12166  12231   # Save the log message, to keep $0 and so on meaningful, and to
 12167  12232   # report actual input values of CONFIG_FILES etc. instead of their
 12168  12233   # values after options handling.
 12169  12234   ac_log="
 12170         -This file was extended by sqlite $as_me 3.21.0, which was
        12235  +This file was extended by sqlite $as_me 3.26.0, which was
 12171  12236   generated by GNU Autoconf 2.69.  Invocation command line was
 12172  12237   
 12173  12238     CONFIG_FILES    = $CONFIG_FILES
 12174  12239     CONFIG_HEADERS  = $CONFIG_HEADERS
 12175  12240     CONFIG_LINKS    = $CONFIG_LINKS
 12176  12241     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12177  12242     $ $0 $@
................................................................................
 12229  12294   
 12230  12295   Report bugs to the package provider."
 12231  12296   
 12232  12297   _ACEOF
 12233  12298   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12234  12299   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12235  12300   ac_cs_version="\\
 12236         -sqlite config.status 3.21.0
        12301  +sqlite config.status 3.26.0
 12237  12302   configured by $0, generated by GNU Autoconf 2.69,
 12238  12303     with options \\"\$ac_cs_config\\"
 12239  12304   
 12240  12305   Copyright (C) 2012 Free Software Foundation, Inc.
 12241  12306   This config.status script is free software; the Free Software Foundation
 12242  12307   gives unlimited permission to copy, distribute and modify it."
 12243  12308   

Changes to configure.ac.

   178    178   fi
   179    179   AC_SUBST(BUILD_CC)
   180    180   
   181    181   ##########
   182    182   # Do we want to support multithreaded use of sqlite
   183    183   #
   184    184   AC_ARG_ENABLE(threadsafe, 
   185         -AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]),,enable_threadsafe=yes)
          185  +AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
   186    186   AC_MSG_CHECKING([whether to support threadsafe operation])
   187    187   if test "$enable_threadsafe" = "no"; then
   188    188     SQLITE_THREADSAFE=0
   189    189     AC_MSG_RESULT([no])
   190    190   else
   191    191     SQLITE_THREADSAFE=1
   192    192     AC_MSG_RESULT([yes])
................................................................................
   553    553   # Figure out what C libraries are required to compile programs
   554    554   # that use "fdatasync()" function.
   555    555   #
   556    556   AC_SEARCH_LIBS(fdatasync, [rt])
   557    557   
   558    558   #########
   559    559   # check for debug enabled
   560         -AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
   561         -      [use_debug=$enableval],[use_debug=no])
   562         -if test "${use_debug}" = "yes" ; then
          560  +AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
          561  +if test "${enable_debug}" = "yes" ; then
   563    562     TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
   564    563   else
   565    564     TARGET_DEBUG="-DNDEBUG"
   566    565   fi
   567    566   AC_SUBST(TARGET_DEBUG)
   568    567   
   569    568   #########
   570    569   # See whether we should use the amalgamation to build
   571    570   AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
   572         -      [Disable the amalgamation and instead build all files separately]),
   573         -      [use_amalgamation=$enableval],[use_amalgamation=yes])
   574         -if test "${use_amalgamation}" != "yes" ; then
          571  +      [Disable the amalgamation and instead build all files separately]))
          572  +if test "${enable_amalgamation}" == "no" ; then
   575    573     USE_AMALGAMATION=0
   576    574   fi
   577    575   AC_SUBST(USE_AMALGAMATION)
   578    576   
          577  +#########
          578  +# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
          579  +AC_CHECK_HEADERS(zlib.h)
          580  +AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""])
          581  +AC_SUBST(HAVE_ZLIB)
          582  +
   579    583   #########
   580    584   # See whether we should allow loadable extensions
   581    585   AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
   582         -      [Disable loading of external extensions]),
   583         -      [use_loadextension=$enableval],[use_loadextension=yes])
   584         -if test "${use_loadextension}" = "yes" ; then
          586  +      [Disable loading of external extensions]),,[enable_load_extension=yes])
          587  +if test "${enable_load_extension}" = "yes" ; then
   585    588     OPT_FEATURE_FLAGS=""
   586    589     AC_SEARCH_LIBS(dlopen, dl)
   587    590   else
   588    591     OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
   589    592   fi
   590    593   
   591    594   ##########
   592    595   # Do we want to support memsys3 and/or memsys5
   593    596   #
   594    597   AC_ARG_ENABLE(memsys5, 
   595         -  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]),
   596         -  [enable_memsys5=yes],[enable_memsys5=no])
          598  +  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
   597    599   AC_MSG_CHECKING([whether to support MEMSYS5])
   598    600   if test "${enable_memsys5}" = "yes"; then
   599    601     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
   600    602     AC_MSG_RESULT([yes])
   601    603   else
   602    604     AC_MSG_RESULT([no])
   603    605   fi
   604    606   AC_ARG_ENABLE(memsys3, 
   605         -  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]),
   606         -  [enable_memsys3=yes],[enable_memsys3=no])
          607  +  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
   607    608   AC_MSG_CHECKING([whether to support MEMSYS3])
   608    609   if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
   609    610     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
   610    611     AC_MSG_RESULT([yes])
   611    612   else
   612    613     AC_MSG_RESULT([no])
   613    614   fi
   614    615   
   615    616   #########
   616    617   # See whether we should enable Full Text Search extensions
   617    618   AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
   618         -      [Enable the FTS3 extension]),
   619         -      [enable_fts3=yes],[enable_fts3=no])
          619  +      [Enable the FTS3 extension]))
   620    620   if test "${enable_fts3}" = "yes" ; then
   621    621     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3"
   622    622   fi
   623    623   AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
   624         -      [Enable the FTS4 extension]),
   625         -      [enable_fts4=yes],[enable_fts4=no])
          624  +      [Enable the FTS4 extension]))
   626    625   if test "${enable_fts4}" = "yes" ; then
   627    626     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4"
   628    627     AC_SEARCH_LIBS([log],[m])
   629    628   fi
   630    629   AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
   631         -      [Enable the FTS5 extension]),
   632         -      [enable_fts5=yes],[enable_fts5=no])
          630  +      [Enable the FTS5 extension]))
   633    631   if test "${enable_fts5}" = "yes" ; then
   634    632     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5"
   635    633     AC_SEARCH_LIBS([log],[m])
   636    634   fi
   637    635   
   638    636   #########
   639    637   # See whether we should enable JSON1
   640         -AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
   641         -      [Enable the JSON1 extension]),
   642         -      [enable_json1=yes],[enable_json1=no])
          638  +AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],[Enable the JSON1 extension]))
   643    639   if test "${enable_json1}" = "yes" ; then
   644    640     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1"
   645    641   fi
   646    642   
   647    643   #########
   648    644   # See whether we should enable the LIMIT clause on UPDATE and DELETE
   649    645   # statements.
   650    646   AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
   651         -      [Enable the UPDATE/DELETE LIMIT clause]),
   652         -      [enable_udlimit=yes],[enable_udlimit=no])
          647  +      [Enable the UPDATE/DELETE LIMIT clause]))
   653    648   if test "${enable_udlimit}" = "yes" ; then
   654    649     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT"
   655    650   fi
          651  +
          652  +#########
          653  +# See whether we should enable GEOPOLY
          654  +AC_ARG_ENABLE(geopoly, AC_HELP_STRING([--enable-geopoly],
          655  +      [Enable the GEOPOLY extension]),
          656  +      [enable_geopoly=yes],[enable_geopoly=no])
          657  +if test "${enable_geopoly}" = "yes" ; then
          658  +  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY"
          659  +  enable_rtree=yes
          660  +fi
   656    661   
   657    662   #########
   658    663   # See whether we should enable RTREE
   659    664   AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
   660         -      [Enable the RTREE extension]),
   661         -      [enable_rtree=yes],[enable_rtree=no])
          665  +      [Enable the RTREE extension]))
   662    666   if test "${enable_rtree}" = "yes" ; then
   663    667     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE"
   664    668   fi
   665    669   
   666    670   #########
   667    671   # See whether we should enable the SESSION extension
   668    672   AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
   669         -      [Enable the SESSION extension]),
   670         -      [enable_session=yes],[enable_session=no])
          673  +      [Enable the SESSION extension]))
   671    674   if test "${enable_session}" = "yes" ; then
   672    675     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION"
   673    676     OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK"
   674    677   fi
   675    678   
   676    679   #########
   677    680   # attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter
................................................................................
   723    726   done
   724    727   BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS
   725    728   
   726    729   
   727    730   #########
   728    731   # See whether we should use GCOV
   729    732   AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
   730         -      [Enable coverage testing using gcov]),
   731         -      [use_gcov=$enableval],[use_gcov=no])
          733  +      [Enable coverage testing using gcov]))
   732    734   if test "${use_gcov}" = "yes" ; then
   733    735     USE_GCOV=1
   734    736   else
   735    737     USE_GCOV=0
   736    738   fi
   737    739   AC_SUBST(USE_GCOV)
   738    740   

Added doc/F2FS.txt.

            1  +
            2  +SQLite's OS layer contains the following definitions used in F2FS related
            3  +calls:
            4  +
            5  +#define F2FS_IOCTL_MAGIC        0xf5
            6  +#define F2FS_IOC_START_ATOMIC_WRITE     _IO(F2FS_IOCTL_MAGIC, 1)
            7  +#define F2FS_IOC_COMMIT_ATOMIC_WRITE    _IO(F2FS_IOCTL_MAGIC, 2)
            8  +#define F2FS_IOC_START_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 3)
            9  +#define F2FS_IOC_ABORT_VOLATILE_WRITE   _IO(F2FS_IOCTL_MAGIC, 5)
           10  +#define F2FS_IOC_GET_FEATURES           _IOR(F2FS_IOCTL_MAGIC, 12, u32)
           11  +#define F2FS_FEATURE_ATOMIC_WRITE       0x0004
           12  +
           13  +After opening a database file on Linux (including Android), SQLite determines
           14  +whether or not a file supports F2FS atomic commits as follows:
           15  +
           16  +  u32 flags = 0;
           17  +  rc = ioctl(fd, F2FS_IOC_GET_FEATURES, &flags);
           18  +  if( rc==0 && (flags & F2FS_FEATURE_ATOMIC_WRITE) ){
           19  +    /* File supports F2FS atomic commits */
           20  +  }else{
           21  +    /* File does NOT support F2FS atomic commits */
           22  +  }
           23  +
           24  +where "fd" is the file-descriptor open on the database file.
           25  +
           26  +Usually, when writing to a database file that supports atomic commits, SQLite
           27  +accumulates the entire transaction in heap memory, deferring all writes to the
           28  +db file until the transaction is committed.
           29  +
           30  +When it is time to commit a transaction on a file that supports atomic
           31  +commits, SQLite does:
           32  +
           33  +  /* Take an F_WRLCK lock on the database file. This prevents any other
           34  +  ** SQLite clients from reading or writing the file until the lock
           35  +  ** is released.  */
           36  +  rc = fcntl(fd, F_SETLK, ...);
           37  +  if( rc!=0 ) goto failed;
           38  +
           39  +  rc = ioctl(fd, F2FS_IOC_START_ATOMIC_WRITE);
           40  +  if( rc!=0 ) goto fallback_to_legacy_journal_commit;
           41  +
           42  +  foreach (dirty page){
           43  +    rc = write(fd, ...dirty page...);
           44  +    if( rc!=0 ){
           45  +      ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
           46  +      goto fallback_to_legacy_journal_commit;
           47  +    }
           48  +  }
           49  +
           50  +  rc = ioctl(fd, F2FS_IOC_COMMIT_ATOMIC_WRITE);
           51  +  if( rc!=0 ){
           52  +    ioctl(fd, F2FS_IOC_ABORT_VOLATILE_WRITE);
           53  +    goto fallback_to_legacy_journal_commit;
           54  +  }
           55  +
           56  +  /* If we get there, the transaction has been successfully 
           57  +  ** committed to persistent storage. The following call
           58  +  ** relinquishes the F_WRLCK lock.  */
           59  +  fcntl(fd, F_SETLK, ...);
           60  +
           61  +Assumptions:
           62  +
           63  +1. After either of the F2FS_IOC_ABORT_VOLATILE_WRITE calls return,
           64  +   the database file is in the state that it was in before
           65  +   F2FS_IOC_START_ATOMIC_WRITE was invoked. Even if the ioctl()
           66  +   fails - we're ignoring the return code.
           67  +
           68  +   This is true regardless of the type of error that occurred in
           69  +   ioctl() or write().
           70  +
           71  +2. If the system fails before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
           72  +   completed, then following a reboot the database file is in the
           73  +   state that it was in before F2FS_IOC_START_ATOMIC_WRITE was invoked.
           74  +   Or, if the write was commited right before the system failed, in a 
           75  +   state indicating that all write() calls were successfully committed
           76  +   to persistent storage before the failure occurred.
           77  +
           78  +3. If the process crashes before the F2FS_IOC_COMMIT_ATOMIC_WRITE is
           79  +   completed then the file is automatically restored to the state that
           80  +   it was in before F2FS_IOC_START_ATOMIC_WRITE was called. This occurs
           81  +   before the posix advisory lock is automatically dropped - there is
           82  +   no chance that another client will be able to read the file in a
           83  +   half-committed state before the rollback operation occurs.
           84  +
           85  +
           86  +
           87  +

Changes to doc/lemon.html.

    95     95   As of this writing, the following command-line options are supported:
    96     96   <ul>
    97     97   <li><b>-b</b>
    98     98   Show only the basis for each parser state in the report file.
    99     99   <li><b>-c</b>
   100    100   Do not compress the generated action tables.  The parser will be a
   101    101   little larger and slower, but it will detect syntax errors sooner.
          102  +<li><b>-d</b><i>directory</i>
          103  +Write all output files into <i>directory</i>.  Normally, output files
          104  +are written into the directory that contains the input grammar file.
   102    105   <li><b>-D<i>name</i></b>
   103    106   Define C preprocessor macro <i>name</i>.  This macro is usable by
   104    107   "<tt><a href='#pifdef'>%ifdef</a></tt>" and
   105    108   "<tt><a href='#pifdef'>%ifndef</a></tt>" lines
   106    109   in the grammar file.
   107    110   <li><b>-g</b>
   108    111   Do not generate a parser.  Instead write the input grammar to standard
................................................................................
   675    678   </pre></p>
   676    679   
   677    680   <p>Then the Parse() function generated will have an 4th parameter
   678    681   of type "MyStruct*" and all action routines will have access to
   679    682   a variable named "pAbc" that is the value of the 4th parameter
   680    683   in the most recent call to Parse().</p>
   681    684   
          685  +<p>The <tt>%extra_context</tt> directive works the same except that it
          686  +is passed in on the ParseAlloc() or ParseInit() routines instead of
          687  +on Parse().
          688  +
          689  +<a name='extractx'></a>
          690  +<h4>The <tt>%extra_context</tt> directive</h4>
          691  +
          692  +The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
          693  +to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
          694  +doesn't do anything itself with these extra argument, but it does
          695  +store the value make it available to C-code action routines, destructors,
          696  +and so forth.  For example, if the grammar file contains:</p>
          697  +
          698  +<p><pre>
          699  +    %extra_context { MyStruct *pAbc }
          700  +</pre></p>
          701  +
          702  +<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
          703  +of type "MyStruct*" and all action routines will have access to
          704  +a variable named "pAbc" that is the value of that 2th parameter.</p>
          705  +
          706  +<p>The <tt>%extra_argument</tt> directive works the same except that it
          707  +is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
          708  +
   682    709   <a name='pfallback'></a>
   683    710   <h4>The <tt>%fallback</tt> directive</h4>
   684    711   
   685    712   <p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
   686    713   or more tokens.  The alternative meaning is tried if the original token
   687    714   would have generated a syntax error.</p>
   688    715   
................................................................................
   745    772   
   746    773   <p><pre>
   747    774      %include {#include &lt;unistd.h&gt;}
   748    775   </pre></p>
   749    776   
   750    777   <p>This might be needed, for example, if some of the C actions in the
   751    778   grammar call functions that are prototyped in unistd.h.</p>
          779  +
          780  +<p>Use the <tt><a href="#pcode">%code</a></tt> directive to add code to
          781  +the end of the generated parser.</p>
   752    782   
   753    783   <a name='pleft'></a>
   754    784   <h4>The <tt>%left</tt> directive</h4>
   755    785   
   756    786   The <tt>%left</tt> directive is used (along with the
   757    787   <tt><a href='#pright'>%right</a></tt> and
   758    788   <tt><a href='#pnonassoc'>%nonassoc</a></tt> directives) to declare

Added ext/expert/README.md.

            1  +## SQLite Expert Extension
            2  +
            3  +This folder contains code for a simple system to propose useful indexes
            4  +given a database and a set of SQL queries. It works as follows:
            5  +
            6  +  1. The user database schema is copied to a temporary database.
            7  +
            8  +  1. All SQL queries are prepared against the temporary database.
            9  +     Information regarding the WHERE and ORDER BY clauses, and other query
           10  +     features that affect index selection are recorded.
           11  +
           12  +  1. The information gathered in step 2 is used to create candidate 
           13  +     indexes - indexes that the planner might have made use of in the previous
           14  +     step, had they been available.
           15  +
           16  +  1. A subset of the data in the user database is used to generate statistics
           17  +     for all existing indexes and the candidate indexes generated in step 3
           18  +     above.
           19  +
           20  +  1. The SQL queries are prepared a second time. If the planner uses any
           21  +     of the indexes created in step 3, they are recommended to the user.
           22  +
           23  +# C API
           24  +
           25  +The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
           26  +as follows:
           27  +
           28  +  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
           29  +     A database handle opened by the user is passed as an argument.
           30  +
           31  +  1. The sqlite3expert object is configured with one or more SQL statements
           32  +     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
           33  +     specify a single SQL statement, or multiple statements separated by
           34  +     semi-colons.
           35  +  
           36  +  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
           37  +     configure the size of the data subset used to generate index statistics.
           38  +     Using a smaller subset of the data can speed up the analysis.
           39  +
           40  +  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
           41  +
           42  +  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
           43  +     components of the results of the analysis.
           44  +
           45  +  1. **sqlite3\_expert\_destroy()** is called to free all resources.
           46  +
           47  +Refer to comments in sqlite3expert.h for further details.
           48  +
           49  +# sqlite3_expert application
           50  +
           51  +The file "expert.c" contains the code for a command line application that
           52  +uses the API described above. It can be compiled with (for example):
           53  +
           54  +<pre>
           55  +  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
           56  +</pre>
           57  +
           58  +Assuming the database is named "test.db", it can then be run to analyze a
           59  +single query:
           60  +
           61  +<pre>
           62  +  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
           63  +</pre>
           64  +
           65  +Or an entire text file worth of queries with:
           66  +
           67  +<pre>
           68  +  ./sqlite3_expert -file &lt;text-file&gt; test.db
           69  +</pre>
           70  +
           71  +By default, sqlite3\_expert generates index statistics using all the data in
           72  +the user database. For a large database, this may be prohibitively time
           73  +consuming. The "-sample" option may be used to configure sqlite3\_expert to
           74  +generate statistics based on an integer percentage of the user database as
           75  +follows:
           76  +
           77  +<pre>
           78  +  # Generate statistics based on 25% of the user database rows:
           79  +  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
           80  +
           81  +  # Do not generate any statistics at all:
           82  +  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
           83  +</pre>

Added ext/expert/expert.c.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +
           15  +#include <sqlite3.h>
           16  +#include <stdio.h>
           17  +#include <stdlib.h>
           18  +#include <string.h>
           19  +#include "sqlite3expert.h"
           20  +
           21  +
           22  +static void option_requires_argument(const char *zOpt){
           23  +  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
           24  +  exit(-3);
           25  +}
           26  +
           27  +static int option_integer_arg(const char *zVal){
           28  +  return atoi(zVal);
           29  +}
           30  +
           31  +static void usage(char **argv){
           32  +  fprintf(stderr, "\n");
           33  +  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
           34  +  fprintf(stderr, "\n");
           35  +  fprintf(stderr, "Options are:\n");
           36  +  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
           37  +  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
           38  +  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
           39  +  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
           40  +  exit(-1);
           41  +}
           42  +
           43  +static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
           44  +  FILE *in = fopen(zFile, "rb");
           45  +  long nIn;
           46  +  size_t nRead;
           47  +  char *pBuf;
           48  +  int rc;
           49  +  if( in==0 ){
           50  +    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
           51  +    return SQLITE_ERROR;
           52  +  }
           53  +  fseek(in, 0, SEEK_END);
           54  +  nIn = ftell(in);
           55  +  rewind(in);
           56  +  pBuf = sqlite3_malloc64( nIn+1 );
           57  +  nRead = fread(pBuf, nIn, 1, in);
           58  +  fclose(in);
           59  +  if( nRead!=1 ){
           60  +    sqlite3_free(pBuf);
           61  +    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
           62  +    return SQLITE_ERROR;
           63  +  }
           64  +  pBuf[nIn] = 0;
           65  +  rc = sqlite3_expert_sql(p, pBuf, pzErr);
           66  +  sqlite3_free(pBuf);
           67  +  return rc;
           68  +}
           69  +
           70  +int main(int argc, char **argv){
           71  +  const char *zDb;
           72  +  int rc = 0;
           73  +  char *zErr = 0;
           74  +  int i;
           75  +  int iVerbose = 1;               /* -verbose option */
           76  +
           77  +  sqlite3 *db = 0;
           78  +  sqlite3expert *p = 0;
           79  +
           80  +  if( argc<2 ) usage(argv);
           81  +  zDb = argv[argc-1];
           82  +  if( zDb[0]=='-' ) usage(argv);
           83  +  rc = sqlite3_open(zDb, &db);
           84  +  if( rc!=SQLITE_OK ){
           85  +    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
           86  +    exit(-2);
           87  +  }
           88  +
           89  +  p = sqlite3_expert_new(db, &zErr);
           90  +  if( p==0 ){
           91  +    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
           92  +    rc = 1;
           93  +  }else{
           94  +    for(i=1; i<(argc-1); i++){
           95  +      char *zArg = argv[i];
           96  +      int nArg;
           97  +      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
           98  +      nArg = (int)strlen(zArg);
           99  +      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
          100  +        if( ++i==(argc-1) ) option_requires_argument("-file");
          101  +        rc = readSqlFromFile(p, argv[i], &zErr);
          102  +      }
          103  +
          104  +      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
          105  +        if( ++i==(argc-1) ) option_requires_argument("-sql");
          106  +        rc = sqlite3_expert_sql(p, argv[i], &zErr);
          107  +      }
          108  +
          109  +      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
          110  +        int iSample;
          111  +        if( ++i==(argc-1) ) option_requires_argument("-sample");
          112  +        iSample = option_integer_arg(argv[i]);
          113  +        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
          114  +      }
          115  +
          116  +      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
          117  +        if( ++i==(argc-1) ) option_requires_argument("-verbose");
          118  +        iVerbose = option_integer_arg(argv[i]);
          119  +      }
          120  +
          121  +      else{
          122  +        usage(argv);
          123  +      }
          124  +    }
          125  +  }
          126  +
          127  +  if( rc==SQLITE_OK ){
          128  +    rc = sqlite3_expert_analyze(p, &zErr);
          129  +  }
          130  +
          131  +  if( rc==SQLITE_OK ){
          132  +    int nQuery = sqlite3_expert_count(p);
          133  +    if( iVerbose>0 ){
          134  +      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
          135  +      fprintf(stdout, "-- Candidates -------------------------------\n");
          136  +      fprintf(stdout, "%s\n", zCand);
          137  +    }
          138  +    for(i=0; i<nQuery; i++){
          139  +      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
          140  +      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
          141  +      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
          142  +      if( zIdx==0 ) zIdx = "(no new indexes)\n";
          143  +      if( iVerbose>0 ){
          144  +        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
          145  +        fprintf(stdout, "%s\n\n", zSql);
          146  +      }
          147  +      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
          148  +    }
          149  +  }else{
          150  +    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
          151  +  }
          152  +
          153  +  sqlite3_expert_destroy(p);
          154  +  sqlite3_free(zErr);
          155  +  return rc;
          156  +}

Added ext/expert/expert1.test.

            1  +# 2009 Nov 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# The focus of this file is testing the CLI shell tool. Specifically,
           13  +# the ".recommend" command.
           14  +#
           15  +#
           16  +
           17  +# Test plan:
           18  +#
           19  +#
           20  +if {![info exists testdir]} {
           21  +  set testdir [file join [file dirname [info script]] .. .. test]
           22  +}
           23  +source $testdir/tester.tcl
           24  +set testprefix expert1
           25  +
           26  +if {[info commands sqlite3_expert_new]==""} {
           27  +  finish_test
           28  +  return
           29  +}
           30  +
           31  +set CLI [test_binary_name sqlite3]
           32  +set CMD [test_binary_name sqlite3_expert]
           33  +
           34  +proc squish {txt} {
           35  +  regsub -all {[[:space:]]+} $txt { }
           36  +}
           37  +
           38  +proc do_setup_rec_test {tn setup sql res} {
           39  +  reset_db
           40  +  db eval $setup
           41  +  uplevel [list do_rec_test $tn $sql $res]
           42  +}
           43  +
           44  +foreach {tn setup} {
           45  +  1 {
           46  +    if {![file executable $CMD]} { continue }
           47  +
           48  +    proc do_rec_test {tn sql res} {
           49  +      set res [squish [string trim $res]]
           50  +      set tst [subst -nocommands { 
           51  +        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
           52  +      }]
           53  +      uplevel [list do_test $tn $tst $res]
           54  +    }
           55  +  }
           56  +  2 {
           57  +    if {[info commands sqlite3_expert_new]==""} { continue }
           58  +
           59  +    proc do_rec_test {tn sql res} {
           60  +      set expert [sqlite3_expert_new db]
           61  +      $expert sql $sql
           62  +      $expert analyze
           63  +
           64  +      set result [list]
           65  +      for {set i 0} {$i < [$expert count]} {incr i} {
           66  +        set idx [string trim [$expert report $i indexes]]
           67  +        if {$idx==""} {set idx "(no new indexes)"}
           68  +        lappend result $idx
           69  +        lappend result [string trim [$expert report $i plan]]
           70  +      }
           71  +
           72  +      $expert destroy
           73  +
           74  +      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
           75  +      uplevel [list do_test $tn $tst [string trim [squish $res]]]
           76  +    }
           77  +  }
           78  +  3 {
           79  +    if {![file executable $CLI]} { continue }
           80  +
           81  +    proc do_rec_test {tn sql res} {
           82  +      set res [squish [string trim $res]]
           83  +      set tst [subst -nocommands { 
           84  +        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
           85  +      }]
           86  +      uplevel [list do_test $tn $tst $res]
           87  +    }
           88  +  }
           89  +} {
           90  +
           91  +  eval $setup
           92  +
           93  +
           94  +do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
           95  +  SELECT * FROM t1
           96  +} {
           97  +  (no new indexes)
           98  +  SCAN TABLE t1
           99  +}
          100  +
          101  +do_setup_rec_test $tn.2 {
          102  +  CREATE TABLE t1(a, b, c);
          103  +} {
          104  +  SELECT * FROM t1 WHERE b>?;
          105  +} {
          106  +  CREATE INDEX t1_idx_00000062 ON t1(b);
          107  +  SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
          108  +}
          109  +
          110  +do_setup_rec_test $tn.3 {
          111  +  CREATE TABLE t1(a, b, c);
          112  +} {
          113  +  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
          114  +} {
          115  +  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
          116  +  SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
          117  +}
          118  +
          119  +do_setup_rec_test $tn.4 {
          120  +  CREATE TABLE t1(a, b, c);
          121  +} {
          122  +  SELECT a FROM t1 ORDER BY b;
          123  +} {
          124  +  CREATE INDEX t1_idx_00000062 ON t1(b);
          125  +  SCAN TABLE t1 USING INDEX t1_idx_00000062
          126  +}
          127  +
          128  +do_setup_rec_test $tn.5 {
          129  +  CREATE TABLE t1(a, b, c);
          130  +} {
          131  +  SELECT a FROM t1 WHERE a=? ORDER BY b;
          132  +} {
          133  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
          134  +  SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
          135  +}
          136  +
          137  +do_setup_rec_test $tn.6 {
          138  +  CREATE TABLE t1(a, b, c);
          139  +} {
          140  +  SELECT min(a) FROM t1
          141  +} {
          142  +  CREATE INDEX t1_idx_00000061 ON t1(a);
          143  +  SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
          144  +}
          145  +
          146  +do_setup_rec_test $tn.7 {
          147  +  CREATE TABLE t1(a, b, c);
          148  +} {
          149  +  SELECT * FROM t1 ORDER BY a, b, c;
          150  +} {
          151  +  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
          152  +  SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
          153  +}
          154  +
          155  +#do_setup_rec_test $tn.1.8 {
          156  +#  CREATE TABLE t1(a, b, c);
          157  +#} {
          158  +#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
          159  +#} {
          160  +#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
          161  +#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
          162  +#}
          163  +
          164  +do_setup_rec_test $tn.8.1 {
          165  +  CREATE TABLE t1(a COLLATE NOCase, b, c);
          166  +} {
          167  +  SELECT * FROM t1 WHERE a=?
          168  +} {
          169  +  CREATE INDEX t1_idx_00000061 ON t1(a);
          170  +  SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
          171  +}
          172  +do_setup_rec_test $tn.8.2 {
          173  +  CREATE TABLE t1(a, b COLLATE nocase, c);
          174  +} {
          175  +  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
          176  +} {
          177  +  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
          178  +  SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
          179  +}
          180  +
          181  +
          182  +# Tables with names that require quotes.
          183  +#
          184  +do_setup_rec_test $tn.9.1 {
          185  +  CREATE TABLE "t t"(a, b, c);
          186  +} {
          187  +  SELECT * FROM "t t" WHERE a=?
          188  +} {
          189  +  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
          190  +  SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
          191  +}
          192  +
          193  +do_setup_rec_test $tn.9.2 {
          194  +  CREATE TABLE "t t"(a, b, c);
          195  +} {
          196  +  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
          197  +} {
          198  +  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
          199  +  SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
          200  +}
          201  +
          202  +# Columns with names that require quotes.
          203  +#
          204  +do_setup_rec_test $tn.10.1 {
          205  +  CREATE TABLE t3(a, "b b", c);
          206  +} {
          207  +  SELECT * FROM t3 WHERE "b b" = ?
          208  +} {
          209  +  CREATE INDEX t3_idx_00050c52 ON t3('b b');
          210  +  SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
          211  +}
          212  +
          213  +do_setup_rec_test $tn.10.2 {
          214  +  CREATE TABLE t3(a, "b b", c);
          215  +} {
          216  +  SELECT * FROM t3 ORDER BY "b b"
          217  +} {
          218  +  CREATE INDEX t3_idx_00050c52 ON t3('b b');
          219  +  SCAN TABLE t3 USING INDEX t3_idx_00050c52
          220  +}
          221  +
          222  +# Transitive constraints
          223  +#
          224  +do_setup_rec_test $tn.11.1 {
          225  +  CREATE TABLE t5(a, b);
          226  +  CREATE TABLE t6(c, d);
          227  +} {
          228  +  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
          229  +} {
          230  +  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
          231  +  CREATE INDEX t6_idx_00000063 ON t6(c);
          232  +  SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
          233  +  SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
          234  +}
          235  +
          236  +# OR terms.
          237  +#
          238  +do_setup_rec_test $tn.12.1 {
          239  +  CREATE TABLE t7(a, b);
          240  +} {
          241  +  SELECT * FROM t7 WHERE a=? OR b=?
          242  +} {
          243  +  CREATE INDEX t7_idx_00000062 ON t7(b);
          244  +  CREATE INDEX t7_idx_00000061 ON t7(a);
          245  +  MULTI-INDEX OR
          246  +    SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
          247  +    SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
          248  +}
          249  +
          250  +# rowid terms.
          251  +#
          252  +do_setup_rec_test $tn.13.1 {
          253  +  CREATE TABLE t8(a, b);
          254  +} {
          255  +  SELECT * FROM t8 WHERE rowid=?
          256  +} {
          257  +  (no new indexes)
          258  +  SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
          259  +}
          260  +do_setup_rec_test $tn.13.2 {
          261  +  CREATE TABLE t8(a, b);
          262  +} {
          263  +  SELECT * FROM t8 ORDER BY rowid
          264  +} {
          265  +  (no new indexes)
          266  +  SCAN TABLE t8
          267  +}
          268  +do_setup_rec_test $tn.13.3 {
          269  +  CREATE TABLE t8(a, b);
          270  +} {
          271  +  SELECT * FROM t8 WHERE a=? ORDER BY rowid
          272  +} {
          273  +  CREATE INDEX t8_idx_00000061 ON t8(a); 
          274  +  SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
          275  +}
          276  +
          277  +# Triggers
          278  +#
          279  +do_setup_rec_test $tn.14 {
          280  +  CREATE TABLE t9(a, b, c);
          281  +  CREATE TABLE t10(a, b, c);
          282  +  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
          283  +    UPDATE t10 SET a=new.a WHERE b = new.b;
          284  +  END;
          285  +} {
          286  +  INSERT INTO t9 VALUES(?, ?, ?);
          287  +} {
          288  +  CREATE INDEX t10_idx_00000062 ON t10(b); 
          289  +  SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
          290  +}
          291  +
          292  +do_setup_rec_test $tn.15 {
          293  +  CREATE TABLE t1(a, b);
          294  +  CREATE TABLE t2(c, d);
          295  +
          296  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          297  +  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
          298  +
          299  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          300  +  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
          301  +} {
          302  +  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
          303  +} {
          304  +  CREATE INDEX t2_idx_00000064 ON t2(d);
          305  +  SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
          306  +  SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
          307  +}
          308  +
          309  +do_setup_rec_test $tn.16 {
          310  +  CREATE TABLE t1(a, b);
          311  +} {
          312  +  SELECT * FROM t1 WHERE b IS NOT NULL;
          313  +} {
          314  +  (no new indexes)
          315  +  SCAN TABLE t1
          316  +}
          317  +
          318  +}
          319  +
          320  +proc do_candidates_test {tn sql res} {
          321  +  set res [squish [string trim $res]]
          322  +
          323  +  set expert [sqlite3_expert_new db]
          324  +  $expert sql $sql
          325  +  $expert analyze
          326  +
          327  +  set candidates [squish [string trim [$expert report 0 candidates]]]
          328  +  $expert destroy
          329  +
          330  +  uplevel [list do_test $tn [list set {} $candidates] $res]
          331  +}
          332  +
          333  +
          334  +reset_db
          335  +do_execsql_test 4.0 {
          336  +  CREATE TABLE t1(a, b);
          337  +  CREATE TABLE t2(c, d);
          338  +
          339  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          340  +  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
          341  +
          342  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          343  +  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
          344  +}
          345  +do_candidates_test 4.1 {
          346  +  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
          347  +} {
          348  +  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
          349  +  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
          350  +  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
          351  +  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
          352  +}
          353  +
          354  +do_candidates_test 4.2 {
          355  +  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
          356  +} {
          357  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
          358  +  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
          359  +}
          360  +
          361  +do_execsql_test 4.3 {
          362  +  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
          363  +  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
          364  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
          365  +
          366  +  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
          367  +  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
          368  +  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
          369  +
          370  +  ANALYZE;
          371  +  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
          372  +} {
          373  +  t1 t1_idx_00000061 {100 50} 
          374  +  t1 t1_idx_00000062 {100 20}
          375  +  t1 t1_idx_000123a7 {100 50 17}
          376  +  t2 t2_idx_00000063 {100 20} 
          377  +  t2 t2_idx_00000064 {100 5} 
          378  +  t2 t2_idx_0001295b {100 20 5}
          379  +}
          380  +
          381  +
          382  +finish_test

Added ext/expert/sqlite3expert.c.

            1  +/*
            2  +** 2017 April 09
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +#include "sqlite3expert.h"
           14  +#include <assert.h>
           15  +#include <string.h>
           16  +#include <stdio.h>
           17  +
           18  +#ifndef SQLITE_OMIT_VIRTUALTABLE 
           19  +
           20  +typedef sqlite3_int64 i64;
           21  +typedef sqlite3_uint64 u64;
           22  +
           23  +typedef struct IdxColumn IdxColumn;
           24  +typedef struct IdxConstraint IdxConstraint;
           25  +typedef struct IdxScan IdxScan;
           26  +typedef struct IdxStatement IdxStatement;
           27  +typedef struct IdxTable IdxTable;
           28  +typedef struct IdxWrite IdxWrite;
           29  +
           30  +#define STRLEN  (int)strlen
           31  +
           32  +/*
           33  +** A temp table name that we assume no user database will actually use.
           34  +** If this assumption proves incorrect triggers on the table with the
           35  +** conflicting name will be ignored.
           36  +*/
           37  +#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776"
           38  +
           39  +/*
           40  +** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
           41  +** any other type of single-ended range constraint on a column).
           42  +**
           43  +** pLink:
           44  +**   Used to temporarily link IdxConstraint objects into lists while
           45  +**   creating candidate indexes.
           46  +*/
           47  +struct IdxConstraint {
           48  +  char *zColl;                    /* Collation sequence */
           49  +  int bRange;                     /* True for range, false for eq */
           50  +  int iCol;                       /* Constrained table column */
           51  +  int bFlag;                      /* Used by idxFindCompatible() */
           52  +  int bDesc;                      /* True if ORDER BY <expr> DESC */
           53  +  IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
           54  +  IdxConstraint *pLink;           /* See above */
           55  +};
           56  +
           57  +/*
           58  +** A single scan of a single table.
           59  +*/
           60  +struct IdxScan {
           61  +  IdxTable *pTab;                 /* Associated table object */
           62  +  int iDb;                        /* Database containing table zTable */
           63  +  i64 covering;                   /* Mask of columns required for cov. index */
           64  +  IdxConstraint *pOrder;          /* ORDER BY columns */
           65  +  IdxConstraint *pEq;             /* List of == constraints */
           66  +  IdxConstraint *pRange;          /* List of < constraints */
           67  +  IdxScan *pNextScan;             /* Next IdxScan object for same analysis */
           68  +};
           69  +
           70  +/*
           71  +** Information regarding a single database table. Extracted from 
           72  +** "PRAGMA table_info" by function idxGetTableInfo().
           73  +*/
           74  +struct IdxColumn {
           75  +  char *zName;
           76  +  char *zColl;
           77  +  int iPk;
           78  +};
           79  +struct IdxTable {
           80  +  int nCol;
           81  +  char *zName;                    /* Table name */
           82  +  IdxColumn *aCol;
           83  +  IdxTable *pNext;                /* Next table in linked list of all tables */
           84  +};
           85  +
           86  +/*
           87  +** An object of the following type is created for each unique table/write-op
           88  +** seen. The objects are stored in a singly-linked list beginning at
           89  +** sqlite3expert.pWrite.
           90  +*/
           91  +struct IdxWrite {
           92  +  IdxTable *pTab;
           93  +  int eOp;                        /* SQLITE_UPDATE, DELETE or INSERT */
           94  +  IdxWrite *pNext;
           95  +};
           96  +
           97  +/*
           98  +** Each statement being analyzed is represented by an instance of this
           99  +** structure.
          100  +*/
          101  +struct IdxStatement {
          102  +  int iId;                        /* Statement number */
          103  +  char *zSql;                     /* SQL statement */
          104  +  char *zIdx;                     /* Indexes */
          105  +  char *zEQP;                     /* Plan */
          106  +  IdxStatement *pNext;
          107  +};
          108  +
          109  +
          110  +/*
          111  +** A hash table for storing strings. With space for a payload string
          112  +** with each entry. Methods are:
          113  +**
          114  +**   idxHashInit()
          115  +**   idxHashClear()
          116  +**   idxHashAdd()
          117  +**   idxHashSearch()
          118  +*/
          119  +#define IDX_HASH_SIZE 1023
          120  +typedef struct IdxHashEntry IdxHashEntry;
          121  +typedef struct IdxHash IdxHash;
          122  +struct IdxHashEntry {
          123  +  char *zKey;                     /* nul-terminated key */
          124  +  char *zVal;                     /* nul-terminated value string */
          125  +  char *zVal2;                    /* nul-terminated value string 2 */
          126  +  IdxHashEntry *pHashNext;        /* Next entry in same hash bucket */
          127  +  IdxHashEntry *pNext;            /* Next entry in hash */
          128  +};
          129  +struct IdxHash {
          130  +  IdxHashEntry *pFirst;
          131  +  IdxHashEntry *aHash[IDX_HASH_SIZE];
          132  +};
          133  +
          134  +/*
          135  +** sqlite3expert object.
          136  +*/
          137  +struct sqlite3expert {
          138  +  int iSample;                    /* Percentage of tables to sample for stat1 */
          139  +  sqlite3 *db;                    /* User database */
          140  +  sqlite3 *dbm;                   /* In-memory db for this analysis */
          141  +  sqlite3 *dbv;                   /* Vtab schema for this analysis */
          142  +  IdxTable *pTable;               /* List of all IdxTable objects */
          143  +  IdxScan *pScan;                 /* List of scan objects */
          144  +  IdxWrite *pWrite;               /* List of write objects */
          145  +  IdxStatement *pStatement;       /* List of IdxStatement objects */
          146  +  int bRun;                       /* True once analysis has run */
          147  +  char **pzErrmsg;
          148  +  int rc;                         /* Error code from whereinfo hook */
          149  +  IdxHash hIdx;                   /* Hash containing all candidate indexes */
          150  +  char *zCandidates;              /* For EXPERT_REPORT_CANDIDATES */
          151  +};
          152  +
          153  +
          154  +/*
          155  +** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
          156  +** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
          157  +*/
          158  +static void *idxMalloc(int *pRc, int nByte){
          159  +  void *pRet;
          160  +  assert( *pRc==SQLITE_OK );
          161  +  assert( nByte>0 );
          162  +  pRet = sqlite3_malloc(nByte);
          163  +  if( pRet ){
          164  +    memset(pRet, 0, nByte);
          165  +  }else{
          166  +    *pRc = SQLITE_NOMEM;
          167  +  }
          168  +  return pRet;
          169  +}
          170  +
          171  +/*
          172  +** Initialize an IdxHash hash table.
          173  +*/
          174  +static void idxHashInit(IdxHash *pHash){
          175  +  memset(pHash, 0, sizeof(IdxHash));
          176  +}
          177  +
          178  +/*
          179  +** Reset an IdxHash hash table.
          180  +*/
          181  +static void idxHashClear(IdxHash *pHash){
          182  +  int i;
          183  +  for(i=0; i<IDX_HASH_SIZE; i++){
          184  +    IdxHashEntry *pEntry;
          185  +    IdxHashEntry *pNext;
          186  +    for(pEntry=pHash->aHash[i]; pEntry; pEntry=pNext){
          187  +      pNext = pEntry->pHashNext;
          188  +      sqlite3_free(pEntry->zVal2);
          189  +      sqlite3_free(pEntry);
          190  +    }
          191  +  }
          192  +  memset(pHash, 0, sizeof(IdxHash));
          193  +}
          194  +
          195  +/*
          196  +** Return the index of the hash bucket that the string specified by the
          197  +** arguments to this function belongs.
          198  +*/
          199  +static int idxHashString(const char *z, int n){
          200  +  unsigned int ret = 0;
          201  +  int i;
          202  +  for(i=0; i<n; i++){
          203  +    ret += (ret<<3) + (unsigned char)(z[i]);
          204  +  }
          205  +  return (int)(ret % IDX_HASH_SIZE);
          206  +}
          207  +
          208  +/*
          209  +** If zKey is already present in the hash table, return non-zero and do
          210  +** nothing. Otherwise, add an entry with key zKey and payload string zVal to
          211  +** the hash table passed as the second argument. 
          212  +*/
          213  +static int idxHashAdd(
          214  +  int *pRc, 
          215  +  IdxHash *pHash, 
          216  +  const char *zKey,
          217  +  const char *zVal
          218  +){
          219  +  int nKey = STRLEN(zKey);
          220  +  int iHash = idxHashString(zKey, nKey);
          221  +  int nVal = (zVal ? STRLEN(zVal) : 0);
          222  +  IdxHashEntry *pEntry;
          223  +  assert( iHash>=0 );
          224  +  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
          225  +    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
          226  +      return 1;
          227  +    }
          228  +  }
          229  +  pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1);
          230  +  if( pEntry ){
          231  +    pEntry->zKey = (char*)&pEntry[1];
          232  +    memcpy(pEntry->zKey, zKey, nKey);
          233  +    if( zVal ){
          234  +      pEntry->zVal = &pEntry->zKey[nKey+1];
          235  +      memcpy(pEntry->zVal, zVal, nVal);
          236  +    }
          237  +    pEntry->pHashNext = pHash->aHash[iHash];
          238  +    pHash->aHash[iHash] = pEntry;
          239  +
          240  +    pEntry->pNext = pHash->pFirst;
          241  +    pHash->pFirst = pEntry;
          242  +  }
          243  +  return 0;
          244  +}
          245  +
          246  +/*
          247  +** If zKey/nKey is present in the hash table, return a pointer to the 
          248  +** hash-entry object.
          249  +*/
          250  +static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
          251  +  int iHash;
          252  +  IdxHashEntry *pEntry;
          253  +  if( nKey<0 ) nKey = STRLEN(zKey);
          254  +  iHash = idxHashString(zKey, nKey);
          255  +  assert( iHash>=0 );
          256  +  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
          257  +    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
          258  +      return pEntry;
          259  +    }
          260  +  }
          261  +  return 0;
          262  +}
          263  +
          264  +/*
          265  +** If the hash table contains an entry with a key equal to the string
          266  +** passed as the final two arguments to this function, return a pointer
          267  +** to the payload string. Otherwise, if zKey/nKey is not present in the
          268  +** hash table, return NULL.
          269  +*/
          270  +static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
          271  +  IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey);
          272  +  if( pEntry ) return pEntry->zVal;
          273  +  return 0;
          274  +}
          275  +
          276  +/*
          277  +** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
          278  +** variable to point to a copy of nul-terminated string zColl.
          279  +*/
          280  +static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
          281  +  IdxConstraint *pNew;
          282  +  int nColl = STRLEN(zColl);
          283  +
          284  +  assert( *pRc==SQLITE_OK );
          285  +  pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
          286  +  if( pNew ){
          287  +    pNew->zColl = (char*)&pNew[1];
          288  +    memcpy(pNew->zColl, zColl, nColl+1);
          289  +  }
          290  +  return pNew;
          291  +}
          292  +
          293  +/*
          294  +** An error associated with database handle db has just occurred. Pass
          295  +** the error message to callback function xOut.
          296  +*/
          297  +static void idxDatabaseError(
          298  +  sqlite3 *db,                    /* Database handle */
          299  +  char **pzErrmsg                 /* Write error here */
          300  +){
          301  +  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
          302  +}
          303  +
          304  +/*
          305  +** Prepare an SQL statement.
          306  +*/
          307  +static int idxPrepareStmt(
          308  +  sqlite3 *db,                    /* Database handle to compile against */
          309  +  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
          310  +  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
          311  +  const char *zSql                /* SQL statement to compile */
          312  +){
          313  +  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
          314  +  if( rc!=SQLITE_OK ){
          315  +    *ppStmt = 0;
          316  +    idxDatabaseError(db, pzErrmsg);
          317  +  }
          318  +  return rc;
          319  +}
          320  +
          321  +/*
          322  +** Prepare an SQL statement using the results of a printf() formatting.
          323  +*/
          324  +static int idxPrintfPrepareStmt(
          325  +  sqlite3 *db,                    /* Database handle to compile against */
          326  +  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
          327  +  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
          328  +  const char *zFmt,               /* printf() format of SQL statement */
          329  +  ...                             /* Trailing printf() arguments */
          330  +){
          331  +  va_list ap;
          332  +  int rc;
          333  +  char *zSql;
          334  +  va_start(ap, zFmt);
          335  +  zSql = sqlite3_vmprintf(zFmt, ap);
          336  +  if( zSql==0 ){
          337  +    rc = SQLITE_NOMEM;
          338  +  }else{
          339  +    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
          340  +    sqlite3_free(zSql);
          341  +  }
          342  +  va_end(ap);
          343  +  return rc;
          344  +}
          345  +
          346  +
          347  +/*************************************************************************
          348  +** Beginning of virtual table implementation.
          349  +*/
          350  +typedef struct ExpertVtab ExpertVtab;
          351  +struct ExpertVtab {
          352  +  sqlite3_vtab base;
          353  +  IdxTable *pTab;
          354  +  sqlite3expert *pExpert;
          355  +};
          356  +
          357  +typedef struct ExpertCsr ExpertCsr;
          358  +struct ExpertCsr {
          359  +  sqlite3_vtab_cursor base;
          360  +  sqlite3_stmt *pData;
          361  +};
          362  +
          363  +static char *expertDequote(const char *zIn){
          364  +  int n = STRLEN(zIn);
          365  +  char *zRet = sqlite3_malloc(n);
          366  +
          367  +  assert( zIn[0]=='\'' );
          368  +  assert( zIn[n-1]=='\'' );
          369  +
          370  +  if( zRet ){
          371  +    int iOut = 0;
          372  +    int iIn = 0;
          373  +    for(iIn=1; iIn<(n-1); iIn++){
          374  +      if( zIn[iIn]=='\'' ){
          375  +        assert( zIn[iIn+1]=='\'' );
          376  +        iIn++;
          377  +      }
          378  +      zRet[iOut++] = zIn[iIn];
          379  +    }
          380  +    zRet[iOut] = '\0';
          381  +  }
          382  +
          383  +  return zRet;
          384  +}
          385  +
          386  +/* 
          387  +** This function is the implementation of both the xConnect and xCreate
          388  +** methods of the r-tree virtual table.
          389  +**
          390  +**   argv[0]   -> module name
          391  +**   argv[1]   -> database name
          392  +**   argv[2]   -> table name
          393  +**   argv[...] -> column names...
          394  +*/
          395  +static int expertConnect(
          396  +  sqlite3 *db,
          397  +  void *pAux,
          398  +  int argc, const char *const*argv,
          399  +  sqlite3_vtab **ppVtab,
          400  +  char **pzErr
          401  +){
          402  +  sqlite3expert *pExpert = (sqlite3expert*)pAux;
          403  +  ExpertVtab *p = 0;
          404  +  int rc;
          405  +
          406  +  if( argc!=4 ){
          407  +    *pzErr = sqlite3_mprintf("internal error!");
          408  +    rc = SQLITE_ERROR;
          409  +  }else{
          410  +    char *zCreateTable = expertDequote(argv[3]);
          411  +    if( zCreateTable ){
          412  +      rc = sqlite3_declare_vtab(db, zCreateTable);
          413  +      if( rc==SQLITE_OK ){
          414  +        p = idxMalloc(&rc, sizeof(ExpertVtab));
          415  +      }
          416  +      if( rc==SQLITE_OK ){
          417  +        p->pExpert = pExpert;
          418  +        p->pTab = pExpert->pTable;
          419  +        assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 );
          420  +      }
          421  +      sqlite3_free(zCreateTable);
          422  +    }else{
          423  +      rc = SQLITE_NOMEM;
          424  +    }
          425  +  }
          426  +
          427  +  *ppVtab = (sqlite3_vtab*)p;
          428  +  return rc;
          429  +}
          430  +
          431  +static int expertDisconnect(sqlite3_vtab *pVtab){
          432  +  ExpertVtab *p = (ExpertVtab*)pVtab;
          433  +  sqlite3_free(p);
          434  +  return SQLITE_OK;
          435  +}
          436  +
          437  +static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
          438  +  ExpertVtab *p = (ExpertVtab*)pVtab;
          439  +  int rc = SQLITE_OK;
          440  +  int n = 0;
          441  +  IdxScan *pScan;
          442  +  const int opmask = 
          443  +    SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT |
          444  +    SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE |
          445  +    SQLITE_INDEX_CONSTRAINT_LE;
          446  +
          447  +  pScan = idxMalloc(&rc, sizeof(IdxScan));
          448  +  if( pScan ){
          449  +    int i;
          450  +
          451  +    /* Link the new scan object into the list */
          452  +    pScan->pTab = p->pTab;
          453  +    pScan->pNextScan = p->pExpert->pScan;
          454  +    p->pExpert->pScan = pScan;
          455  +
          456  +    /* Add the constraints to the IdxScan object */
          457  +    for(i=0; i<pIdxInfo->nConstraint; i++){
          458  +      struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
          459  +      if( pCons->usable 
          460  +       && pCons->iColumn>=0 
          461  +       && p->pTab->aCol[pCons->iColumn].iPk==0
          462  +       && (pCons->op & opmask) 
          463  +      ){
          464  +        IdxConstraint *pNew;
          465  +        const char *zColl = sqlite3_vtab_collation(pIdxInfo, i);
          466  +        pNew = idxNewConstraint(&rc, zColl);
          467  +        if( pNew ){
          468  +          pNew->iCol = pCons->iColumn;
          469  +          if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
          470  +            pNew->pNext = pScan->pEq;
          471  +            pScan->pEq = pNew;
          472  +          }else{
          473  +            pNew->bRange = 1;
          474  +            pNew->pNext = pScan->pRange;
          475  +            pScan->pRange = pNew;
          476  +          }
          477  +        }
          478  +        n++;
          479  +        pIdxInfo->aConstraintUsage[i].argvIndex = n;
          480  +      }
          481  +    }
          482  +
          483  +    /* Add the ORDER BY to the IdxScan object */
          484  +    for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
          485  +      int iCol = pIdxInfo->aOrderBy[i].iColumn;
          486  +      if( iCol>=0 ){
          487  +        IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
          488  +        if( pNew ){
          489  +          pNew->iCol = iCol;
          490  +          pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
          491  +          pNew->pNext = pScan->pOrder;
          492  +          pNew->pLink = pScan->pOrder;
          493  +          pScan->pOrder = pNew;
          494  +          n++;
          495  +        }
          496  +      }
          497  +    }
          498  +  }
          499  +
          500  +  pIdxInfo->estimatedCost = 1000000.0 / (n+1);
          501  +  return rc;
          502  +}
          503  +
          504  +static int expertUpdate(
          505  +  sqlite3_vtab *pVtab, 
          506  +  int nData, 
          507  +  sqlite3_value **azData, 
          508  +  sqlite_int64 *pRowid
          509  +){
          510  +  (void)pVtab;
          511  +  (void)nData;
          512  +  (void)azData;
          513  +  (void)pRowid;
          514  +  return SQLITE_OK;
          515  +}
          516  +
          517  +/* 
          518  +** Virtual table module xOpen method.
          519  +*/
          520  +static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
          521  +  int rc = SQLITE_OK;
          522  +  ExpertCsr *pCsr;
          523  +  (void)pVTab;
          524  +  pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
          525  +  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
          526  +  return rc;
          527  +}
          528  +
          529  +/* 
          530  +** Virtual table module xClose method.
          531  +*/
          532  +static int expertClose(sqlite3_vtab_cursor *cur){
          533  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          534  +  sqlite3_finalize(pCsr->pData);
          535  +  sqlite3_free(pCsr);
          536  +  return SQLITE_OK;
          537  +}
          538  +
          539  +/*
          540  +** Virtual table module xEof method.
          541  +**
          542  +** Return non-zero if the cursor does not currently point to a valid 
          543  +** record (i.e if the scan has finished), or zero otherwise.
          544  +*/
          545  +static int expertEof(sqlite3_vtab_cursor *cur){
          546  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          547  +  return pCsr->pData==0;
          548  +}
          549  +
          550  +/* 
          551  +** Virtual table module xNext method.
          552  +*/
          553  +static int expertNext(sqlite3_vtab_cursor *cur){
          554  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          555  +  int rc = SQLITE_OK;
          556  +
          557  +  assert( pCsr->pData );
          558  +  rc = sqlite3_step(pCsr->pData);
          559  +  if( rc!=SQLITE_ROW ){
          560  +    rc = sqlite3_finalize(pCsr->pData);
          561  +    pCsr->pData = 0;
          562  +  }else{
          563  +    rc = SQLITE_OK;
          564  +  }
          565  +
          566  +  return rc;
          567  +}
          568  +
          569  +/* 
          570  +** Virtual table module xRowid method.
          571  +*/
          572  +static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          573  +  (void)cur;
          574  +  *pRowid = 0;
          575  +  return SQLITE_OK;
          576  +}
          577  +
          578  +/* 
          579  +** Virtual table module xColumn method.
          580  +*/
          581  +static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
          582  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          583  +  sqlite3_value *pVal;
          584  +  pVal = sqlite3_column_value(pCsr->pData, i);
          585  +  if( pVal ){
          586  +    sqlite3_result_value(ctx, pVal);
          587  +  }
          588  +  return SQLITE_OK;
          589  +}
          590  +
          591  +/* 
          592  +** Virtual table module xFilter method.
          593  +*/
          594  +static int expertFilter(
          595  +  sqlite3_vtab_cursor *cur, 
          596  +  int idxNum, const char *idxStr,
          597  +  int argc, sqlite3_value **argv
          598  +){
          599  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          600  +  ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
          601  +  sqlite3expert *pExpert = pVtab->pExpert;
          602  +  int rc;
          603  +
          604  +  (void)idxNum;
          605  +  (void)idxStr;
          606  +  (void)argc;
          607  +  (void)argv;
          608  +  rc = sqlite3_finalize(pCsr->pData);
          609  +  pCsr->pData = 0;
          610  +  if( rc==SQLITE_OK ){
          611  +    rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
          612  +        "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
          613  +    );
          614  +  }
          615  +
          616  +  if( rc==SQLITE_OK ){
          617  +    rc = expertNext(cur);
          618  +  }
          619  +  return rc;
          620  +}
          621  +
          622  +static int idxRegisterVtab(sqlite3expert *p){
          623  +  static sqlite3_module expertModule = {
          624  +    2,                            /* iVersion */
          625  +    expertConnect,                /* xCreate - create a table */
          626  +    expertConnect,                /* xConnect - connect to an existing table */
          627  +    expertBestIndex,              /* xBestIndex - Determine search strategy */
          628  +    expertDisconnect,             /* xDisconnect - Disconnect from a table */
          629  +    expertDisconnect,             /* xDestroy - Drop a table */
          630  +    expertOpen,                   /* xOpen - open a cursor */
          631  +    expertClose,                  /* xClose - close a cursor */
          632  +    expertFilter,                 /* xFilter - configure scan constraints */
          633  +    expertNext,                   /* xNext - advance a cursor */
          634  +    expertEof,                    /* xEof */
          635  +    expertColumn,                 /* xColumn - read data */
          636  +    expertRowid,                  /* xRowid - read data */
          637  +    expertUpdate,                 /* xUpdate - write data */
          638  +    0,                            /* xBegin - begin transaction */
          639  +    0,                            /* xSync - sync transaction */
          640  +    0,                            /* xCommit - commit transaction */
          641  +    0,                            /* xRollback - rollback transaction */
          642  +    0,                            /* xFindFunction - function overloading */
          643  +    0,                            /* xRename - rename the table */
          644  +    0,                            /* xSavepoint */
          645  +    0,                            /* xRelease */
          646  +    0,                            /* xRollbackTo */
          647  +    0,                            /* xShadowName */
          648  +  };
          649  +
          650  +  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
          651  +}
          652  +/*
          653  +** End of virtual table implementation.
          654  +*************************************************************************/
          655  +/*
          656  +** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
          657  +** is called, set it to the return value of sqlite3_finalize() before
          658  +** returning. Otherwise, discard the sqlite3_finalize() return value.
          659  +*/
          660  +static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
          661  +  int rc = sqlite3_finalize(pStmt);
          662  +  if( *pRc==SQLITE_OK ) *pRc = rc;
          663  +}
          664  +
          665  +/*
          666  +** Attempt to allocate an IdxTable structure corresponding to table zTab
          667  +** in the main database of connection db. If successful, set (*ppOut) to
          668  +** point to the new object and return SQLITE_OK. Otherwise, return an
          669  +** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be
          670  +** set to point to an error string.
          671  +**
          672  +** It is the responsibility of the caller to eventually free either the
          673  +** IdxTable object or error message using sqlite3_free().
          674  +*/
          675  +static int idxGetTableInfo(
          676  +  sqlite3 *db,                    /* Database connection to read details from */
          677  +  const char *zTab,               /* Table name */
          678  +  IdxTable **ppOut,               /* OUT: New object (if successful) */
          679  +  char **pzErrmsg                 /* OUT: Error message (if not) */
          680  +){
          681  +  sqlite3_stmt *p1 = 0;
          682  +  int nCol = 0;
          683  +  int nTab = STRLEN(zTab);
          684  +  int nByte = sizeof(IdxTable) + nTab + 1;
          685  +  IdxTable *pNew = 0;
          686  +  int rc, rc2;
          687  +  char *pCsr = 0;
          688  +
          689  +  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
          690  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
          691  +    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
          692  +    nByte += 1 + STRLEN(zCol);
          693  +    rc = sqlite3_table_column_metadata(
          694  +        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
          695  +    );
          696  +    nByte += 1 + STRLEN(zCol);
          697  +    nCol++;
          698  +  }
          699  +  rc2 = sqlite3_reset(p1);
          700  +  if( rc==SQLITE_OK ) rc = rc2;
          701  +
          702  +  nByte += sizeof(IdxColumn) * nCol;
          703  +  if( rc==SQLITE_OK ){
          704  +    pNew = idxMalloc(&rc, nByte);
          705  +  }
          706  +  if( rc==SQLITE_OK ){
          707  +    pNew->aCol = (IdxColumn*)&pNew[1];
          708  +    pNew->nCol = nCol;
          709  +    pCsr = (char*)&pNew->aCol[nCol];
          710  +  }
          711  +
          712  +  nCol = 0;
          713  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
          714  +    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
          715  +    int nCopy = STRLEN(zCol) + 1;
          716  +    pNew->aCol[nCol].zName = pCsr;
          717  +    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
          718  +    memcpy(pCsr, zCol, nCopy);
          719  +    pCsr += nCopy;
          720  +
          721  +    rc = sqlite3_table_column_metadata(
          722  +        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
          723  +    );
          724  +    if( rc==SQLITE_OK ){
          725  +      nCopy = STRLEN(zCol) + 1;
          726  +      pNew->aCol[nCol].zColl = pCsr;
          727  +      memcpy(pCsr, zCol, nCopy);
          728  +      pCsr += nCopy;
          729  +    }
          730  +
          731  +    nCol++;
          732  +  }
          733  +  idxFinalize(&rc, p1);
          734  +
          735  +  if( rc!=SQLITE_OK ){
          736  +    sqlite3_free(pNew);
          737  +    pNew = 0;
          738  +  }else{
          739  +    pNew->zName = pCsr;
          740  +    memcpy(pNew->zName, zTab, nTab+1);
          741  +  }
          742  +
          743  +  *ppOut = pNew;
          744  +  return rc;
          745  +}
          746  +
          747  +/*
          748  +** This function is a no-op if *pRc is set to anything other than 
          749  +** SQLITE_OK when it is called.
          750  +**
          751  +** If *pRc is initially set to SQLITE_OK, then the text specified by
          752  +** the printf() style arguments is appended to zIn and the result returned
          753  +** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on
          754  +** zIn before returning.
          755  +*/
          756  +static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
          757  +  va_list ap;
          758  +  char *zAppend = 0;
          759  +  char *zRet = 0;
          760  +  int nIn = zIn ? STRLEN(zIn) : 0;
          761  +  int nAppend = 0;
          762  +  va_start(ap, zFmt);
          763  +  if( *pRc==SQLITE_OK ){
          764  +    zAppend = sqlite3_vmprintf(zFmt, ap);
          765  +    if( zAppend ){
          766  +      nAppend = STRLEN(zAppend);
          767  +      zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
          768  +    }
          769  +    if( zAppend && zRet ){
          770  +      if( nIn ) memcpy(zRet, zIn, nIn);
          771  +      memcpy(&zRet[nIn], zAppend, nAppend+1);
          772  +    }else{
          773  +      sqlite3_free(zRet);
          774  +      zRet = 0;
          775  +      *pRc = SQLITE_NOMEM;
          776  +    }
          777  +    sqlite3_free(zAppend);
          778  +    sqlite3_free(zIn);
          779  +  }
          780  +  va_end(ap);
          781  +  return zRet;
          782  +}
          783  +
          784  +/*
          785  +** Return true if zId must be quoted in order to use it as an SQL
          786  +** identifier, or false otherwise.
          787  +*/
          788  +static int idxIdentifierRequiresQuotes(const char *zId){
          789  +  int i;
          790  +  for(i=0; zId[i]; i++){
          791  +    if( !(zId[i]=='_')
          792  +     && !(zId[i]>='0' && zId[i]<='9')
          793  +     && !(zId[i]>='a' && zId[i]<='z')
          794  +     && !(zId[i]>='A' && zId[i]<='Z')
          795  +    ){
          796  +      return 1;
          797  +    }
          798  +  }
          799  +  return 0;
          800  +}
          801  +
          802  +/*
          803  +** This function appends an index column definition suitable for constraint
          804  +** pCons to the string passed as zIn and returns the result.
          805  +*/
          806  +static char *idxAppendColDefn(
          807  +  int *pRc,                       /* IN/OUT: Error code */
          808  +  char *zIn,                      /* Column defn accumulated so far */
          809  +  IdxTable *pTab,                 /* Table index will be created on */
          810  +  IdxConstraint *pCons
          811  +){
          812  +  char *zRet = zIn;
          813  +  IdxColumn *p = &pTab->aCol[pCons->iCol];
          814  +  if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
          815  +
          816  +  if( idxIdentifierRequiresQuotes(p->zName) ){
          817  +    zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
          818  +  }else{
          819  +    zRet = idxAppendText(pRc, zRet, "%s", p->zName);
          820  +  }
          821  +
          822  +  if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
          823  +    if( idxIdentifierRequiresQuotes(pCons->zColl) ){
          824  +      zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
          825  +    }else{
          826  +      zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
          827  +    }
          828  +  }
          829  +
          830  +  if( pCons->bDesc ){
          831  +    zRet = idxAppendText(pRc, zRet, " DESC");
          832  +  }
          833  +  return zRet;
          834  +}
          835  +
          836  +/*
          837  +** Search database dbm for an index compatible with the one idxCreateFromCons()
          838  +** would create from arguments pScan, pEq and pTail. If no error occurs and 
          839  +** such an index is found, return non-zero. Or, if no such index is found,
          840  +** return zero.
          841  +**
          842  +** If an error occurs, set *pRc to an SQLite error code and return zero.
          843  +*/
          844  +static int idxFindCompatible(
          845  +  int *pRc,                       /* OUT: Error code */
          846  +  sqlite3* dbm,                   /* Database to search */
          847  +  IdxScan *pScan,                 /* Scan for table to search for index on */
          848  +  IdxConstraint *pEq,             /* List of == constraints */
          849  +  IdxConstraint *pTail            /* List of range constraints */
          850  +){
          851  +  const char *zTbl = pScan->pTab->zName;
          852  +  sqlite3_stmt *pIdxList = 0;
          853  +  IdxConstraint *pIter;
          854  +  int nEq = 0;                    /* Number of elements in pEq */
          855  +  int rc;
          856  +
          857  +  /* Count the elements in list pEq */
          858  +  for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
          859  +
          860  +  rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
          861  +  while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
          862  +    int bMatch = 1;
          863  +    IdxConstraint *pT = pTail;
          864  +    sqlite3_stmt *pInfo = 0;
          865  +    const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
          866  +
          867  +    /* Zero the IdxConstraint.bFlag values in the pEq list */
          868  +    for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0;
          869  +
          870  +    rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
          871  +    while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
          872  +      int iIdx = sqlite3_column_int(pInfo, 0);
          873  +      int iCol = sqlite3_column_int(pInfo, 1);
          874  +      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
          875  +
          876  +      if( iIdx<nEq ){
          877  +        for(pIter=pEq; pIter; pIter=pIter->pLink){
          878  +          if( pIter->bFlag ) continue;
          879  +          if( pIter->iCol!=iCol ) continue;
          880  +          if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
          881  +          pIter->bFlag = 1;
          882  +          break;
          883  +        }
          884  +        if( pIter==0 ){
          885  +          bMatch = 0;
          886  +          break;
          887  +        }
          888  +      }else{
          889  +        if( pT ){
          890  +          if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){
          891  +            bMatch = 0;
          892  +            break;
          893  +          }
          894  +          pT = pT->pLink;
          895  +        }
          896  +      }
          897  +    }
          898  +    idxFinalize(&rc, pInfo);
          899  +
          900  +    if( rc==SQLITE_OK && bMatch ){
          901  +      sqlite3_finalize(pIdxList);
          902  +      return 1;
          903  +    }
          904  +  }
          905  +  idxFinalize(&rc, pIdxList);
          906  +
          907  +  *pRc = rc;
          908  +  return 0;
          909  +}
          910  +
          911  +static int idxCreateFromCons(
          912  +  sqlite3expert *p,
          913  +  IdxScan *pScan,
          914  +  IdxConstraint *pEq, 
          915  +  IdxConstraint *pTail
          916  +){
          917  +  sqlite3 *dbm = p->dbm;
          918  +  int rc = SQLITE_OK;
          919  +  if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
          920  +    IdxTable *pTab = pScan->pTab;
          921  +    char *zCols = 0;
          922  +    char *zIdx = 0;
          923  +    IdxConstraint *pCons;
          924  +    unsigned int h = 0;
          925  +    const char *zFmt;
          926  +
          927  +    for(pCons=pEq; pCons; pCons=pCons->pLink){
          928  +      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
          929  +    }
          930  +    for(pCons=pTail; pCons; pCons=pCons->pLink){
          931  +      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
          932  +    }
          933  +
          934  +    if( rc==SQLITE_OK ){
          935  +      /* Hash the list of columns to come up with a name for the index */
          936  +      const char *zTable = pScan->pTab->zName;
          937  +      char *zName;                /* Index name */
          938  +      int i;
          939  +      for(i=0; zCols[i]; i++){
          940  +        h += ((h<<3) + zCols[i]);
          941  +      }
          942  +      zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
          943  +      if( zName==0 ){ 
          944  +        rc = SQLITE_NOMEM;
          945  +      }else{
          946  +        if( idxIdentifierRequiresQuotes(zTable) ){
          947  +          zFmt = "CREATE INDEX '%q' ON %Q(%s)";
          948  +        }else{
          949  +          zFmt = "CREATE INDEX %s ON %s(%s)";
          950  +        }
          951  +        zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols);
          952  +        if( !zIdx ){
          953  +          rc = SQLITE_NOMEM;
          954  +        }else{
          955  +          rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
          956  +          idxHashAdd(&rc, &p->hIdx, zName, zIdx);
          957  +        }
          958  +        sqlite3_free(zName);
          959  +        sqlite3_free(zIdx);
          960  +      }
          961  +    }
          962  +
          963  +    sqlite3_free(zCols);
          964  +  }
          965  +  return rc;
          966  +}
          967  +
          968  +/*
          969  +** Return true if list pList (linked by IdxConstraint.pLink) contains
          970  +** a constraint compatible with *p. Otherwise return false.
          971  +*/
          972  +static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
          973  +  IdxConstraint *pCmp;
          974  +  for(pCmp=pList; pCmp; pCmp=pCmp->pLink){
          975  +    if( p->iCol==pCmp->iCol ) return 1;
          976  +  }
          977  +  return 0;
          978  +}
          979  +
          980  +static int idxCreateFromWhere(
          981  +  sqlite3expert *p, 
          982  +  IdxScan *pScan,                 /* Create indexes for this scan */
          983  +  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
          984  +){
          985  +  IdxConstraint *p1 = 0;
          986  +  IdxConstraint *pCon;
          987  +  int rc;
          988  +
          989  +  /* Gather up all the == constraints. */
          990  +  for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
          991  +    if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
          992  +      pCon->pLink = p1;
          993  +      p1 = pCon;
          994  +    }
          995  +  }
          996  +
          997  +  /* Create an index using the == constraints collected above. And the
          998  +  ** range constraint/ORDER BY terms passed in by the caller, if any. */
          999  +  rc = idxCreateFromCons(p, pScan, p1, pTail);
         1000  +
         1001  +  /* If no range/ORDER BY passed by the caller, create a version of the
         1002  +  ** index for each range constraint.  */
         1003  +  if( pTail==0 ){
         1004  +    for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
         1005  +      assert( pCon->pLink==0 );
         1006  +      if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
         1007  +        rc = idxCreateFromCons(p, pScan, p1, pCon);
         1008  +      }
         1009  +    }
         1010  +  }
         1011  +
         1012  +  return rc;
         1013  +}
         1014  +
         1015  +/*
         1016  +** Create candidate indexes in database [dbm] based on the data in 
         1017  +** linked-list pScan.
         1018  +*/
         1019  +static int idxCreateCandidates(sqlite3expert *p){
         1020  +  int rc = SQLITE_OK;
         1021  +  IdxScan *pIter;
         1022  +
         1023  +  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
         1024  +    rc = idxCreateFromWhere(p, pIter, 0);
         1025  +    if( rc==SQLITE_OK && pIter->pOrder ){
         1026  +      rc = idxCreateFromWhere(p, pIter, pIter->pOrder);
         1027  +    }
         1028  +  }
         1029  +
         1030  +  return rc;
         1031  +}
         1032  +
         1033  +/*
         1034  +** Free all elements of the linked list starting at pConstraint.
         1035  +*/
         1036  +static void idxConstraintFree(IdxConstraint *pConstraint){
         1037  +  IdxConstraint *pNext;
         1038  +  IdxConstraint *p;
         1039  +
         1040  +  for(p=pConstraint; p; p=pNext){
         1041  +    pNext = p->pNext;
         1042  +    sqlite3_free(p);
         1043  +  }
         1044  +}
         1045  +
         1046  +/*
         1047  +** Free all elements of the linked list starting from pScan up until pLast
         1048  +** (pLast is not freed).
         1049  +*/
         1050  +static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
         1051  +  IdxScan *p;
         1052  +  IdxScan *pNext;
         1053  +  for(p=pScan; p!=pLast; p=pNext){
         1054  +    pNext = p->pNextScan;
         1055  +    idxConstraintFree(p->pOrder);
         1056  +    idxConstraintFree(p->pEq);
         1057  +    idxConstraintFree(p->pRange);
         1058  +    sqlite3_free(p);
         1059  +  }
         1060  +}
         1061  +
         1062  +/*
         1063  +** Free all elements of the linked list starting from pStatement up 
         1064  +** until pLast (pLast is not freed).
         1065  +*/
         1066  +static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
         1067  +  IdxStatement *p;
         1068  +  IdxStatement *pNext;
         1069  +  for(p=pStatement; p!=pLast; p=pNext){
         1070  +    pNext = p->pNext;
         1071  +    sqlite3_free(p->zEQP);
         1072  +    sqlite3_free(p->zIdx);
         1073  +    sqlite3_free(p);
         1074  +  }
         1075  +}
         1076  +
         1077  +/*
         1078  +** Free the linked list of IdxTable objects starting at pTab.
         1079  +*/
         1080  +static void idxTableFree(IdxTable *pTab){
         1081  +  IdxTable *pIter;
         1082  +  IdxTable *pNext;
         1083  +  for(pIter=pTab; pIter; pIter=pNext){
         1084  +    pNext = pIter->pNext;
         1085  +    sqlite3_free(pIter);
         1086  +  }
         1087  +}
         1088  +
         1089  +/*
         1090  +** Free the linked list of IdxWrite objects starting at pTab.
         1091  +*/
         1092  +static void idxWriteFree(IdxWrite *pTab){
         1093  +  IdxWrite *pIter;
         1094  +  IdxWrite *pNext;
         1095  +  for(pIter=pTab; pIter; pIter=pNext){
         1096  +    pNext = pIter->pNext;
         1097  +    sqlite3_free(pIter);
         1098  +  }
         1099  +}
         1100  +
         1101  +
         1102  +
         1103  +/*
         1104  +** This function is called after candidate indexes have been created. It
         1105  +** runs all the queries to see which indexes they prefer, and populates
         1106  +** IdxStatement.zIdx and IdxStatement.zEQP with the results.
         1107  +*/
         1108  +int idxFindIndexes(
         1109  +  sqlite3expert *p,
         1110  +  char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
         1111  +){
         1112  +  IdxStatement *pStmt;
         1113  +  sqlite3 *dbm = p->dbm;
         1114  +  int rc = SQLITE_OK;
         1115  +
         1116  +  IdxHash hIdx;
         1117  +  idxHashInit(&hIdx);
         1118  +
         1119  +  for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
         1120  +    IdxHashEntry *pEntry;
         1121  +    sqlite3_stmt *pExplain = 0;
         1122  +    idxHashClear(&hIdx);
         1123  +    rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
         1124  +        "EXPLAIN QUERY PLAN %s", pStmt->zSql
         1125  +    );
         1126  +    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
         1127  +      /* int iId = sqlite3_column_int(pExplain, 0); */
         1128  +      /* int iParent = sqlite3_column_int(pExplain, 1); */
         1129  +      /* int iNotUsed = sqlite3_column_int(pExplain, 2); */
         1130  +      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
         1131  +      int nDetail = STRLEN(zDetail);
         1132  +      int i;
         1133  +
         1134  +      for(i=0; i<nDetail; i++){
         1135  +        const char *zIdx = 0;
         1136  +        if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
         1137  +          zIdx = &zDetail[i+13];
         1138  +        }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
         1139  +          zIdx = &zDetail[i+22];
         1140  +        }
         1141  +        if( zIdx ){
         1142  +          const char *zSql;
         1143  +          int nIdx = 0;
         1144  +          while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
         1145  +            nIdx++;
         1146  +          }
         1147  +          zSql = idxHashSearch(&p->hIdx, zIdx, nIdx);
         1148  +          if( zSql ){
         1149  +            idxHashAdd(&rc, &hIdx, zSql, 0);
         1150  +            if( rc ) goto find_indexes_out;
         1151  +          }
         1152  +          break;
         1153  +        }
         1154  +      }
         1155  +
         1156  +      if( zDetail[0]!='-' ){
         1157  +        pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail);
         1158  +      }
         1159  +    }
         1160  +
         1161  +    for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
         1162  +      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
         1163  +    }
         1164  +
         1165  +    idxFinalize(&rc, pExplain);
         1166  +  }
         1167  +
         1168  + find_indexes_out:
         1169  +  idxHashClear(&hIdx);
         1170  +  return rc;
         1171  +}
         1172  +
         1173  +static int idxAuthCallback(
         1174  +  void *pCtx,
         1175  +  int eOp,
         1176  +  const char *z3,
         1177  +  const char *z4,
         1178  +  const char *zDb,
         1179  +  const char *zTrigger
         1180  +){
         1181  +  int rc = SQLITE_OK;
         1182  +  (void)z4;
         1183  +  (void)zTrigger;
         1184  +  if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){
         1185  +    if( sqlite3_stricmp(zDb, "main")==0 ){
         1186  +      sqlite3expert *p = (sqlite3expert*)pCtx;
         1187  +      IdxTable *pTab;
         1188  +      for(pTab=p->pTable; pTab; pTab=pTab->pNext){
         1189  +        if( 0==sqlite3_stricmp(z3, pTab->zName) ) break;
         1190  +      }
         1191  +      if( pTab ){
         1192  +        IdxWrite *pWrite;
         1193  +        for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){
         1194  +          if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break;
         1195  +        }
         1196  +        if( pWrite==0 ){
         1197  +          pWrite = idxMalloc(&rc, sizeof(IdxWrite));
         1198  +          if( rc==SQLITE_OK ){
         1199  +            pWrite->pTab = pTab;
         1200  +            pWrite->eOp = eOp;
         1201  +            pWrite->pNext = p->pWrite;
         1202  +            p->pWrite = pWrite;
         1203  +          }
         1204  +        }
         1205  +      }
         1206  +    }
         1207  +  }
         1208  +  return rc;
         1209  +}
         1210  +
         1211  +static int idxProcessOneTrigger(
         1212  +  sqlite3expert *p, 
         1213  +  IdxWrite *pWrite, 
         1214  +  char **pzErr
         1215  +){
         1216  +  static const char *zInt = UNIQUE_TABLE_NAME;
         1217  +  static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME;
         1218  +  IdxTable *pTab = pWrite->pTab;
         1219  +  const char *zTab = pTab->zName;
         1220  +  const char *zSql = 
         1221  +    "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_master "
         1222  +    "WHERE tbl_name = %Q AND type IN ('table', 'trigger') "
         1223  +    "ORDER BY type;";
         1224  +  sqlite3_stmt *pSelect = 0;
         1225  +  int rc = SQLITE_OK;
         1226  +  char *zWrite = 0;
         1227  +
         1228  +  /* Create the table and its triggers in the temp schema */
         1229  +  rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab);
         1230  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){
         1231  +    const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0);
         1232  +    rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr);
         1233  +  }
         1234  +  idxFinalize(&rc, pSelect);
         1235  +
         1236  +  /* Rename the table in the temp schema to zInt */
         1237  +  if( rc==SQLITE_OK ){
         1238  +    char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt);
         1239  +    if( z==0 ){
         1240  +      rc = SQLITE_NOMEM;
         1241  +    }else{
         1242  +      rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr);
         1243  +      sqlite3_free(z);
         1244  +    }
         1245  +  }
         1246  +
         1247  +  switch( pWrite->eOp ){
         1248  +    case SQLITE_INSERT: {
         1249  +      int i;
         1250  +      zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt);
         1251  +      for(i=0; i<pTab->nCol; i++){
         1252  +        zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", ");
         1253  +      }
         1254  +      zWrite = idxAppendText(&rc, zWrite, ")");
         1255  +      break;
         1256  +    }
         1257  +    case SQLITE_UPDATE: {
         1258  +      int i;
         1259  +      zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt);
         1260  +      for(i=0; i<pTab->nCol; i++){
         1261  +        zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", 
         1262  +            pTab->aCol[i].zName
         1263  +        );
         1264  +      }
         1265  +      break;
         1266  +    }
         1267  +    default: {
         1268  +      assert( pWrite->eOp==SQLITE_DELETE );
         1269  +      if( rc==SQLITE_OK ){
         1270  +        zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt);
         1271  +        if( zWrite==0 ) rc = SQLITE_NOMEM;
         1272  +      }
         1273  +    }
         1274  +  }
         1275  +
         1276  +  if( rc==SQLITE_OK ){
         1277  +    sqlite3_stmt *pX = 0;
         1278  +    rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
         1279  +    idxFinalize(&rc, pX);
         1280  +    if( rc!=SQLITE_OK ){
         1281  +      idxDatabaseError(p->dbv, pzErr);
         1282  +    }
         1283  +  }
         1284  +  sqlite3_free(zWrite);
         1285  +
         1286  +  if( rc==SQLITE_OK ){
         1287  +    rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr);
         1288  +  }
         1289  +
         1290  +  return rc;
         1291  +}
         1292  +
         1293  +static int idxProcessTriggers(sqlite3expert *p, char **pzErr){
         1294  +  int rc = SQLITE_OK;
         1295  +  IdxWrite *pEnd = 0;
         1296  +  IdxWrite *pFirst = p->pWrite;
         1297  +
         1298  +  while( rc==SQLITE_OK && pFirst!=pEnd ){
         1299  +    IdxWrite *pIter;
         1300  +    for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){
         1301  +      rc = idxProcessOneTrigger(p, pIter, pzErr);
         1302  +    }
         1303  +    pEnd = pFirst;
         1304  +    pFirst = p->pWrite;
         1305  +  }
         1306  +
         1307  +  return rc;
         1308  +}
         1309  +
         1310  +
         1311  +static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
         1312  +  int rc = idxRegisterVtab(p);
         1313  +  sqlite3_stmt *pSchema = 0;
         1314  +
         1315  +  /* For each table in the main db schema:
         1316  +  **
         1317  +  **   1) Add an entry to the p->pTable list, and
         1318  +  **   2) Create the equivalent virtual table in dbv.
         1319  +  */
         1320  +  rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
         1321  +      "SELECT type, name, sql, 1 FROM sqlite_master "
         1322  +      "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' "
         1323  +      " UNION ALL "
         1324  +      "SELECT type, name, sql, 2 FROM sqlite_master "
         1325  +      "WHERE type = 'trigger'"
         1326  +      "  AND tbl_name IN(SELECT name FROM sqlite_master WHERE type = 'view') "
         1327  +      "ORDER BY 4, 1"
         1328  +  );
         1329  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
         1330  +    const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
         1331  +    const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
         1332  +    const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
         1333  +
         1334  +    if( zType[0]=='v' || zType[1]=='r' ){
         1335  +      rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
         1336  +    }else{
         1337  +      IdxTable *pTab;
         1338  +      rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
         1339  +      if( rc==SQLITE_OK ){
         1340  +        int i;
         1341  +        char *zInner = 0;
         1342  +        char *zOuter = 0;
         1343  +        pTab->pNext = p->pTable;
         1344  +        p->pTable = pTab;
         1345  +
         1346  +        /* The statement the vtab will pass to sqlite3_declare_vtab() */
         1347  +        zInner = idxAppendText(&rc, 0, "CREATE TABLE x(");
         1348  +        for(i=0; i<pTab->nCol; i++){
         1349  +          zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", 
         1350  +              (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl
         1351  +          );
         1352  +        }
         1353  +        zInner = idxAppendText(&rc, zInner, ")");
         1354  +
         1355  +        /* The CVT statement to create the vtab */
         1356  +        zOuter = idxAppendText(&rc, 0, 
         1357  +            "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner
         1358  +        );
         1359  +        if( rc==SQLITE_OK ){
         1360  +          rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg);
         1361  +        }
         1362  +        sqlite3_free(zInner);
         1363  +        sqlite3_free(zOuter);
         1364  +      }
         1365  +    }
         1366  +  }
         1367  +  idxFinalize(&rc, pSchema);
         1368  +  return rc;
         1369  +}
         1370  +
         1371  +struct IdxSampleCtx {
         1372  +  int iTarget;
         1373  +  double target;                  /* Target nRet/nRow value */
         1374  +  double nRow;                    /* Number of rows seen */
         1375  +  double nRet;                    /* Number of rows returned */
         1376  +};
         1377  +
         1378  +static void idxSampleFunc(
         1379  +  sqlite3_context *pCtx,
         1380  +  int argc,
         1381  +  sqlite3_value **argv
         1382  +){
         1383  +  struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
         1384  +  int bRet;
         1385  +
         1386  +  (void)argv;
         1387  +  assert( argc==0 );
         1388  +  if( p->nRow==0.0 ){
         1389  +    bRet = 1;
         1390  +  }else{
         1391  +    bRet = (p->nRet / p->nRow) <= p->target;
         1392  +    if( bRet==0 ){
         1393  +      unsigned short rnd;
         1394  +      sqlite3_randomness(2, (void*)&rnd);
         1395  +      bRet = ((int)rnd % 100) <= p->iTarget;
         1396  +    }
         1397  +  }
         1398  +
         1399  +  sqlite3_result_int(pCtx, bRet);
         1400  +  p->nRow += 1.0;
         1401  +  p->nRet += (double)bRet;
         1402  +}
         1403  +
         1404  +struct IdxRemCtx {
         1405  +  int nSlot;
         1406  +  struct IdxRemSlot {
         1407  +    int eType;                    /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */
         1408  +    i64 iVal;                     /* SQLITE_INTEGER value */
         1409  +    double rVal;                  /* SQLITE_FLOAT value */
         1410  +    int nByte;                    /* Bytes of space allocated at z */
         1411  +    int n;                        /* Size of buffer z */
         1412  +    char *z;                      /* SQLITE_TEXT/BLOB value */
         1413  +  } aSlot[1];
         1414  +};
         1415  +
         1416  +/*
         1417  +** Implementation of scalar function rem().
         1418  +*/
         1419  +static void idxRemFunc(
         1420  +  sqlite3_context *pCtx,
         1421  +  int argc,
         1422  +  sqlite3_value **argv
         1423  +){
         1424  +  struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx);
         1425  +  struct IdxRemSlot *pSlot;
         1426  +  int iSlot;
         1427  +  assert( argc==2 );
         1428  +
         1429  +  iSlot = sqlite3_value_int(argv[0]);
         1430  +  assert( iSlot<=p->nSlot );
         1431  +  pSlot = &p->aSlot[iSlot];
         1432  +
         1433  +  switch( pSlot->eType ){
         1434  +    case SQLITE_NULL:
         1435  +      /* no-op */
         1436  +      break;
         1437  +
         1438  +    case SQLITE_INTEGER:
         1439  +      sqlite3_result_int64(pCtx, pSlot->iVal);
         1440  +      break;
         1441  +
         1442  +    case SQLITE_FLOAT:
         1443  +      sqlite3_result_double(pCtx, pSlot->rVal);
         1444  +      break;
         1445  +
         1446  +    case SQLITE_BLOB:
         1447  +      sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
         1448  +      break;
         1449  +
         1450  +    case SQLITE_TEXT:
         1451  +      sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
         1452  +      break;
         1453  +  }
         1454  +
         1455  +  pSlot->eType = sqlite3_value_type(argv[1]);
         1456  +  switch( pSlot->eType ){
         1457  +    case SQLITE_NULL:
         1458  +      /* no-op */
         1459  +      break;
         1460  +
         1461  +    case SQLITE_INTEGER:
         1462  +      pSlot->iVal = sqlite3_value_int64(argv[1]);
         1463  +      break;
         1464  +
         1465  +    case SQLITE_FLOAT:
         1466  +      pSlot->rVal = sqlite3_value_double(argv[1]);
         1467  +      break;
         1468  +
         1469  +    case SQLITE_BLOB:
         1470  +    case SQLITE_TEXT: {
         1471  +      int nByte = sqlite3_value_bytes(argv[1]);
         1472  +      if( nByte>pSlot->nByte ){
         1473  +        char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2);
         1474  +        if( zNew==0 ){
         1475  +          sqlite3_result_error_nomem(pCtx);
         1476  +          return;
         1477  +        }
         1478  +        pSlot->nByte = nByte*2;
         1479  +        pSlot->z = zNew;
         1480  +      }
         1481  +      pSlot->n = nByte;
         1482  +      if( pSlot->eType==SQLITE_BLOB ){
         1483  +        memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte);
         1484  +      }else{
         1485  +        memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte);
         1486  +      }
         1487  +      break;
         1488  +    }
         1489  +  }
         1490  +}
         1491  +
         1492  +static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){
         1493  +  int rc = SQLITE_OK;
         1494  +  const char *zMax = 
         1495  +    "SELECT max(i.seqno) FROM "
         1496  +    "  sqlite_master AS s, "
         1497  +    "  pragma_index_list(s.name) AS l, "
         1498  +    "  pragma_index_info(l.name) AS i "
         1499  +    "WHERE s.type = 'table'";
         1500  +  sqlite3_stmt *pMax = 0;
         1501  +
         1502  +  *pnMax = 0;
         1503  +  rc = idxPrepareStmt(db, &pMax, pzErr, zMax);
         1504  +  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
         1505  +    *pnMax = sqlite3_column_int(pMax, 0) + 1;
         1506  +  }
         1507  +  idxFinalize(&rc, pMax);
         1508  +
         1509  +  return rc;
         1510  +}
         1511  +
         1512  +static int idxPopulateOneStat1(
         1513  +  sqlite3expert *p,
         1514  +  sqlite3_stmt *pIndexXInfo,
         1515  +  sqlite3_stmt *pWriteStat,
         1516  +  const char *zTab,
         1517  +  const char *zIdx,
         1518  +  char **pzErr
         1519  +){
         1520  +  char *zCols = 0;
         1521  +  char *zOrder = 0;
         1522  +  char *zQuery = 0;
         1523  +  int nCol = 0;
         1524  +  int i;
         1525  +  sqlite3_stmt *pQuery = 0;
         1526  +  int *aStat = 0;
         1527  +  int rc = SQLITE_OK;
         1528  +
         1529  +  assert( p->iSample>0 );
         1530  +
         1531  +  /* Formulate the query text */
         1532  +  sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
         1533  +  while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
         1534  +    const char *zComma = zCols==0 ? "" : ", ";
         1535  +    const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
         1536  +    const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
         1537  +    zCols = idxAppendText(&rc, zCols, 
         1538  +        "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
         1539  +    );
         1540  +    zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
         1541  +  }
         1542  +  sqlite3_reset(pIndexXInfo);
         1543  +  if( rc==SQLITE_OK ){
         1544  +    if( p->iSample==100 ){
         1545  +      zQuery = sqlite3_mprintf(
         1546  +          "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
         1547  +      );
         1548  +    }else{
         1549  +      zQuery = sqlite3_mprintf(
         1550  +          "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder
         1551  +      );
         1552  +    }
         1553  +  }
         1554  +  sqlite3_free(zCols);
         1555  +  sqlite3_free(zOrder);
         1556  +
         1557  +  /* Formulate the query text */
         1558  +  if( rc==SQLITE_OK ){
         1559  +    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
         1560  +    rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
         1561  +  }
         1562  +  sqlite3_free(zQuery);
         1563  +
         1564  +  if( rc==SQLITE_OK ){
         1565  +    aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1));
         1566  +  }
         1567  +  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
         1568  +    IdxHashEntry *pEntry;
         1569  +    char *zStat = 0;
         1570  +    for(i=0; i<=nCol; i++) aStat[i] = 1;
         1571  +    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
         1572  +      aStat[0]++;
         1573  +      for(i=0; i<nCol; i++){
         1574  +        if( sqlite3_column_int(pQuery, i)==0 ) break;
         1575  +      }
         1576  +      for(/*no-op*/; i<nCol; i++){
         1577  +        aStat[i+1]++;
         1578  +      }
         1579  +    }
         1580  +
         1581  +    if( rc==SQLITE_OK ){
         1582  +      int s0 = aStat[0];
         1583  +      zStat = sqlite3_mprintf("%d", s0);
         1584  +      if( zStat==0 ) rc = SQLITE_NOMEM;
         1585  +      for(i=1; rc==SQLITE_OK && i<=nCol; i++){
         1586  +        zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]);
         1587  +      }
         1588  +    }
         1589  +
         1590  +    if( rc==SQLITE_OK ){
         1591  +      sqlite3_bind_text(pWriteStat, 1, zTab, -1, SQLITE_STATIC);
         1592  +      sqlite3_bind_text(pWriteStat, 2, zIdx, -1, SQLITE_STATIC);
         1593  +      sqlite3_bind_text(pWriteStat, 3, zStat, -1, SQLITE_STATIC);
         1594  +      sqlite3_step(pWriteStat);
         1595  +      rc = sqlite3_reset(pWriteStat);
         1596  +    }
         1597  +
         1598  +    pEntry = idxHashFind(&p->hIdx, zIdx, STRLEN(zIdx));
         1599  +    if( pEntry ){
         1600  +      assert( pEntry->zVal2==0 );
         1601  +      pEntry->zVal2 = zStat;
         1602  +    }else{
         1603  +      sqlite3_free(zStat);
         1604  +    }
         1605  +  }
         1606  +  sqlite3_free(aStat);
         1607  +  idxFinalize(&rc, pQuery);
         1608  +
         1609  +  return rc;
         1610  +}
         1611  +
         1612  +static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
         1613  +  int rc;
         1614  +  char *zSql;
         1615  +
         1616  +  rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
         1617  +  if( rc!=SQLITE_OK ) return rc;
         1618  +
         1619  +  zSql = sqlite3_mprintf(
         1620  +      "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
         1621  +  );
         1622  +  if( zSql==0 ) return SQLITE_NOMEM;
         1623  +  rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
         1624  +  sqlite3_free(zSql);
         1625  +
         1626  +  return rc;
         1627  +}
         1628  +
         1629  +/*
         1630  +** This function is called as part of sqlite3_expert_analyze(). Candidate
         1631  +** indexes have already been created in database sqlite3expert.dbm, this
         1632  +** function populates sqlite_stat1 table in the same database.
         1633  +**
         1634  +** The stat1 data is generated by querying the 
         1635  +*/
         1636  +static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
         1637  +  int rc = SQLITE_OK;
         1638  +  int nMax =0;
         1639  +  struct IdxRemCtx *pCtx = 0;
         1640  +  struct IdxSampleCtx samplectx; 
         1641  +  int i;
         1642  +  i64 iPrev = -100000;
         1643  +  sqlite3_stmt *pAllIndex = 0;
         1644  +  sqlite3_stmt *pIndexXInfo = 0;
         1645  +  sqlite3_stmt *pWrite = 0;
         1646  +
         1647  +  const char *zAllIndex =
         1648  +    "SELECT s.rowid, s.name, l.name FROM "
         1649  +    "  sqlite_master AS s, "
         1650  +    "  pragma_index_list(s.name) AS l "
         1651  +    "WHERE s.type = 'table'";
         1652  +  const char *zIndexXInfo = 
         1653  +    "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
         1654  +  const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
         1655  +
         1656  +  /* If iSample==0, no sqlite_stat1 data is required. */
         1657  +  if( p->iSample==0 ) return SQLITE_OK;
         1658  +
         1659  +  rc = idxLargestIndex(p->dbm, &nMax, pzErr);
         1660  +  if( nMax<=0 || rc!=SQLITE_OK ) return rc;
         1661  +
         1662  +  rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0);
         1663  +
         1664  +  if( rc==SQLITE_OK ){
         1665  +    int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax);
         1666  +    pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte);
         1667  +  }
         1668  +
         1669  +  if( rc==SQLITE_OK ){
         1670  +    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
         1671  +    rc = sqlite3_create_function(
         1672  +        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
         1673  +    );
         1674  +  }
         1675  +  if( rc==SQLITE_OK ){
         1676  +    rc = sqlite3_create_function(
         1677  +        p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
         1678  +    );
         1679  +  }
         1680  +
         1681  +  if( rc==SQLITE_OK ){
         1682  +    pCtx->nSlot = nMax+1;
         1683  +    rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex);
         1684  +  }
         1685  +  if( rc==SQLITE_OK ){
         1686  +    rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo);
         1687  +  }
         1688  +  if( rc==SQLITE_OK ){
         1689  +    rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite);
         1690  +  }
         1691  +
         1692  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
         1693  +    i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
         1694  +    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
         1695  +    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
         1696  +    if( p->iSample<100 && iPrev!=iRowid ){
         1697  +      samplectx.target = (double)p->iSample / 100.0;
         1698  +      samplectx.iTarget = p->iSample;
         1699  +      samplectx.nRow = 0.0;
         1700  +      samplectx.nRet = 0.0;
         1701  +      rc = idxBuildSampleTable(p, zTab);
         1702  +      if( rc!=SQLITE_OK ) break;
         1703  +    }
         1704  +    rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
         1705  +    iPrev = iRowid;
         1706  +  }
         1707  +  if( rc==SQLITE_OK && p->iSample<100 ){
         1708  +    rc = sqlite3_exec(p->dbv, 
         1709  +        "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0
         1710  +    );
         1711  +  }
         1712  +
         1713  +  idxFinalize(&rc, pAllIndex);
         1714  +  idxFinalize(&rc, pIndexXInfo);
         1715  +  idxFinalize(&rc, pWrite);
         1716  +
         1717  +  for(i=0; i<pCtx->nSlot; i++){
         1718  +    sqlite3_free(pCtx->aSlot[i].z);
         1719  +  }
         1720  +  sqlite3_free(pCtx);
         1721  +
         1722  +  if( rc==SQLITE_OK ){
         1723  +    rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
         1724  +  }
         1725  +
         1726  +  sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
         1727  +  return rc;
         1728  +}
         1729  +
         1730  +/*
         1731  +** Allocate a new sqlite3expert object.
         1732  +*/
         1733  +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
         1734  +  int rc = SQLITE_OK;
         1735  +  sqlite3expert *pNew;
         1736  +
         1737  +  pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
         1738  +
         1739  +  /* Open two in-memory databases to work with. The "vtab database" (dbv)
         1740  +  ** will contain a virtual table corresponding to each real table in
         1741  +  ** the user database schema, and a copy of each view. It is used to
         1742  +  ** collect information regarding the WHERE, ORDER BY and other clauses
         1743  +  ** of the user's query.
         1744  +  */
         1745  +  if( rc==SQLITE_OK ){
         1746  +    pNew->db = db;
         1747  +    pNew->iSample = 100;
         1748  +    rc = sqlite3_open(":memory:", &pNew->dbv);
         1749  +  }
         1750  +  if( rc==SQLITE_OK ){
         1751  +    rc = sqlite3_open(":memory:", &pNew->dbm);
         1752  +    if( rc==SQLITE_OK ){
         1753  +      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
         1754  +    }
         1755  +  }
         1756  +  
         1757  +
         1758  +  /* Copy the entire schema of database [db] into [dbm]. */
         1759  +  if( rc==SQLITE_OK ){
         1760  +    sqlite3_stmt *pSql;
         1761  +    rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         1762  +        "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
         1763  +        " AND sql NOT LIKE 'CREATE VIRTUAL %%'"
         1764  +    );
         1765  +    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
         1766  +      const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
         1767  +      rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
         1768  +    }
         1769  +    idxFinalize(&rc, pSql);
         1770  +  }
         1771  +
         1772  +  /* Create the vtab schema */
         1773  +  if( rc==SQLITE_OK ){
         1774  +    rc = idxCreateVtabSchema(pNew, pzErrmsg);
         1775  +  }
         1776  +
         1777  +  /* Register the auth callback with dbv */
         1778  +  if( rc==SQLITE_OK ){
         1779  +    sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew);
         1780  +  }
         1781  +
         1782  +  /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
         1783  +  ** return the new sqlite3expert handle.  */
         1784  +  if( rc!=SQLITE_OK ){
         1785  +    sqlite3_expert_destroy(pNew);
         1786  +    pNew = 0;
         1787  +  }
         1788  +  return pNew;
         1789  +}
         1790  +
         1791  +/*
         1792  +** Configure an sqlite3expert object.
         1793  +*/
         1794  +int sqlite3_expert_config(sqlite3expert *p, int op, ...){
         1795  +  int rc = SQLITE_OK;
         1796  +  va_list ap;
         1797  +  va_start(ap, op);
         1798  +  switch( op ){
         1799  +    case EXPERT_CONFIG_SAMPLE: {
         1800  +      int iVal = va_arg(ap, int);
         1801  +      if( iVal<0 ) iVal = 0;
         1802  +      if( iVal>100 ) iVal = 100;
         1803  +      p->iSample = iVal;
         1804  +      break;
         1805  +    }
         1806  +    default:
         1807  +      rc = SQLITE_NOTFOUND;
         1808  +      break;
         1809  +  }
         1810  +
         1811  +  va_end(ap);
         1812  +  return rc;
         1813  +}
         1814  +
         1815  +/*
         1816  +** Add an SQL statement to the analysis.
         1817  +*/
         1818  +int sqlite3_expert_sql(
         1819  +  sqlite3expert *p,               /* From sqlite3_expert_new() */
         1820  +  const char *zSql,               /* SQL statement to add */
         1821  +  char **pzErr                    /* OUT: Error message (if any) */
         1822  +){
         1823  +  IdxScan *pScanOrig = p->pScan;
         1824  +  IdxStatement *pStmtOrig = p->pStatement;
         1825  +  int rc = SQLITE_OK;
         1826  +  const char *zStmt = zSql;
         1827  +
         1828  +  if( p->bRun ) return SQLITE_MISUSE;
         1829  +
         1830  +  while( rc==SQLITE_OK && zStmt && zStmt[0] ){
         1831  +    sqlite3_stmt *pStmt = 0;
         1832  +    rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
         1833  +    if( rc==SQLITE_OK ){
         1834  +      if( pStmt ){
         1835  +        IdxStatement *pNew;
         1836  +        const char *z = sqlite3_sql(pStmt);
         1837  +        int n = STRLEN(z);
         1838  +        pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
         1839  +        if( rc==SQLITE_OK ){
         1840  +          pNew->zSql = (char*)&pNew[1];
         1841  +          memcpy(pNew->zSql, z, n+1);
         1842  +          pNew->pNext = p->pStatement;
         1843  +          if( p->pStatement ) pNew->iId = p->pStatement->iId+1;
         1844  +          p->pStatement = pNew;
         1845  +        }
         1846  +        sqlite3_finalize(pStmt);
         1847  +      }
         1848  +    }else{
         1849  +      idxDatabaseError(p->dbv, pzErr);
         1850  +    }
         1851  +  }
         1852  +
         1853  +  if( rc!=SQLITE_OK ){
         1854  +    idxScanFree(p->pScan, pScanOrig);
         1855  +    idxStatementFree(p->pStatement, pStmtOrig);
         1856  +    p->pScan = pScanOrig;
         1857  +    p->pStatement = pStmtOrig;
         1858  +  }
         1859  +
         1860  +  return rc;
         1861  +}
         1862  +
         1863  +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
         1864  +  int rc;
         1865  +  IdxHashEntry *pEntry;
         1866  +
         1867  +  /* Do trigger processing to collect any extra IdxScan structures */
         1868  +  rc = idxProcessTriggers(p, pzErr);
         1869  +
         1870  +  /* Create candidate indexes within the in-memory database file */
         1871  +  if( rc==SQLITE_OK ){
         1872  +    rc = idxCreateCandidates(p);
         1873  +  }
         1874  +
         1875  +  /* Generate the stat1 data */
         1876  +  if( rc==SQLITE_OK ){
         1877  +    rc = idxPopulateStat1(p, pzErr);
         1878  +  }
         1879  +
         1880  +  /* Formulate the EXPERT_REPORT_CANDIDATES text */
         1881  +  for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
         1882  +    p->zCandidates = idxAppendText(&rc, p->zCandidates, 
         1883  +        "%s;%s%s\n", pEntry->zVal, 
         1884  +        pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2
         1885  +    );
         1886  +  }
         1887  +
         1888  +  /* Figure out which of the candidate indexes are preferred by the query
         1889  +  ** planner and report the results to the user.  */
         1890  +  if( rc==SQLITE_OK ){
         1891  +    rc = idxFindIndexes(p, pzErr);
         1892  +  }
         1893  +
         1894  +  if( rc==SQLITE_OK ){
         1895  +    p->bRun = 1;
         1896  +  }
         1897  +  return rc;
         1898  +}
         1899  +
         1900  +/*
         1901  +** Return the total number of statements that have been added to this
         1902  +** sqlite3expert using sqlite3_expert_sql().
         1903  +*/
         1904  +int sqlite3_expert_count(sqlite3expert *p){
         1905  +  int nRet = 0;
         1906  +  if( p->pStatement ) nRet = p->pStatement->iId+1;
         1907  +  return nRet;
         1908  +}
         1909  +
         1910  +/*
         1911  +** Return a component of the report.
         1912  +*/
         1913  +const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
         1914  +  const char *zRet = 0;
         1915  +  IdxStatement *pStmt;
         1916  +
         1917  +  if( p->bRun==0 ) return 0;
         1918  +  for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
         1919  +  switch( eReport ){
         1920  +    case EXPERT_REPORT_SQL:
         1921  +      if( pStmt ) zRet = pStmt->zSql;
         1922  +      break;
         1923  +    case EXPERT_REPORT_INDEXES:
         1924  +      if( pStmt ) zRet = pStmt->zIdx;
         1925  +      break;
         1926  +    case EXPERT_REPORT_PLAN:
         1927  +      if( pStmt ) zRet = pStmt->zEQP;
         1928  +      break;
         1929  +    case EXPERT_REPORT_CANDIDATES:
         1930  +      zRet = p->zCandidates;
         1931  +      break;
         1932  +  }
         1933  +  return zRet;
         1934  +}
         1935  +
         1936  +/*
         1937  +** Free an sqlite3expert object.
         1938  +*/
         1939  +void sqlite3_expert_destroy(sqlite3expert *p){
         1940  +  if( p ){
         1941  +    sqlite3_close(p->dbm);
         1942  +    sqlite3_close(p->dbv);
         1943  +    idxScanFree(p->pScan, 0);
         1944  +    idxStatementFree(p->pStatement, 0);
         1945  +    idxTableFree(p->pTable);
         1946  +    idxWriteFree(p->pWrite);
         1947  +    idxHashClear(&p->hIdx);
         1948  +    sqlite3_free(p->zCandidates);
         1949  +    sqlite3_free(p);
         1950  +  }
         1951  +}
         1952  +
         1953  +#endif /* ifndef SQLITE_OMIT_VIRTUAL_TABLE */

Added ext/expert/sqlite3expert.h.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +
           15  +#include "sqlite3.h"
           16  +
           17  +typedef struct sqlite3expert sqlite3expert;
           18  +
           19  +/*
           20  +** Create a new sqlite3expert object.
           21  +**
           22  +** If successful, a pointer to the new object is returned and (*pzErr) set
           23  +** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
           24  +** an English-language error message. In this case it is the responsibility
           25  +** of the caller to eventually free the error message buffer using
           26  +** sqlite3_free().
           27  +*/
           28  +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
           29  +
           30  +/*
           31  +** Configure an sqlite3expert object.
           32  +**
           33  +** EXPERT_CONFIG_SAMPLE:
           34  +**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
           35  +**   each candidate index. This involves scanning and sorting the entire
           36  +**   contents of each user database table once for each candidate index
           37  +**   associated with the table. For large databases, this can be 
           38  +**   prohibitively slow. This option allows the sqlite3expert object to
           39  +**   be configured so that sqlite_stat1 data is instead generated based on a
           40  +**   subset of each table, or so that no sqlite_stat1 data is used at all.
           41  +**
           42  +**   A single integer argument is passed to this option. If the value is less
           43  +**   than or equal to zero, then no sqlite_stat1 data is generated or used by
           44  +**   the analysis - indexes are recommended based on the database schema only.
           45  +**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
           46  +**   generated for each candidate index (this is the default). Finally, if the
           47  +**   value falls between 0 and 100, then it represents the percentage of user
           48  +**   table rows that should be considered when generating sqlite_stat1 data.
           49  +**
           50  +**   Examples:
           51  +**
           52  +**     // Do not generate any sqlite_stat1 data
           53  +**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
           54  +**
           55  +**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
           56  +**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
           57  +*/
           58  +int sqlite3_expert_config(sqlite3expert *p, int op, ...);
           59  +
           60  +#define EXPERT_CONFIG_SAMPLE 1    /* int */
           61  +
           62  +/*
           63  +** Specify zero or more SQL statements to be included in the analysis.
           64  +**
           65  +** Buffer zSql must contain zero or more complete SQL statements. This
           66  +** function parses all statements contained in the buffer and adds them
           67  +** to the internal list of statements to analyze. If successful, SQLITE_OK
           68  +** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
           69  +** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
           70  +** may be set to point to an English language error message. In this case
           71  +** the caller is responsible for eventually freeing the error message buffer
           72  +** using sqlite3_free().
           73  +**
           74  +** If an error does occur while processing one of the statements in the
           75  +** buffer passed as the second argument, none of the statements in the
           76  +** buffer are added to the analysis.
           77  +**
           78  +** This function must be called before sqlite3_expert_analyze(). If a call
           79  +** to this function is made on an sqlite3expert object that has already
           80  +** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
           81  +** immediately and no statements are added to the analysis.
           82  +*/
           83  +int sqlite3_expert_sql(
           84  +  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
           85  +  const char *zSql,               /* SQL statement(s) to add */
           86  +  char **pzErr                    /* OUT: Error message (if any) */
           87  +);
           88  +
           89  +
           90  +/*
           91  +** This function is called after the sqlite3expert object has been configured
           92  +** with all SQL statements using sqlite3_expert_sql() to actually perform
           93  +** the analysis. Once this function has been called, it is not possible to
           94  +** add further SQL statements to the analysis.
           95  +**
           96  +** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
           97  +** an error occurs, an SQLite error code is returned and (*pzErr) set to 
           98  +** point to a buffer containing an English language error message. In this
           99  +** case it is the responsibility of the caller to eventually free the buffer
          100  +** using sqlite3_free().
          101  +**
          102  +** If an error does occur within this function, the sqlite3expert object
          103  +** is no longer useful for any purpose. At that point it is no longer
          104  +** possible to add further SQL statements to the object or to re-attempt
          105  +** the analysis. The sqlite3expert object must still be freed using a call
          106  +** sqlite3_expert_destroy().
          107  +*/
          108  +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
          109  +
          110  +/*
          111  +** Return the total number of statements loaded using sqlite3_expert_sql().
          112  +** The total number of SQL statements may be different from the total number
          113  +** to calls to sqlite3_expert_sql().
          114  +*/
          115  +int sqlite3_expert_count(sqlite3expert*);
          116  +
          117  +/*
          118  +** Return a component of the report.
          119  +**
          120  +** This function is called after sqlite3_expert_analyze() to extract the
          121  +** results of the analysis. Each call to this function returns either a
          122  +** NULL pointer or a pointer to a buffer containing a nul-terminated string.
          123  +** The value passed as the third argument must be one of the EXPERT_REPORT_*
          124  +** #define constants defined below.
          125  +**
          126  +** For some EXPERT_REPORT_* parameters, the buffer returned contains 
          127  +** information relating to a specific SQL statement. In these cases that
          128  +** SQL statement is identified by the value passed as the second argument.
          129  +** SQL statements are numbered from 0 in the order in which they are parsed.
          130  +** If an out-of-range value (less than zero or equal to or greater than the
          131  +** value returned by sqlite3_expert_count()) is passed as the second argument
          132  +** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
          133  +**
          134  +** EXPERT_REPORT_SQL:
          135  +**   Return the text of SQL statement iStmt.
          136  +**
          137  +** EXPERT_REPORT_INDEXES:
          138  +**   Return a buffer containing the CREATE INDEX statements for all recommended
          139  +**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
          140  +**   is returned.
          141  +**
          142  +** EXPERT_REPORT_PLAN:
          143  +**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
          144  +**   iStmt after the proposed indexes have been added to the database schema.
          145  +**
          146  +** EXPERT_REPORT_CANDIDATES:
          147  +**   Return a pointer to a buffer containing the CREATE INDEX statements 
          148  +**   for all indexes that were tested (for all SQL statements). The iStmt
          149  +**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
          150  +*/
          151  +const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
          152  +
          153  +/*
          154  +** Values for the third argument passed to sqlite3_expert_report().
          155  +*/
          156  +#define EXPERT_REPORT_SQL        1
          157  +#define EXPERT_REPORT_INDEXES    2
          158  +#define EXPERT_REPORT_PLAN       3
          159  +#define EXPERT_REPORT_CANDIDATES 4
          160  +
          161  +/*
          162  +** Free an (sqlite3expert*) handle and all associated resources. There 
          163  +** should be one call to this function for each successful call to 
          164  +** sqlite3-expert_new().
          165  +*/
          166  +void sqlite3_expert_destroy(sqlite3expert*);
          167  +
          168  +

Added ext/expert/test_expert.c.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +#if defined(SQLITE_TEST)
           15  +
           16  +#include "sqlite3expert.h"
           17  +#include <assert.h>
           18  +#include <string.h>
           19  +
           20  +#if defined(INCLUDE_SQLITE_TCL_H)
           21  +#  include "sqlite_tcl.h"
           22  +#else
           23  +#  include "tcl.h"
           24  +#  ifndef SQLITE_TCLAPI
           25  +#    define SQLITE_TCLAPI
           26  +#  endif
           27  +#endif
           28  +
           29  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           30  +
           31  +/*
           32  +** Extract an sqlite3* db handle from the object passed as the second
           33  +** argument. If successful, set *pDb to point to the db handle and return
           34  +** TCL_OK. Otherwise, return TCL_ERROR.
           35  +*/
           36  +static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
           37  +  Tcl_CmdInfo info;
           38  +  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
           39  +    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
           40  +    return TCL_ERROR;
           41  +  }
           42  +
           43  +  *pDb = *(sqlite3 **)info.objClientData;
           44  +  return TCL_OK;
           45  +}
           46  +
           47  +
           48  +/*
           49  +** Tclcmd:  $expert sql SQL
           50  +**          $expert analyze
           51  +**          $expert count
           52  +**          $expert report STMT EREPORT
           53  +**          $expert destroy
           54  +*/
           55  +static int SQLITE_TCLAPI testExpertCmd(
           56  +  void *clientData,
           57  +  Tcl_Interp *interp,
           58  +  int objc,
           59  +  Tcl_Obj *CONST objv[]
           60  +){
           61  +  sqlite3expert *pExpert = (sqlite3expert*)clientData;
           62  +  struct Subcmd {
           63  +    const char *zSub;
           64  +    int nArg;
           65  +    const char *zMsg;
           66  +  } aSub[] = {
           67  +    { "sql",       1, "TABLE",        }, /* 0 */
           68  +    { "analyze",   0, "",             }, /* 1 */
           69  +    { "count",     0, "",             }, /* 2 */
           70  +    { "report",    2, "STMT EREPORT", }, /* 3 */
           71  +    { "destroy",   0, "",             }, /* 4 */
           72  +    { 0 }
           73  +  };
           74  +  int iSub;
           75  +  int rc = TCL_OK;
           76  +  char *zErr = 0;
           77  +
           78  +  if( objc<2 ){
           79  +    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
           80  +    return TCL_ERROR;
           81  +  }
           82  +  rc = Tcl_GetIndexFromObjStruct(interp, 
           83  +      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
           84  +  );
           85  +  if( rc!=TCL_OK ) return rc;
           86  +  if( objc!=2+aSub[iSub].nArg ){
           87  +    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
           88  +    return TCL_ERROR;
           89  +  }
           90  +
           91  +  switch( iSub ){
           92  +    case 0: {      /* sql */
           93  +      char *zArg = Tcl_GetString(objv[2]);
           94  +      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
           95  +      break;
           96  +    }
           97  +
           98  +    case 1: {      /* analyze */
           99  +      rc = sqlite3_expert_analyze(pExpert, &zErr);
          100  +      break;
          101  +    }
          102  +
          103  +    case 2: {      /* count */
          104  +      int n = sqlite3_expert_count(pExpert);
          105  +      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
          106  +      break;
          107  +    }
          108  +
          109  +    case 3: {      /* report */
          110  +      const char *aEnum[] = {
          111  +        "sql", "indexes", "plan", "candidates", 0
          112  +      };
          113  +      int iEnum;
          114  +      int iStmt;
          115  +      const char *zReport;
          116  +
          117  +      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
          118  +       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
          119  +      ){
          120  +        return TCL_ERROR;
          121  +      }
          122  +
          123  +      assert( EXPERT_REPORT_SQL==1 );
          124  +      assert( EXPERT_REPORT_INDEXES==2 );
          125  +      assert( EXPERT_REPORT_PLAN==3 );
          126  +      assert( EXPERT_REPORT_CANDIDATES==4 );
          127  +      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
          128  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
          129  +      break;
          130  +    }
          131  +
          132  +    default:       /* destroy */
          133  +      assert( iSub==4 );     
          134  +      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
          135  +      break;
          136  +  }
          137  +
          138  +  if( rc!=TCL_OK ){
          139  +    if( zErr ){
          140  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
          141  +    }else{
          142  +      extern const char *sqlite3ErrName(int);
          143  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
          144  +    }
          145  +  }
          146  +  sqlite3_free(zErr);
          147  +  return rc;
          148  +}
          149  +
          150  +static void SQLITE_TCLAPI testExpertDel(void *clientData){
          151  +  sqlite3expert *pExpert = (sqlite3expert*)clientData;
          152  +  sqlite3_expert_destroy(pExpert);
          153  +}
          154  +
          155  +/*
          156  +** sqlite3_expert_new DB
          157  +*/
          158  +static int SQLITE_TCLAPI test_sqlite3_expert_new(
          159  +  void * clientData,
          160  +  Tcl_Interp *interp,
          161  +  int objc,
          162  +  Tcl_Obj *CONST objv[]
          163  +){
          164  +  static int iCmd = 0;
          165  +  sqlite3 *db;
          166  +  char *zCmd = 0;
          167  +  char *zErr = 0;
          168  +  sqlite3expert *pExpert;
          169  +  int rc = TCL_OK;
          170  +
          171  +  if( objc!=2 ){
          172  +    Tcl_WrongNumArgs(interp, 1, objv, "DB");
          173  +    return TCL_ERROR;
          174  +  }
          175  +  if( dbHandleFromObj(interp, objv[1], &db) ){
          176  +    return TCL_ERROR;
          177  +  }
          178  +
          179  +  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
          180  +  if( zCmd==0 ){
          181  +    Tcl_AppendResult(interp, "out of memory", (char*)0);
          182  +    return TCL_ERROR;
          183  +  }
          184  +
          185  +  pExpert = sqlite3_expert_new(db, &zErr);
          186  +  if( pExpert==0 ){
          187  +    Tcl_AppendResult(interp, zErr, (char*)0);
          188  +    rc = TCL_ERROR;
          189  +  }else{
          190  +    void *p = (void*)pExpert;
          191  +    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
          192  +    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
          193  +  }
          194  +
          195  +  sqlite3_free(zCmd);
          196  +  sqlite3_free(zErr);
          197  +  return rc;
          198  +}
          199  +
          200  +#endif  /* ifndef SQLITE_OMIT_VIRTUALTABLE */
          201  +
          202  +int TestExpert_Init(Tcl_Interp *interp){
          203  +#ifndef SQLITE_OMIT_VIRTUALTABLE
          204  +  struct Cmd {
          205  +    const char *zCmd;
          206  +    Tcl_ObjCmdProc *xProc;
          207  +  } aCmd[] = {
          208  +    { "sqlite3_expert_new", test_sqlite3_expert_new },
          209  +  };
          210  +  int i;
          211  +
          212  +  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
          213  +    struct Cmd *p = &aCmd[i];
          214  +    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
          215  +  }
          216  +#endif
          217  +  return TCL_OK;
          218  +}
          219  +
          220  +#endif

Changes to ext/fts3/fts3.c.

  1817   1817     sqlite3_int64 *piFirst,         /* OUT: Selected child node */
  1818   1818     sqlite3_int64 *piLast           /* OUT: Selected child node */
  1819   1819   ){
  1820   1820     int rc = SQLITE_OK;             /* Return code */
  1821   1821     const char *zCsr = zNode;       /* Cursor to iterate through node */
  1822   1822     const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  1823   1823     char *zBuffer = 0;              /* Buffer to load terms into */
  1824         -  int nAlloc = 0;                 /* Size of allocated buffer */
         1824  +  i64 nAlloc = 0;                 /* Size of allocated buffer */
  1825   1825     int isFirstTerm = 1;            /* True when processing first term on page */
  1826   1826     sqlite3_int64 iChild;           /* Block id of child node to descend to */
  1827   1827   
  1828   1828     /* Skip over the 'height' varint that occurs at the start of every 
  1829   1829     ** interior node. Then load the blockid of the left-child of the b-tree
  1830   1830     ** node into variable iChild.  
  1831   1831     **
................................................................................
  1855   1855       if( !isFirstTerm ){
  1856   1856         zCsr += fts3GetVarint32(zCsr, &nPrefix);
  1857   1857       }
  1858   1858       isFirstTerm = 0;
  1859   1859       zCsr += fts3GetVarint32(zCsr, &nSuffix);
  1860   1860       
  1861   1861       assert( nPrefix>=0 && nSuffix>=0 );
  1862         -    if( &zCsr[nSuffix]>zEnd ){
         1862  +    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
  1863   1863         rc = FTS_CORRUPT_VTAB;
  1864   1864         goto finish_scan;
  1865   1865       }
  1866         -    if( nPrefix+nSuffix>nAlloc ){
         1866  +    if( (i64)nPrefix+nSuffix>nAlloc ){
  1867   1867         char *zNew;
  1868         -      nAlloc = (nPrefix+nSuffix) * 2;
  1869         -      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
         1868  +      nAlloc = ((i64)nPrefix+nSuffix) * 2;
         1869  +      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
  1870   1870         if( !zNew ){
  1871   1871           rc = SQLITE_NOMEM;
  1872   1872           goto finish_scan;
  1873   1873         }
  1874   1874         zBuffer = zNew;
  1875   1875       }
  1876   1876       assert( zBuffer );
................................................................................
  3804   3804   **
  3805   3805   ** Flush the contents of the pending-terms table to disk.
  3806   3806   */
  3807   3807   static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  3808   3808     int rc = SQLITE_OK;
  3809   3809     UNUSED_PARAMETER(iSavepoint);
  3810   3810     assert( ((Fts3Table *)pVtab)->inTransaction );
  3811         -  assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
         3811  +  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
  3812   3812     TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
  3813   3813     if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
  3814   3814       rc = fts3SyncMethod(pVtab);
  3815   3815     }
  3816   3816     return rc;
  3817   3817   }
  3818   3818   
................................................................................
  3841   3841     UNUSED_PARAMETER(iSavepoint);
  3842   3842     assert( p->inTransaction );
  3843   3843     assert( p->mxSavepoint >= iSavepoint );
  3844   3844     TESTONLY( p->mxSavepoint = iSavepoint );
  3845   3845     sqlite3Fts3PendingTermsClear(p);
  3846   3846     return SQLITE_OK;
  3847   3847   }
         3848  +
         3849  +/*
         3850  +** Return true if zName is the extension on one of the shadow tables used
         3851  +** by this module.
         3852  +*/
         3853  +static int fts3ShadowName(const char *zName){
         3854  +  static const char *azName[] = {
         3855  +    "content", "docsize", "segdir", "segments", "stat", 
         3856  +  };
         3857  +  unsigned int i;
         3858  +  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
         3859  +    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
         3860  +  }
         3861  +  return 0;
         3862  +}
  3848   3863   
  3849   3864   static const sqlite3_module fts3Module = {
  3850         -  /* iVersion      */ 2,
         3865  +  /* iVersion      */ 3,
  3851   3866     /* xCreate       */ fts3CreateMethod,
  3852   3867     /* xConnect      */ fts3ConnectMethod,
  3853   3868     /* xBestIndex    */ fts3BestIndexMethod,
  3854   3869     /* xDisconnect   */ fts3DisconnectMethod,
  3855   3870     /* xDestroy      */ fts3DestroyMethod,
  3856   3871     /* xOpen         */ fts3OpenMethod,
  3857   3872     /* xClose        */ fts3CloseMethod,
................................................................................
  3866   3881     /* xCommit       */ fts3CommitMethod,
  3867   3882     /* xRollback     */ fts3RollbackMethod,
  3868   3883     /* xFindFunction */ fts3FindFunctionMethod,
  3869   3884     /* xRename */       fts3RenameMethod,
  3870   3885     /* xSavepoint    */ fts3SavepointMethod,
  3871   3886     /* xRelease      */ fts3ReleaseMethod,
  3872   3887     /* xRollbackTo   */ fts3RollbackToMethod,
         3888  +  /* xShadowName   */ fts3ShadowName,
  3873   3889   };
  3874   3890   
  3875   3891   /*
  3876   3892   ** This function is registered as the module destructor (called when an
  3877   3893   ** FTS3 enabled database connection is closed). It frees the memory
  3878   3894   ** allocated for the tokenizer hash table.
  3879   3895   */
................................................................................
  3959   3975       ){
  3960   3976         rc = SQLITE_NOMEM;
  3961   3977       }
  3962   3978     }
  3963   3979   
  3964   3980   #ifdef SQLITE_TEST
  3965   3981     if( rc==SQLITE_OK ){
  3966         -    rc = sqlite3Fts3ExprInitTestInterface(db);
         3982  +    rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
  3967   3983     }
  3968   3984   #endif
  3969   3985   
  3970   3986     /* Create the virtual table wrapper around the hash-table and overload 
  3971   3987     ** the four scalar functions. If this is successful, register the
  3972   3988     ** module with sqlite.
  3973   3989     */
................................................................................
  4146   4162       }
  4147   4163       assert( pToken->pSegcsr==0 );
  4148   4164     }
  4149   4165   
  4150   4166     return rc;
  4151   4167   }
  4152   4168   
         4169  +#ifndef SQLITE_DISABLE_FTS4_DEFERRED
  4153   4170   /*
  4154   4171   ** This function is called on each phrase after the position lists for
  4155   4172   ** any deferred tokens have been loaded into memory. It updates the phrases
  4156   4173   ** current position list to include only those positions that are really
  4157   4174   ** instances of the phrase (after considering deferred tokens). If this
  4158   4175   ** means that the phrase does not appear in the current row, doclist.pList
  4159   4176   ** and doclist.nList are both zeroed.
................................................................................
  4249   4266         }
  4250   4267         sqlite3_free(aPoslist);
  4251   4268       }
  4252   4269     }
  4253   4270   
  4254   4271     return SQLITE_OK;
  4255   4272   }
         4273  +#endif /* SQLITE_DISABLE_FTS4_DEFERRED */
  4256   4274   
  4257   4275   /*
  4258   4276   ** Maximum number of tokens a phrase may have to be considered for the
  4259   4277   ** incremental doclists strategy.
  4260   4278   */
  4261   4279   #define MAX_INCR_PHRASE_TOKENS 4
  4262   4280   

Changes to ext/fts3/fts3Int.h.

   580    580   
   581    581   /* fts3_expr.c */
   582    582   int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
   583    583     char **, int, int, int, const char *, int, Fts3Expr **, char **
   584    584   );
   585    585   void sqlite3Fts3ExprFree(Fts3Expr *);
   586    586   #ifdef SQLITE_TEST
   587         -int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
          587  +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
   588    588   int sqlite3Fts3InitTerm(sqlite3 *db);
   589    589   #endif
   590    590   
   591    591   int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
   592    592     sqlite3_tokenizer_cursor **
   593    593   );
   594    594   

Changes to ext/fts3/fts3_aux.c.

   535    535        0,                           /* xSync         */
   536    536        0,                           /* xCommit       */
   537    537        0,                           /* xRollback     */
   538    538        0,                           /* xFindFunction */
   539    539        0,                           /* xRename       */
   540    540        0,                           /* xSavepoint    */
   541    541        0,                           /* xRelease      */
   542         -     0                            /* xRollbackTo   */
          542  +     0,                           /* xRollbackTo   */
          543  +     0                            /* xShadowName   */
   543    544     };
   544    545     int rc;                         /* Return code */
   545    546   
   546    547     rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0);
   547    548     return rc;
   548    549   }
   549    550   
   550    551   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

Changes to ext/fts3/fts3_expr.c.

  1104   1104   ** Everything after this point is just test code.
  1105   1105   */
  1106   1106   
  1107   1107   #ifdef SQLITE_TEST
  1108   1108   
  1109   1109   #include <stdio.h>
  1110   1110   
  1111         -/*
  1112         -** Function to query the hash-table of tokenizers (see README.tokenizers).
  1113         -*/
  1114         -static int queryTestTokenizer(
  1115         -  sqlite3 *db, 
  1116         -  const char *zName,  
  1117         -  const sqlite3_tokenizer_module **pp
  1118         -){
  1119         -  int rc;
  1120         -  sqlite3_stmt *pStmt;
  1121         -  const char zSql[] = "SELECT fts3_tokenizer(?)";
  1122         -
  1123         -  *pp = 0;
  1124         -  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  1125         -  if( rc!=SQLITE_OK ){
  1126         -    return rc;
  1127         -  }
  1128         -
  1129         -  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  1130         -  if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1131         -    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
  1132         -      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
  1133         -    }
  1134         -  }
  1135         -
  1136         -  return sqlite3_finalize(pStmt);
  1137         -}
  1138         -
  1139   1111   /*
  1140   1112   ** Return a pointer to a buffer containing a text representation of the
  1141   1113   ** expression passed as the first argument. The buffer is obtained from
  1142   1114   ** sqlite3_malloc(). It is the responsibility of the caller to use 
  1143   1115   ** sqlite3_free() to release the memory. If an OOM condition is encountered,
  1144   1116   ** NULL is returned.
  1145   1117   **
................................................................................
  1199   1171   ** to parse the query expression (see README.tokenizers). The second argument
  1200   1172   ** is the query expression to parse. Each subsequent argument is the name
  1201   1173   ** of a column of the fts3 table that the query expression may refer to.
  1202   1174   ** For example:
  1203   1175   **
  1204   1176   **   SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
  1205   1177   */
  1206         -static void fts3ExprTest(
         1178  +static void fts3ExprTestCommon(
         1179  +  int bRebalance,
  1207   1180     sqlite3_context *context,
  1208   1181     int argc,
  1209   1182     sqlite3_value **argv
  1210   1183   ){
  1211         -  sqlite3_tokenizer_module const *pModule = 0;
  1212   1184     sqlite3_tokenizer *pTokenizer = 0;
  1213   1185     int rc;
  1214   1186     char **azCol = 0;
  1215   1187     const char *zExpr;
  1216   1188     int nExpr;
  1217   1189     int nCol;
  1218   1190     int ii;
  1219   1191     Fts3Expr *pExpr;
  1220   1192     char *zBuf = 0;
  1221         -  sqlite3 *db = sqlite3_context_db_handle(context);
         1193  +  Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
         1194  +  const char *zTokenizer = 0;
         1195  +  char *zErr = 0;
  1222   1196   
  1223   1197     if( argc<3 ){
  1224   1198       sqlite3_result_error(context, 
  1225   1199           "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
  1226   1200       );
  1227   1201       return;
  1228   1202     }
  1229   1203   
  1230         -  rc = queryTestTokenizer(db,
  1231         -                          (const char *)sqlite3_value_text(argv[0]), &pModule);
  1232         -  if( rc==SQLITE_NOMEM ){
  1233         -    sqlite3_result_error_nomem(context);
  1234         -    goto exprtest_out;
  1235         -  }else if( !pModule ){
  1236         -    sqlite3_result_error(context, "No such tokenizer module", -1);
  1237         -    goto exprtest_out;
         1204  +  zTokenizer = (const char*)sqlite3_value_text(argv[0]);
         1205  +  rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
         1206  +  if( rc!=SQLITE_OK ){
         1207  +    if( rc==SQLITE_NOMEM ){
         1208  +      sqlite3_result_error_nomem(context);
         1209  +    }else{
         1210  +      sqlite3_result_error(context, zErr, -1);
         1211  +    }
         1212  +    sqlite3_free(zErr);
         1213  +    return;
  1238   1214     }
  1239   1215   
  1240         -  rc = pModule->xCreate(0, 0, &pTokenizer);
  1241         -  assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
  1242         -  if( rc==SQLITE_NOMEM ){
  1243         -    sqlite3_result_error_nomem(context);
  1244         -    goto exprtest_out;
  1245         -  }
  1246         -  pTokenizer->pModule = pModule;
  1247         -
  1248   1216     zExpr = (const char *)sqlite3_value_text(argv[1]);
  1249   1217     nExpr = sqlite3_value_bytes(argv[1]);
  1250   1218     nCol = argc-2;
  1251   1219     azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
  1252   1220     if( !azCol ){
  1253   1221       sqlite3_result_error_nomem(context);
  1254   1222       goto exprtest_out;
  1255   1223     }
  1256   1224     for(ii=0; ii<nCol; ii++){
  1257   1225       azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
  1258   1226     }
  1259   1227   
  1260         -  if( sqlite3_user_data(context) ){
         1228  +  if( bRebalance ){
  1261   1229       char *zDummy = 0;
  1262   1230       rc = sqlite3Fts3ExprParse(
  1263   1231           pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
  1264   1232       );
  1265   1233       assert( rc==SQLITE_OK || pExpr==0 );
  1266   1234       sqlite3_free(zDummy);
  1267   1235     }else{
................................................................................
  1279   1247       sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
  1280   1248       sqlite3_free(zBuf);
  1281   1249     }
  1282   1250   
  1283   1251     sqlite3Fts3ExprFree(pExpr);
  1284   1252   
  1285   1253   exprtest_out:
  1286         -  if( pModule && pTokenizer ){
  1287         -    rc = pModule->xDestroy(pTokenizer);
         1254  +  if( pTokenizer ){
         1255  +    rc = pTokenizer->pModule->xDestroy(pTokenizer);
  1288   1256     }
  1289   1257     sqlite3_free(azCol);
  1290   1258   }
         1259  +
         1260  +static void fts3ExprTest(
         1261  +  sqlite3_context *context,
         1262  +  int argc,
         1263  +  sqlite3_value **argv
         1264  +){
         1265  +  fts3ExprTestCommon(0, context, argc, argv);
         1266  +}
         1267  +static void fts3ExprTestRebalance(
         1268  +  sqlite3_context *context,
         1269  +  int argc,
         1270  +  sqlite3_value **argv
         1271  +){
         1272  +  fts3ExprTestCommon(1, context, argc, argv);
         1273  +}
  1291   1274   
  1292   1275   /*
  1293   1276   ** Register the query expression parser test function fts3_exprtest() 
  1294   1277   ** with database connection db. 
  1295   1278   */
  1296         -int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
         1279  +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
  1297   1280     int rc = sqlite3_create_function(
  1298         -      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
         1281  +      db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
  1299   1282     );
  1300   1283     if( rc==SQLITE_OK ){
  1301   1284       rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", 
  1302         -        -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
         1285  +        -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
  1303   1286       );
  1304   1287     }
  1305   1288     return rc;
  1306   1289   }
  1307   1290   
  1308   1291   #endif
  1309   1292   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

Changes to ext/fts3/fts3_term.c.

   357    357        0,                           /* xSync         */
   358    358        0,                           /* xCommit       */
   359    359        0,                           /* xRollback     */
   360    360        0,                           /* xFindFunction */
   361    361        0,                           /* xRename       */
   362    362        0,                           /* xSavepoint    */
   363    363        0,                           /* xRelease      */
   364         -     0                            /* xRollbackTo   */
          364  +     0,                           /* xRollbackTo   */
          365  +     0                            /* xShadowName   */
   365    366     };
   366    367     int rc;                         /* Return code */
   367    368   
   368    369     rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0);
   369    370     return rc;
   370    371   }
   371    372   
   372    373   #endif
   373    374   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

Changes to ext/fts3/fts3_tokenize_vtab.c.

   439    439        0,                           /* xSync         */
   440    440        0,                           /* xCommit       */
   441    441        0,                           /* xRollback     */
   442    442        0,                           /* xFindFunction */
   443    443        0,                           /* xRename       */
   444    444        0,                           /* xSavepoint    */
   445    445        0,                           /* xRelease      */
   446         -     0                            /* xRollbackTo   */
          446  +     0,                           /* xRollbackTo   */
          447  +     0                            /* xShadowName   */
   447    448     };
   448    449     int rc;                         /* Return code */
   449    450   
   450    451     rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash);
   451    452     return rc;
   452    453   }
   453    454   
   454    455   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

Changes to ext/fts3/fts3_write.c.

  1370   1370     rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  1371   1371     if( rc!=SQLITE_OK ) return rc;
  1372   1372     
  1373   1373     /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  1374   1374     ** safe (no risk of overread) even if the node data is corrupted. */
  1375   1375     pNext += fts3GetVarint32(pNext, &nPrefix);
  1376   1376     pNext += fts3GetVarint32(pNext, &nSuffix);
  1377         -  if( nPrefix<0 || nSuffix<=0 
  1378         -   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
         1377  +  if( nSuffix<=0 
         1378  +   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
         1379  +   || nPrefix>pReader->nTermAlloc
  1379   1380     ){
  1380   1381       return FTS_CORRUPT_VTAB;
  1381   1382     }
  1382   1383   
  1383         -  if( nPrefix+nSuffix>pReader->nTermAlloc ){
  1384         -    int nNew = (nPrefix+nSuffix)*2;
  1385         -    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
         1384  +  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
         1385  +  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
         1386  +  ** overflow - hence the (i64) casts.  */
         1387  +  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
         1388  +    i64 nNew = ((i64)nPrefix+nSuffix)*2;
         1389  +    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
  1386   1390       if( !zNew ){
  1387   1391         return SQLITE_NOMEM;
  1388   1392       }
  1389   1393       pReader->zTerm = zNew;
  1390   1394       pReader->nTermAlloc = nNew;
  1391   1395     }
  1392   1396   
................................................................................
  1400   1404     pReader->aDoclist = pNext;
  1401   1405     pReader->pOffsetList = 0;
  1402   1406   
  1403   1407     /* Check that the doclist does not appear to extend past the end of the
  1404   1408     ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  1405   1409     ** of these statements is untrue, then the data structure is corrupt.
  1406   1410     */
  1407         -  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
         1411  +  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
  1408   1412      || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  1409   1413     ){
  1410   1414       return FTS_CORRUPT_VTAB;
  1411   1415     }
  1412   1416     return SQLITE_OK;
  1413   1417   }
  1414   1418   
................................................................................
  1904   1908     sqlite3_stmt *pStmt;
  1905   1909     int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
  1906   1910     if( rc==SQLITE_OK ){
  1907   1911       sqlite3_bind_int64(pStmt, 1, iBlock);
  1908   1912       sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
  1909   1913       sqlite3_step(pStmt);
  1910   1914       rc = sqlite3_reset(pStmt);
         1915  +    sqlite3_bind_null(pStmt, 2);
  1911   1916     }
  1912   1917     return rc;
  1913   1918   }
  1914   1919   
  1915   1920   /*
  1916   1921   ** Find the largest relative level number in the table. If successful, set
  1917   1922   ** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs,
................................................................................
  1960   1965         char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData);
  1961   1966         if( !zEnd ) return SQLITE_NOMEM;
  1962   1967         sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free);
  1963   1968       }
  1964   1969       sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
  1965   1970       sqlite3_step(pStmt);
  1966   1971       rc = sqlite3_reset(pStmt);
         1972  +    sqlite3_bind_null(pStmt, 6);
  1967   1973     }
  1968   1974     return rc;
  1969   1975   }
  1970   1976   
  1971   1977   /*
  1972   1978   ** Return the size of the common prefix (if any) shared by zPrev and
  1973   1979   ** zNext, in bytes. For example, 
................................................................................
  3439   3445       *pRC = rc;
  3440   3446       return;
  3441   3447     }
  3442   3448     sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
  3443   3449     sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
  3444   3450     sqlite3_step(pStmt);
  3445   3451     *pRC = sqlite3_reset(pStmt);
         3452  +  sqlite3_bind_null(pStmt, 2);
  3446   3453     sqlite3_free(a);
  3447   3454   }
  3448   3455   
  3449   3456   /*
  3450   3457   ** Merge the entire database so that there is one segment for each 
  3451   3458   ** iIndex/iLangid combination.
  3452   3459   */
................................................................................
  3723   3730       p->aNode = 0;
  3724   3731     }else{
  3725   3732       if( bFirst==0 ){
  3726   3733         p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
  3727   3734       }
  3728   3735       p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
  3729   3736   
         3737  +    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
         3738  +      return SQLITE_CORRUPT_VTAB;
         3739  +    }
  3730   3740       blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
  3731   3741       if( rc==SQLITE_OK ){
  3732   3742         memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
  3733   3743         p->term.n = nPrefix+nSuffix;
  3734   3744         p->iOff += nSuffix;
  3735   3745         if( p->iChild==0 ){
  3736   3746           p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
         3747  +        if( (p->nNode-p->iOff)<p->nDoclist ){
         3748  +          return SQLITE_CORRUPT_VTAB;
         3749  +        }
  3737   3750           p->aDoclist = &p->aNode[p->iOff];
  3738   3751           p->iOff += p->nDoclist;
  3739   3752         }
  3740   3753       }
  3741   3754     }
  3742   3755   
  3743   3756     assert( p->iOff<=p->nNode );
  3744         -
  3745   3757     return rc;
  3746   3758   }
  3747   3759   
  3748   3760   /*
  3749   3761   ** Release all dynamic resources held by node-reader object *p.
  3750   3762   */
  3751   3763   static void nodeReaderRelease(NodeReader *p){
................................................................................
  4627   4639       if( rc==SQLITE_OK ){
  4628   4640         sqlite3_bind_int64(pChomp, 1, iNewStart);
  4629   4641         sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC);
  4630   4642         sqlite3_bind_int64(pChomp, 3, iAbsLevel);
  4631   4643         sqlite3_bind_int(pChomp, 4, iIdx);
  4632   4644         sqlite3_step(pChomp);
  4633   4645         rc = sqlite3_reset(pChomp);
         4646  +      sqlite3_bind_null(pChomp, 2);
  4634   4647       }
  4635   4648     }
  4636   4649   
  4637   4650     sqlite3_free(root.a);
  4638   4651     sqlite3_free(block.a);
  4639   4652     return rc;
  4640   4653   }
................................................................................
  4706   4719   
  4707   4720     rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0);
  4708   4721     if( rc==SQLITE_OK ){
  4709   4722       sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
  4710   4723       sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
  4711   4724       sqlite3_step(pReplace);
  4712   4725       rc = sqlite3_reset(pReplace);
         4726  +    sqlite3_bind_null(pReplace, 2);
  4713   4727     }
  4714   4728   
  4715   4729     return rc;
  4716   4730   }
  4717   4731   
  4718   4732   /*
  4719   4733   ** Load an incr-merge hint from the database. The incr-merge hint, if one 
................................................................................
  5520   5534     sqlite3_vtab *pVtab,            /* FTS3 vtab object */
  5521   5535     int nArg,                       /* Size of argument array */
  5522   5536     sqlite3_value **apVal,          /* Array of arguments */
  5523   5537     sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
  5524   5538   ){
  5525   5539     Fts3Table *p = (Fts3Table *)pVtab;
  5526   5540     int rc = SQLITE_OK;             /* Return Code */
  5527         -  int isRemove = 0;               /* True for an UPDATE or DELETE */
  5528   5541     u32 *aSzIns = 0;                /* Sizes of inserted documents */
  5529   5542     u32 *aSzDel = 0;                /* Sizes of deleted documents */
  5530   5543     int nChng = 0;                  /* Net change in number of documents */
  5531   5544     int bInsertDone = 0;
  5532   5545   
  5533   5546     /* At this point it must be known if the %_stat table exists or not.
  5534   5547     ** So bHasStat may not be 2.  */
................................................................................
  5618   5631       goto update_out;
  5619   5632     }
  5620   5633   
  5621   5634     /* If this is a DELETE or UPDATE operation, remove the old record. */
  5622   5635     if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  5623   5636       assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
  5624   5637       rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
  5625         -    isRemove = 1;
  5626   5638     }
  5627   5639     
  5628   5640     /* If this is an INSERT or UPDATE operation, insert the new record. */
  5629   5641     if( nArg>1 && rc==SQLITE_OK ){
  5630   5642       int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
  5631   5643       if( bInsertDone==0 ){
  5632   5644         rc = fts3InsertData(p, apVal, pRowid);
  5633   5645         if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
  5634   5646           rc = FTS_CORRUPT_VTAB;
  5635   5647         }
  5636   5648       }
  5637         -    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
         5649  +    if( rc==SQLITE_OK ){
  5638   5650         rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
  5639   5651       }
  5640   5652       if( rc==SQLITE_OK ){
  5641   5653         assert( p->iPrevDocid==*pRowid );
  5642   5654         rc = fts3InsertTerms(p, iLangid, apVal, aSzIns);
  5643   5655       }
  5644   5656       if( p->bHasDocsize ){

Changes to ext/fts3/unicode/mkunicode.tcl.

   524    524       tl_print_if_entry $entry
   525    525     }
   526    526   
   527    527     puts ""
   528    528     puts "  return ret;"
   529    529     puts "\}"
   530    530   }
          531  +
          532  +proc code {txt} {
          533  +  set txt [string trimright $txt]
          534  +  set txt [string trimleft $txt "\n"]
          535  +  set n [expr {[string length $txt] - [string length [string trim $txt]]}]
          536  +  set ret ""
          537  +  foreach L [split $txt "\n"] {
          538  +    append ret "[string range $L $n end]\n"
          539  +  }
          540  +  return [uplevel "subst -nocommands {$ret}"]
          541  +}
          542  +
          543  +proc intarray {lInt} {
          544  +  set ret ""
          545  +  set n [llength $lInt]
          546  +  for {set i 0} {$i < $n} {incr i 10} {
          547  +    append ret "\n    "
          548  +    foreach int [lrange $lInt $i [expr $i+9]] {
          549  +      append ret [format "%-7s" "$int, "]
          550  +    }
          551  +  }
          552  +  append ret "\n  "
          553  +  set ret
          554  +}
          555  +
          556  +proc categories_switch {Cvar first lSecond} {
          557  +  upvar $Cvar C
          558  +  set ret ""
          559  +  append ret "case '$first':\n"
          560  +  append ret "          switch( zCat\[1\] ){\n"
          561  +  foreach s $lSecond {
          562  +    append ret "            case '$s': aArray\[$C($first$s)\] = 1; break;\n"
          563  +  }
          564  +  append ret "            case '*': \n"
          565  +  foreach s $lSecond {
          566  +    append ret "              aArray\[$C($first$s)\] = 1;\n"
          567  +  }
          568  +  append ret "              break;\n"
          569  +  append ret "            default: return 1;"
          570  +  append ret "          }\n"
          571  +  append ret "          break;\n"
          572  +}
          573  +
          574  +# Argument is a list. Each element of which is itself a list of two elements:
          575  +#
          576  +#   * the codepoint
          577  +#   * the category
          578  +#
          579  +# List elements are sorted in order of codepoint.
          580  +#
          581  +proc print_categories {lMap} {
          582  +  set categories {
          583  +    Cc Cf Cn Cs
          584  +    Ll Lm Lo Lt Lu
          585  +    Mc Me Mn
          586  +    Nd Nl No
          587  +    Pc Pd Pe Pf Pi Po Ps
          588  +    Sc Sk Sm So
          589  +    Zl Zp Zs
          590  +
          591  +    LC Co
          592  +  }
          593  +
          594  +  for {set i 0} {$i < [llength $categories]} {incr i} {
          595  +    set C([lindex $categories $i]) [expr 1+$i]
          596  +  }
          597  +
          598  +  set caseC [categories_switch C C {c f n s o}]
          599  +  set caseL [categories_switch C L {l m o t u C}]
          600  +  set caseM [categories_switch C M {c e n}]
          601  +  set caseN [categories_switch C N {d l o}]
          602  +  set caseP [categories_switch C P {c d e f i o s}]
          603  +  set caseS [categories_switch C S {c k m o}]
          604  +  set caseZ [categories_switch C Z {l p s}]
          605  +
          606  +  set nCat [expr [llength [array names C]] + 1]
          607  +  puts [code {
          608  +    int sqlite3Fts5UnicodeNCat(void) { 
          609  +      return $nCat;
          610  +    }
          611  +
          612  +    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
          613  +      aArray[0] = 1;
          614  +      switch( zCat[0] ){
          615  +        $caseC
          616  +        $caseL
          617  +        $caseM
          618  +        $caseN
          619  +        $caseP
          620  +        $caseS
          621  +        $caseZ
          622  +      }
          623  +      return 0;
          624  +    }
          625  +  }]
          626  +
          627  +  set nRepeat 0
          628  +  set first   [lindex $lMap 0 0]
          629  +  set class   [lindex $lMap 0 1]
          630  +  set prev -1
          631  +
          632  +  set CASE(0) "Lu"
          633  +  set CASE(1) "Ll"
          634  +
          635  +  foreach m $lMap {
          636  +    foreach {codepoint cl} $m {}
          637  +    set codepoint [expr "0x$codepoint"]
          638  +    if {$codepoint>=(1<<20)} continue
          639  +
          640  +    set bNew 0
          641  +    if {$codepoint!=($prev+1)} {
          642  +      set bNew 1
          643  +    } elseif {
          644  +      $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
          645  +    } {
          646  +      incr nRepeat
          647  +    } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
          648  +      set class LC
          649  +      incr nRepeat
          650  +    } else {
          651  +      set bNew 1
          652  +    }
          653  +    if {$bNew} {
          654  +      lappend lEntries [list $first $class $nRepeat]
          655  +      set nRepeat 1
          656  +      set first $codepoint
          657  +      set class $cl
          658  +    }
          659  +    set prev $codepoint
          660  +  }
          661  +  if {$nRepeat>0} {
          662  +    lappend lEntries [list $first $class $nRepeat]
          663  +  }
          664  +
          665  +  set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
          666  +  set aMap [list]
          667  +  foreach e $lEntries {
          668  +    foreach {cp class nRepeat} $e {}
          669  +    set block [expr ($cp>>16)]
          670  +    if {$block>0 && [lindex $aBlock $block]==0} {
          671  +      for {set i 1} {$i<=$block} {incr i} {
          672  +        if {[lindex $aBlock $i]==0} {
          673  +          lset aBlock $i [llength $aMap]
          674  +        }
          675  +      }
          676  +    }
          677  +    lappend aMap [expr {$cp & 0xFFFF}]
          678  +    lappend aData [expr {($nRepeat << 5) + $C($class)}]
          679  +  }
          680  +  for {set i 1} {$i<[llength $aBlock]} {incr i} {
          681  +    if {[lindex $aBlock $i]==0} {
          682  +      lset aBlock $i [llength $aMap]
          683  +    }
          684  +  }
          685  +
          686  +  set aBlockArray [intarray $aBlock]
          687  +  set aMapArray [intarray $aMap]
          688  +  set aDataArray [intarray $aData]
          689  +  puts [code {
          690  +    static u16 aFts5UnicodeBlock[] = {$aBlockArray};
          691  +    static u16 aFts5UnicodeMap[] = {$aMapArray};
          692  +    static u16 aFts5UnicodeData[] = {$aDataArray};
          693  +
          694  +    int sqlite3Fts5UnicodeCategory(int iCode) { 
          695  +      int iRes = -1;
          696  +      int iHi;
          697  +      int iLo;
          698  +      int ret;
          699  +      u16 iKey;
          700  +
          701  +      if( iCode>=(1<<20) ){
          702  +        return 0;
          703  +      }
          704  +      iLo = aFts5UnicodeBlock[(iCode>>16)];
          705  +      iHi = aFts5UnicodeBlock[1+(iCode>>16)];
          706  +      iKey = (iCode & 0xFFFF);
          707  +      while( iHi>iLo ){
          708  +        int iTest = (iHi + iLo) / 2;
          709  +        assert( iTest>=iLo && iTest<iHi );
          710  +        if( iKey>=aFts5UnicodeMap[iTest] ){
          711  +          iRes = iTest;
          712  +          iLo = iTest+1;
          713  +        }else{
          714  +          iHi = iTest;
          715  +        }
          716  +      }
          717  +
          718  +      if( iRes<0 ) return 0;
          719  +      if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
          720  +      ret = aFts5UnicodeData[iRes] & 0x1F;
          721  +      if( ret!=$C(LC) ) return ret;
          722  +      return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
          723  +    }
          724  +
          725  +    void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
          726  +      int i = 0;
          727  +      int iTbl = 0;
          728  +      while( i<128 ){
          729  +        int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
          730  +        int n = (aFts5UnicodeData[iTbl] >> 5) + i;
          731  +        for(; i<128 && i<n; i++){
          732  +          aAscii[i] = bToken;
          733  +        }
          734  +        iTbl++;
          735  +      }
          736  +    }
          737  +  }]
          738  +}
          739  +
          740  +proc print_test_categories {lMap} {
          741  +
          742  +  set lCP [list]
          743  +  foreach e $lMap {
          744  +    foreach {cp cat} $e {}
          745  +    if {[expr 0x$cp] < (1<<20)} {
          746  +      lappend lCP "{0x$cp, \"$cat\"}, "
          747  +    }
          748  +  }
          749  +
          750  +  set aCP "\n"
          751  +  for {set i 0} {$i < [llength $lCP]} {incr i 4} {
          752  +    append aCP "    [join [lrange $lCP $i $i+3]]\n"
          753  +  }
          754  +
          755  +
          756  +  puts [code {
          757  +    static int categories_test (int *piCode){
          758  +      struct Codepoint {
          759  +        int iCode;
          760  +        const char *zCat;
          761  +      } aCP[] = {$aCP};
          762  +      int i;
          763  +      int iCP = 0;
          764  +
          765  +      for(i=0; i<1000000; i++){
          766  +        u8 aArray[40];
          767  +        int cat = 0;
          768  +        int c = 0;
          769  +        memset(aArray, 0, sizeof(aArray));
          770  +        if( aCP[iCP].iCode==i ){
          771  +          sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
          772  +          iCP++;
          773  +        }else{
          774  +          aArray[0] = 1;
          775  +        }
          776  +
          777  +        c = sqlite3Fts5UnicodeCategory(i);
          778  +        if( aArray[c]==0 ){
          779  +          *piCode = i;
          780  +          return 1;
          781  +        }
          782  +      }
          783  +
          784  +      return 0;
          785  +    }
          786  +  }]
          787  +}
   531    788   
   532    789   proc print_fold_test {zFunc mappings} {
   533    790     global tl_lookup_table
   534    791   
   535    792     foreach m $mappings {
   536    793       set c [lindex $m 1]
   537    794       if {$c == ""} {
................................................................................
   601    858   }
   602    859   
   603    860   proc print_test_main {} {
   604    861     puts ""
   605    862     puts "#include <stdio.h>"
   606    863     puts ""
   607    864     puts "int main(int argc, char **argv)\{"
   608         -  puts "  int r1, r2;"
          865  +  puts "  int r1, r2, r3;"
   609    866     puts "  int code;"
          867  +  puts "  r3 = 0;"
   610    868     puts "  r1 = isalnum_test(&code);"
   611    869     puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
   612    870     puts "  else printf(\"isalnum(): test passed\\n\");"
   613    871     puts "  r2 = fold_test(&code);"
   614    872     puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
   615    873     puts "  else printf(\"fold(): test passed\\n\");"
   616         -  puts "  return (r1 || r2);"
          874  +  if {$::generate_fts5_code} {
          875  +    puts "  r3 = categories_test(&code);"
          876  +    puts "  if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
          877  +    puts "  else printf(\"categories(): test passed\\n\");"
          878  +  }
          879  +  puts "  return (r1 || r2 || r3);"
   617    880     puts "\}"
   618    881   }
   619    882   
   620    883   # Proces the command line arguments. Exit early if they are not to
   621    884   # our liking.
   622    885   #
   623    886   proc usage {} {
................................................................................
   646    909       default {
   647    910         usage
   648    911       }
   649    912     }
   650    913   }
   651    914   
   652    915   print_fileheader
          916  +
          917  +if {$::generate_test_code} {
          918  +  puts "typedef unsigned short int u16;"
          919  +  puts "typedef unsigned char u8;"
          920  +  puts "#include <string.h>"
          921  +}
   653    922   
   654    923   # Print the isalnum() function to stdout.
   655    924   #
   656    925   set lRange [an_load_separator_ranges]
   657         -print_isalnum ${function_prefix}UnicodeIsalnum $lRange
          926  +if {$generate_fts5_code==0} {
          927  +  print_isalnum ${function_prefix}UnicodeIsalnum $lRange
          928  +}
   658    929   
   659    930   # Leave a gap between the two generated C functions.
   660    931   #
   661    932   puts ""
   662    933   puts ""
   663    934   
   664    935   # Load the fold data. This is used by the [rd_XXX] commands
................................................................................
   672    943   print_isdiacritic ${function_prefix}UnicodeIsdiacritic $mappings
   673    944   puts ""
   674    945   puts ""
   675    946   
   676    947   # Print the fold() function to stdout.
   677    948   #
   678    949   print_fold ${function_prefix}UnicodeFold
          950  +
          951  +if {$generate_fts5_code} {
          952  +  puts ""
          953  +  puts ""
          954  +  print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
          955  +}
   679    956   
   680    957   # Print the test routines and main() function to stdout, if -test 
   681    958   # was specified.
   682    959   #
   683    960   if {$::generate_test_code} {
   684         -  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
          961  +  if {$generate_fts5_code==0} {
          962  +    print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
          963  +  }
   685    964     print_fold_test ${function_prefix}UnicodeFold $mappings
          965  +  print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
   686    966     print_test_main 
   687    967   }
   688    968   
   689    969   if {$generate_fts5_code} {
   690    970     # no-op
   691    971   } else {
   692    972     puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
   693    973     puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
   694    974   }

Changes to ext/fts3/unicode/parseunicode.tcl.

   138    138       foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
   139    139       set b [string trim $b]
   140    140       set d [string trim $d]
   141    141   
   142    142       if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
   143    143     }
   144    144   }
          145  +
          146  +proc cc_load_unicodedata_text {zName} {
          147  +  set fd [open $zName]
          148  +  set lField {
          149  +    code
          150  +    character_name
          151  +    general_category
          152  +    canonical_combining_classes
          153  +    bidirectional_category
          154  +    character_decomposition_mapping
          155  +    decimal_digit_value
          156  +    digit_value
          157  +    numeric_value
          158  +    mirrored
          159  +    unicode_1_name
          160  +    iso10646_comment_field
          161  +    uppercase_mapping
          162  +    lowercase_mapping
          163  +    titlecase_mapping
          164  +  }
          165  +  set lRet [list]
          166  +
          167  +  while { ![eof $fd] } {
          168  +    set line [gets $fd]
          169  +    if {$line == ""} continue
          170  +
          171  +    set fields [split $line ";"]
          172  +    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
          173  +    foreach $lField $fields {}
          174  +
          175  +    lappend lRet [list $code $general_category]
          176  +  }
          177  +
          178  +  close $fd
          179  +  set lRet
          180  +}
   145    181   
   146    182   

Changes to ext/fts5/fts5.h.

   440    440   **            document such as "I won first place" is tokenized, entries are
   441    441   **            added to the FTS index for "i", "won", "first", "1st" and
   442    442   **            "place".
   443    443   **
   444    444   **            This way, even if the tokenizer does not provide synonyms
   445    445   **            when tokenizing query text (it should not - to do would be
   446    446   **            inefficient), it doesn't matter if the user queries for 
   447         -**            'first + place' or '1st + place', as there are entires in the
          447  +**            'first + place' or '1st + place', as there are entries in the
   448    448   **            FTS index corresponding to both forms of the first token.
   449    449   **   </ol>
   450    450   **
   451    451   **   Whether it is parsing document or query text, any call to xToken that
   452    452   **   specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
   453    453   **   is considered to supply a synonym for the previous token. For example,
   454    454   **   when parsing the document "I won first place", a tokenizer that supports
................................................................................
   468    468   **   There is no limit to the number of synonyms that may be provided for a
   469    469   **   single token.
   470    470   **
   471    471   **   In many cases, method (1) above is the best approach. It does not add 
   472    472   **   extra data to the FTS index or require FTS5 to query for multiple terms,
   473    473   **   so it is efficient in terms of disk space and query speed. However, it
   474    474   **   does not support prefix queries very well. If, as suggested above, the
   475         -**   token "first" is subsituted for "1st" by the tokenizer, then the query:
          475  +**   token "first" is substituted for "1st" by the tokenizer, then the query:
   476    476   **
   477    477   **   <codeblock>
   478    478   **     ... MATCH '1s*'</codeblock>
   479    479   **
   480    480   **   will not match documents that contain the token "1st" (as the tokenizer
   481    481   **   will probably not map "1s" to any prefix of "first").
   482    482   **

Changes to ext/fts5/fts5Int.h.

   718    718   Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   719    719     Fts5Parse *pParse, 
   720    720     Fts5ExprPhrase *pPhrase, 
   721    721     Fts5Token *pToken,
   722    722     int bPrefix
   723    723   );
   724    724   
          725  +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
          726  +
   725    727   Fts5ExprNearset *sqlite3Fts5ParseNearset(
   726    728     Fts5Parse*, 
   727    729     Fts5ExprNearset*,
   728    730     Fts5ExprPhrase* 
   729    731   );
   730    732   
   731    733   Fts5Colset *sqlite3Fts5ParseColset(
................................................................................
   778    780   ** End of interface to code in fts5_vocab.c.
   779    781   **************************************************************************/
   780    782   
   781    783   
   782    784   /**************************************************************************
   783    785   ** Interface to automatically generated code in fts5_unicode2.c. 
   784    786   */
   785         -int sqlite3Fts5UnicodeIsalnum(int c);
   786    787   int sqlite3Fts5UnicodeIsdiacritic(int c);
   787    788   int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
          789  +
          790  +int sqlite3Fts5UnicodeCatParse(const char*, u8*);
          791  +int sqlite3Fts5UnicodeCategory(int iCode);
          792  +void sqlite3Fts5UnicodeAscii(u8*, u8*);
   788    793   /*
   789    794   ** End of interface to code in fts5_unicode2.c.
   790    795   **************************************************************************/
   791    796   
   792    797   #endif

Changes to ext/fts5/fts5_aux.c.

   353    353       if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
   354    354       if( iAdj<0 ) iAdj = 0;
   355    355       *piPos = iAdj;
   356    356     }
   357    357   
   358    358     return rc;
   359    359   }
          360  +
          361  +/*
          362  +** Return the value in pVal interpreted as utf-8 text. Except, if pVal 
          363  +** contains a NULL value, return a pointer to a static string zero
          364  +** bytes in length instead of a NULL pointer.
          365  +*/
          366  +static const char *fts5ValueToText(sqlite3_value *pVal){
          367  +  const char *zRet = (const char*)sqlite3_value_text(pVal);
          368  +  return zRet ? zRet : "";
          369  +}
   360    370   
   361    371   /*
   362    372   ** Implementation of snippet() function.
   363    373   */
   364    374   static void fts5SnippetFunction(
   365    375     const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   366    376     Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................
   389    399       sqlite3_result_error(pCtx, zErr, -1);
   390    400       return;
   391    401     }
   392    402   
   393    403     nCol = pApi->xColumnCount(pFts);
   394    404     memset(&ctx, 0, sizeof(HighlightContext));
   395    405     iCol = sqlite3_value_int(apVal[0]);
   396         -  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   397         -  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   398         -  zEllips = (const char*)sqlite3_value_text(apVal[3]);
          406  +  ctx.zOpen = fts5ValueToText(apVal[1]);
          407  +  ctx.zClose = fts5ValueToText(apVal[2]);
          408  +  zEllips = fts5ValueToText(apVal[3]);
   399    409     nToken = sqlite3_value_int(apVal[4]);
   400    410   
   401    411     iBestCol = (iCol>=0 ? iCol : 0);
   402    412     nPhrase = pApi->xPhraseCount(pFts);
   403    413     aSeen = sqlite3_malloc(nPhrase);
   404    414     if( aSeen==0 ){
   405    415       rc = SQLITE_NOMEM;

Changes to ext/fts5/fts5_expr.c.

    32     32   void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64));
    33     33   void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*));
    34     34   void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
    35     35   #ifndef NDEBUG
    36     36   #include <stdio.h>
    37     37   void sqlite3Fts5ParserTrace(FILE*, char*);
    38     38   #endif
           39  +int sqlite3Fts5ParserFallback(int);
    39     40   
    40     41   
    41     42   struct Fts5Expr {
    42     43     Fts5Index *pIndex;
    43     44     Fts5Config *pConfig;
    44     45     Fts5ExprNode *pRoot;
    45     46     int bDesc;                      /* Iterate in descending rowid order */
................................................................................
    83     84   #define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d))
    84     85   
    85     86   /*
    86     87   ** An instance of the following structure represents a single search term
    87     88   ** or term prefix.
    88     89   */
    89     90   struct Fts5ExprTerm {
    90         -  int bPrefix;                    /* True for a prefix term */
           91  +  u8 bPrefix;                     /* True for a prefix term */
           92  +  u8 bFirst;                      /* True if token must be first in column */
    91     93     char *zTerm;                    /* nul-terminated term */
    92     94     Fts5IndexIter *pIter;           /* Iterator for this term */
    93     95     Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
    94     96   };
    95     97   
    96     98   /*
    97     99   ** A phrase. One or more terms that must appear in a contiguous sequence
................................................................................
   164    166       case '{':  tok = FTS5_LCP;   break;
   165    167       case '}':  tok = FTS5_RCP;   break;
   166    168       case ':':  tok = FTS5_COLON; break;
   167    169       case ',':  tok = FTS5_COMMA; break;
   168    170       case '+':  tok = FTS5_PLUS;  break;
   169    171       case '*':  tok = FTS5_STAR;  break;
   170    172       case '-':  tok = FTS5_MINUS; break;
          173  +    case '^':  tok = FTS5_CARET; break;
   171    174       case '\0': tok = FTS5_EOF;   break;
   172    175   
   173    176       case '"': {
   174    177         const char *z2;
   175    178         tok = FTS5_STRING;
   176    179   
   177    180         for(z2=&z[1]; 1; z2++){
................................................................................
   423    426     int *pbMatch                    /* OUT: Set to true if really a match */
   424    427   ){
   425    428     Fts5PoslistWriter writer = {0};
   426    429     Fts5PoslistReader aStatic[4];
   427    430     Fts5PoslistReader *aIter = aStatic;
   428    431     int i;
   429    432     int rc = SQLITE_OK;
          433  +  int bFirst = pPhrase->aTerm[0].bFirst;
   430    434     
   431    435     fts5BufferZero(&pPhrase->poslist);
   432    436   
   433    437     /* If the aStatic[] array is not large enough, allocate a large array
   434    438     ** using sqlite3_malloc(). This approach could be improved upon. */
   435    439     if( pPhrase->nTerm>ArraySize(aStatic) ){
   436    440       int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
................................................................................
   477    481             }
   478    482             if( pPos->iPos>iAdj ) iPos = pPos->iPos-i;
   479    483           }
   480    484         }
   481    485       }while( bMatch==0 );
   482    486   
   483    487       /* Append position iPos to the output */
   484         -    rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
   485         -    if( rc!=SQLITE_OK ) goto ismatch_out;
          488  +    if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){
          489  +      rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
          490  +      if( rc!=SQLITE_OK ) goto ismatch_out;
          491  +    }
   486    492   
   487    493       for(i=0; i<pPhrase->nTerm; i++){
   488    494         if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
   489    495       }
   490    496     }
   491    497   
   492    498    ismatch_out:
................................................................................
   732    738       int i;
   733    739   
   734    740       /* Check that each phrase in the nearset matches the current row.
   735    741       ** Populate the pPhrase->poslist buffers at the same time. If any
   736    742       ** phrase is not a match, break out of the loop early.  */
   737    743       for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   738    744         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   739         -      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
          745  +      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym 
          746  +       || pNear->pColset || pPhrase->aTerm[0].bFirst
          747  +      ){
   740    748           int bMatch = 0;
   741    749           rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
   742    750           if( bMatch==0 ) break;
   743    751         }else{
   744    752           Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
   745    753           fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData);
   746    754         }
................................................................................
   913    921     int bMatch;                     /* True if all terms are at the same rowid */
   914    922     const int bDesc = pExpr->bDesc;
   915    923   
   916    924     /* Check that this node should not be FTS5_TERM */
   917    925     assert( pNear->nPhrase>1 
   918    926          || pNear->apPhrase[0]->nTerm>1 
   919    927          || pNear->apPhrase[0]->aTerm[0].pSynonym
          928  +       || pNear->apPhrase[0]->aTerm[0].bFirst
   920    929     );
   921    930   
   922    931     /* Initialize iLast, the "lastest" rowid any iterator points to. If the
   923    932     ** iterator skips through rowids in the default ascending order, this means
   924    933     ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
   925    934     ** means the minimum rowid.  */
   926    935     if( pLeft->aTerm[0].pSynonym ){
................................................................................
  1436   1445           sqlite3_free(pSyn);
  1437   1446         }
  1438   1447       }
  1439   1448       if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
  1440   1449       sqlite3_free(pPhrase);
  1441   1450     }
  1442   1451   }
         1452  +
         1453  +/*
         1454  +** Set the "bFirst" flag on the first token of the phrase passed as the
         1455  +** only argument.
         1456  +*/
         1457  +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
         1458  +  if( pPhrase && pPhrase->nTerm ){
         1459  +    pPhrase->aTerm[0].bFirst = 1;
         1460  +  }
         1461  +}
  1443   1462   
  1444   1463   /*
  1445   1464   ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
  1446   1465   ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
  1447   1466   ** appended to it and the results returned.
  1448   1467   **
  1449   1468   ** If an OOM error occurs, both the pNear and pPhrase objects are freed and
................................................................................
  1654   1673       }
  1655   1674   
  1656   1675       if( sCtx.pPhrase==0 ){
  1657   1676         /* This happens when parsing a token or quoted phrase that contains
  1658   1677         ** no token characters at all. (e.g ... MATCH '""'). */
  1659   1678         sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
  1660   1679       }else if( sCtx.pPhrase->nTerm ){
  1661         -      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
         1680  +      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
  1662   1681       }
  1663   1682       pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  1664   1683     }
  1665   1684   
  1666   1685     return sCtx.pPhrase;
  1667   1686   }
  1668   1687   
................................................................................
  1715   1734           const char *zTerm = p->zTerm;
  1716   1735           rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
  1717   1736               0, 0);
  1718   1737           tflags = FTS5_TOKEN_COLOCATED;
  1719   1738         }
  1720   1739         if( rc==SQLITE_OK ){
  1721   1740           sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
         1741  +        sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
  1722   1742         }
  1723   1743       }
  1724   1744     }else{
  1725   1745       /* This happens when parsing a token or quoted phrase that contains
  1726   1746       ** no token characters at all. (e.g ... MATCH '""'). */
  1727   1747       sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
  1728   1748     }
................................................................................
  1733   1753       pNew->pConfig = pExpr->pConfig;
  1734   1754       pNew->nPhrase = 1;
  1735   1755       pNew->apExprPhrase[0] = sCtx.pPhrase;
  1736   1756       pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
  1737   1757       pNew->pRoot->pNear->nPhrase = 1;
  1738   1758       sCtx.pPhrase->pNode = pNew->pRoot;
  1739   1759   
  1740         -    if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
         1760  +    if( pOrig->nTerm==1 
         1761  +     && pOrig->aTerm[0].pSynonym==0 
         1762  +     && pOrig->aTerm[0].bFirst==0 
         1763  +    ){
  1741   1764         pNew->pRoot->eType = FTS5_TERM;
  1742   1765         pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
  1743   1766       }else{
  1744   1767         pNew->pRoot->eType = FTS5_STRING;
  1745   1768         pNew->pRoot->xNext = fts5ExprNodeNext_STRING;
  1746   1769       }
  1747   1770     }else{
................................................................................
  2007   2030   
  2008   2031   static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  2009   2032     switch( pNode->eType ){
  2010   2033       case FTS5_STRING: {
  2011   2034         Fts5ExprNearset *pNear = pNode->pNear;
  2012   2035         if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
  2013   2036          && pNear->apPhrase[0]->aTerm[0].pSynonym==0
         2037  +       && pNear->apPhrase[0]->aTerm[0].bFirst==0
  2014   2038         ){
  2015   2039           pNode->eType = FTS5_TERM;
  2016   2040           pNode->xNext = fts5ExprNodeNext_TERM;
  2017   2041         }else{
  2018   2042           pNode->xNext = fts5ExprNodeNext_STRING;
  2019   2043         }
  2020   2044         break;
................................................................................
  2093   2117             pNear->apPhrase[iPhrase]->pNode = pRet;
  2094   2118             if( pNear->apPhrase[iPhrase]->nTerm==0 ){
  2095   2119               pRet->xNext = 0;
  2096   2120               pRet->eType = FTS5_EOF;
  2097   2121             }
  2098   2122           }
  2099   2123   
  2100         -        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
  2101         -         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
  2102         -        ){
  2103         -          assert( pParse->rc==SQLITE_OK );
  2104         -          pParse->rc = SQLITE_ERROR;
  2105         -          assert( pParse->zErr==0 );
  2106         -          pParse->zErr = sqlite3_mprintf(
  2107         -              "fts5: %s queries are not supported (detail!=full)", 
  2108         -              pNear->nPhrase==1 ? "phrase": "NEAR"
  2109         -          );
  2110         -          sqlite3_free(pRet);
  2111         -          pRet = 0;
         2124  +        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
         2125  +          Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
         2126  +          if( pNear->nPhrase!=1 
         2127  +           || pPhrase->nTerm>1
         2128  +           || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
         2129  +          ){
         2130  +            assert( pParse->rc==SQLITE_OK );
         2131  +            pParse->rc = SQLITE_ERROR;
         2132  +            assert( pParse->zErr==0 );
         2133  +            pParse->zErr = sqlite3_mprintf(
         2134  +                "fts5: %s queries are not supported (detail!=full)", 
         2135  +                pNear->nPhrase==1 ? "phrase": "NEAR"
         2136  +                );
         2137  +            sqlite3_free(pRet);
         2138  +            pRet = 0;
         2139  +          }
  2112   2140           }
  2113         -
  2114   2141         }else{
  2115   2142           fts5ExprAddChildren(pRet, pLeft);
  2116   2143           fts5ExprAddChildren(pRet, pRight);
  2117   2144         }
  2118   2145       }
  2119   2146     }
  2120   2147   
................................................................................
  2510   2537   */
  2511   2538   static void fts5ExprIsAlnum(
  2512   2539     sqlite3_context *pCtx,          /* Function call context */
  2513   2540     int nArg,                       /* Number of args */
  2514   2541     sqlite3_value **apVal           /* Function arguments */
  2515   2542   ){
  2516   2543     int iCode;
         2544  +  u8 aArr[32];
  2517   2545     if( nArg!=1 ){
  2518   2546       sqlite3_result_error(pCtx, 
  2519   2547           "wrong number of arguments to function fts5_isalnum", -1
  2520   2548       );
  2521   2549       return;
  2522   2550     }
         2551  +  memset(aArr, 0, sizeof(aArr));
         2552  +  sqlite3Fts5UnicodeCatParse("L*", aArr);
         2553  +  sqlite3Fts5UnicodeCatParse("N*", aArr);
         2554  +  sqlite3Fts5UnicodeCatParse("Co", aArr);
  2523   2555     iCode = sqlite3_value_int(apVal[0]);
  2524         -  sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode));
         2556  +  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
  2525   2557   }
  2526   2558   
  2527   2559   static void fts5ExprFold(
  2528   2560     sqlite3_context *pCtx,          /* Function call context */
  2529   2561     int nArg,                       /* Number of args */
  2530   2562     sqlite3_value **apVal           /* Function arguments */
  2531   2563   ){
................................................................................
  2561   2593     void *pCtx = (void*)pGlobal;
  2562   2594   
  2563   2595     for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){
  2564   2596       struct Fts5ExprFunc *p = &aFunc[i];
  2565   2597       rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
  2566   2598     }
  2567   2599   
  2568         -  /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
         2600  +  /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
         2601  +  ** sqlite3Fts5ParserFallback() are unused */
  2569   2602   #ifndef NDEBUG
  2570   2603     (void)sqlite3Fts5ParserTrace;
  2571   2604   #endif
         2605  +  (void)sqlite3Fts5ParserFallback;
  2572   2606   
  2573   2607     return rc;
  2574   2608   }
  2575   2609   
  2576   2610   /*
  2577   2611   ** Return the number of phrases in expression pExpr.
  2578   2612   */

Changes to ext/fts5/fts5_index.c.

   754    754       if( p->rc ) return;
   755    755     }
   756    756   
   757    757     sqlite3_bind_int64(p->pWriter, 1, iRowid);
   758    758     sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC);
   759    759     sqlite3_step(p->pWriter);
   760    760     p->rc = sqlite3_reset(p->pWriter);
          761  +  sqlite3_bind_null(p->pWriter, 2);
   761    762   }
   762    763   
   763    764   /*
   764    765   ** Execute the following SQL:
   765    766   **
   766    767   **     DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast
   767    768   */
................................................................................
  2382   2383     sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
  2383   2384     if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){
  2384   2385       i64 val = sqlite3_column_int(pIdxSelect, 0);
  2385   2386       iPg = (int)(val>>1);
  2386   2387       bDlidx = (val & 0x0001);
  2387   2388     }
  2388   2389     p->rc = sqlite3_reset(pIdxSelect);
         2390  +  sqlite3_bind_null(pIdxSelect, 2);
  2389   2391   
  2390   2392     if( iPg<pSeg->pgnoFirst ){
  2391   2393       iPg = pSeg->pgnoFirst;
  2392   2394       bDlidx = 0;
  2393   2395     }
  2394   2396   
  2395   2397     pIter->iLeafPgno = iPg - 1;
................................................................................
  3594   3596           sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
  3595   3597           if( p->rc==SQLITE_OK ){
  3596   3598             u8 aBlob[2] = {0xff, 0xff};
  3597   3599             sqlite3_bind_int(pIdxSelect, 1, iSegid);
  3598   3600             sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
  3599   3601             assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
  3600   3602             p->rc = sqlite3_reset(pIdxSelect);
         3603  +          sqlite3_bind_null(pIdxSelect, 2);
  3601   3604           }
  3602   3605         }
  3603   3606   #endif
  3604   3607       }
  3605   3608     }
  3606   3609   
  3607   3610     return iSegid;
................................................................................
  3720   3723       const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:"");
  3721   3724       /* The following was already done in fts5WriteInit(): */
  3722   3725       /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
  3723   3726       sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC);
  3724   3727       sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
  3725   3728       sqlite3_step(p->pIdxWriter);
  3726   3729       p->rc = sqlite3_reset(p->pIdxWriter);
         3730  +    sqlite3_bind_null(p->pIdxWriter, 2);
  3727   3731     }
  3728   3732     pWriter->iBtPage = 0;
  3729   3733   }
  3730   3734   
  3731   3735   /*
  3732   3736   ** This is called once for each leaf page except the first that contains
  3733   3737   ** at least one term. Argument (nTerm/pTerm) is the split-key - a term that
................................................................................
  4905   4909     if( p2->n ){
  4906   4910       i64 iLastRowid = 0;
  4907   4911       Fts5DoclistIter i1;
  4908   4912       Fts5DoclistIter i2;
  4909   4913       Fts5Buffer out = {0, 0, 0};
  4910   4914       Fts5Buffer tmp = {0, 0, 0};
  4911   4915   
  4912         -    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
         4916  +    /* The maximum size of the output is equal to the sum of the two 
         4917  +    ** input sizes + 1 varint (9 bytes). The extra varint is because if the
         4918  +    ** first rowid in one input is a large negative number, and the first in
         4919  +    ** the other a non-negative number, the delta for the non-negative
         4920  +    ** number will be larger on disk than the literal integer value
         4921  +    ** was.  */
         4922  +    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return;
  4913   4923       fts5DoclistIterInit(p1, &i1);
  4914   4924       fts5DoclistIterInit(p2, &i2);
  4915   4925   
  4916   4926       while( 1 ){
  4917   4927         if( i1.iRowid<i2.iRowid ){
  4918   4928           /* Copy entry from i1 */
  4919   4929           fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
................................................................................
  4999   5009         fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  5000   5010         fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
  5001   5011       }
  5002   5012       else if( i2.aPoslist ){
  5003   5013         fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  5004   5014         fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
  5005   5015       }
         5016  +    assert( out.n<=(p1->n+p2->n+9) );
  5006   5017   
  5007   5018       fts5BufferSet(&p->rc, p1, out.n, out.p);
  5008   5019       fts5BufferFree(&tmp);
  5009   5020       fts5BufferFree(&out);
  5010   5021     }
  5011   5022   }
  5012   5023   
................................................................................
  5246   5257     int nChar
  5247   5258   ){
  5248   5259     int n = 0;
  5249   5260     int i;
  5250   5261     for(i=0; i<nChar; i++){
  5251   5262       if( n>=nByte ) return 0;      /* Input contains fewer than nChar chars */
  5252   5263       if( (unsigned char)p[n++]>=0xc0 ){
  5253         -      while( (p[n] & 0xc0)==0x80 ) n++;
         5264  +      while( (p[n] & 0xc0)==0x80 ){
         5265  +        n++;
         5266  +        if( n>=nByte ) break;
         5267  +      }
  5254   5268       }
  5255   5269     }
  5256   5270     return n;
  5257   5271   }
  5258   5272   
  5259   5273   /*
  5260   5274   ** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of
................................................................................
  5384   5398   
  5385   5399       if( p->rc ){
  5386   5400         sqlite3Fts5IterClose((Fts5IndexIter*)pRet);
  5387   5401         pRet = 0;
  5388   5402         fts5CloseReader(p);
  5389   5403       }
  5390   5404   
  5391         -    *ppIter = &pRet->base;
         5405  +    *ppIter = (Fts5IndexIter*)pRet;
  5392   5406       sqlite3Fts5BufferFree(&buf);
  5393   5407     }
  5394   5408     return fts5IndexReturn(p);
  5395   5409   }
  5396   5410   
  5397   5411   /*
  5398   5412   ** Return true if the iterator passed as the only argument is at EOF.

Changes to ext/fts5/fts5_main.c.

   276    276         assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 );
   277    277         p->ts.eState = 0;
   278    278         break;
   279    279   
   280    280       case FTS5_SAVEPOINT:
   281    281         assert( p->ts.eState==1 );
   282    282         assert( iSavepoint>=0 );
   283         -      assert( iSavepoint>p->ts.iSavepoint );
          283  +      assert( iSavepoint>=p->ts.iSavepoint );
   284    284         p->ts.iSavepoint = iSavepoint;
   285    285         break;
   286    286         
   287    287       case FTS5_RELEASE:
   288    288         assert( p->ts.eState==1 );
   289    289         assert( iSavepoint>=0 );
   290    290         assert( iSavepoint<=p->ts.iSavepoint );
................................................................................
   530    530                                       FTS5_BI_ROWID_GE, 0, 0, -1},
   531    531     };
   532    532   
   533    533     int aColMap[3];
   534    534     aColMap[0] = -1;
   535    535     aColMap[1] = nCol;
   536    536     aColMap[2] = nCol+1;
          537  +
          538  +  assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH );
          539  +  assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH );
          540  +  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
          541  +  assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH );
          542  +  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
   537    543   
   538    544     /* Set idxFlags flags for all WHERE clause terms that will be used. */
   539    545     for(i=0; i<pInfo->nConstraint; i++){
   540    546       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   541    547       int iCol = p->iColumn;
   542    548   
   543    549       if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
................................................................................
   549    555           aConstraint[0].iConsIndex = i;
   550    556         }else{
   551    557           /* As there exists an unusable MATCH constraint this is an 
   552    558           ** unusable plan. Set a prohibitively high cost. */
   553    559           pInfo->estimatedCost = 1e50;
   554    560           return SQLITE_OK;
   555    561         }
   556         -    }else{
          562  +    }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){
   557    563         int j;
   558    564         for(j=1; j<ArraySize(aConstraint); j++){
   559    565           struct Constraint *pC = &aConstraint[j];
   560         -        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
          566  +        if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){
   561    567             pC->iConsIndex = i;
   562    568             idxFlags |= pC->fts5op;
   563    569           }
   564    570         }
   565    571       }
   566    572     }
   567    573   
................................................................................
  1195   1201       ** return results to the user for this query. The current cursor 
  1196   1202       ** (pCursor) is used to execute the query issued by function 
  1197   1203       ** fts5CursorFirstSorted() above.  */
  1198   1204       assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
  1199   1205       assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
  1200   1206       assert( pCsr->iLastRowid==LARGEST_INT64 );
  1201   1207       assert( pCsr->iFirstRowid==SMALLEST_INT64 );
         1208  +    if( pTab->pSortCsr->bDesc ){
         1209  +      pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
         1210  +      pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
         1211  +    }else{
         1212  +      pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
         1213  +      pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
         1214  +    }
  1202   1215       pCsr->ePlan = FTS5_PLAN_SOURCE;
  1203   1216       pCsr->pExpr = pTab->pSortCsr->pExpr;
  1204   1217       rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1205   1218     }else if( pMatch ){
  1206   1219       const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1207   1220       if( zExpr==0 ) zExpr = "";
  1208   1221   
................................................................................
  2627   2640     int nArg,                       /* Number of args */
  2628   2641     sqlite3_value **apUnused        /* Function arguments */
  2629   2642   ){
  2630   2643     assert( nArg==0 );
  2631   2644     UNUSED_PARAM2(nArg, apUnused);
  2632   2645     sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
  2633   2646   }
         2647  +
         2648  +/*
         2649  +** Return true if zName is the extension on one of the shadow tables used
         2650  +** by this module.
         2651  +*/
         2652  +static int fts5ShadowName(const char *zName){
         2653  +  static const char *azName[] = {
         2654  +    "config", "content", "data", "docsize", "idx"
         2655  +  };
         2656  +  unsigned int i;
         2657  +  for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
         2658  +    if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
         2659  +  }
         2660  +  return 0;
         2661  +}
  2634   2662   
  2635   2663   static int fts5Init(sqlite3 *db){
  2636   2664     static const sqlite3_module fts5Mod = {
  2637         -    /* iVersion      */ 2,
         2665  +    /* iVersion      */ 3,
  2638   2666       /* xCreate       */ fts5CreateMethod,
  2639   2667       /* xConnect      */ fts5ConnectMethod,
  2640   2668       /* xBestIndex    */ fts5BestIndexMethod,
  2641   2669       /* xDisconnect   */ fts5DisconnectMethod,
  2642   2670       /* xDestroy      */ fts5DestroyMethod,
  2643   2671       /* xOpen         */ fts5OpenMethod,
  2644   2672       /* xClose        */ fts5CloseMethod,
................................................................................
  2653   2681       /* xCommit       */ fts5CommitMethod,
  2654   2682       /* xRollback     */ fts5RollbackMethod,
  2655   2683       /* xFindFunction */ fts5FindFunctionMethod,
  2656   2684       /* xRename       */ fts5RenameMethod,
  2657   2685       /* xSavepoint    */ fts5SavepointMethod,
  2658   2686       /* xRelease      */ fts5ReleaseMethod,
  2659   2687       /* xRollbackTo   */ fts5RollbackToMethod,
         2688  +    /* xShadowName   */ fts5ShadowName
  2660   2689     };
  2661   2690   
  2662   2691     int rc;
  2663   2692     Fts5Global *pGlobal = 0;
  2664   2693   
  2665   2694     pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global));
  2666   2695     if( pGlobal==0 ){

Changes to ext/fts5/fts5_storage.c.

   454    454       sqlite3_stmt *pReplace = 0;
   455    455       rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
   456    456       if( rc==SQLITE_OK ){
   457    457         sqlite3_bind_int64(pReplace, 1, iRowid);
   458    458         sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
   459    459         sqlite3_step(pReplace);
   460    460         rc = sqlite3_reset(pReplace);
          461  +      sqlite3_bind_null(pReplace, 2);
   461    462       }
   462    463     }
   463    464     return rc;
   464    465   }
   465    466   
   466    467   /*
   467    468   ** Load the contents of the "averages" record from disk into the 
................................................................................
  1114   1115       if( pVal ){
  1115   1116         sqlite3_bind_value(pReplace, 2, pVal);
  1116   1117       }else{
  1117   1118         sqlite3_bind_int(pReplace, 2, iVal);
  1118   1119       }
  1119   1120       sqlite3_step(pReplace);
  1120   1121       rc = sqlite3_reset(pReplace);
         1122  +    sqlite3_bind_null(pReplace, 1);
  1121   1123     }
  1122   1124     if( rc==SQLITE_OK && pVal ){
  1123   1125       int iNew = p->pConfig->iCookie + 1;
  1124   1126       rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
  1125   1127       if( rc==SQLITE_OK ){
  1126   1128         p->pConfig->iCookie = iNew;
  1127   1129       }
  1128   1130     }
  1129   1131     return rc;
  1130   1132   }

Changes to ext/fts5/fts5_tcl.c.

   429    429         rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0);
   430    430         break;
   431    431       }
   432    432       CASE(15, "xGetAuxdataInt") {
   433    433         int iVal;
   434    434         int bClear;
   435    435         if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
   436         -      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
          436  +      iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
   437    437         Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
   438    438         break;
   439    439       }
   440    440   
   441    441       CASE(16, "xPhraseForeach") {
   442    442         int iPhrase;
   443    443         int iCol;
................................................................................
   478    478         Fts5PhraseIter iter;
   479    479   
   480    480         if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
   481    481         zColvar = Tcl_GetString(objv[3]);
   482    482   
   483    483         rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
   484    484         if( rc!=SQLITE_OK ){
   485         -        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
          485  +        Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
   486    486           return TCL_ERROR;
   487    487         }
   488    488         for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
   489    489           Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
   490    490           rc = Tcl_EvalObjEx(interp, pScript, 0);
   491    491           if( rc==TCL_CONTINUE ) rc = TCL_OK;
   492    492           if( rc!=TCL_OK ){
................................................................................
   920    920           "sqlite3_fts5_token may only be used by tokenizer callback", 0
   921    921       );
   922    922       return TCL_ERROR;
   923    923     }
   924    924   
   925    925     rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
   926    926     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
   927         -  return TCL_OK;
          927  +  return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
   928    928   
   929    929    usage:
   930    930     Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
   931    931     return TCL_ERROR;
   932    932   }
   933    933   
   934    934   static void f5tDelTokenizer(void *pCtx){

Changes to ext/fts5/fts5_test_tok.c.

   467    467        0,                           /* xSync         */
   468    468        0,                           /* xCommit       */
   469    469        0,                           /* xRollback     */
   470    470        0,                           /* xFindFunction */
   471    471        0,                           /* xRename       */
   472    472        0,                           /* xSavepoint    */
   473    473        0,                           /* xRelease      */
   474         -     0                            /* xRollbackTo   */
          474  +     0,                           /* xRollbackTo   */
          475  +     0                            /* xShadowName   */
   475    476     };
   476    477     int rc;                         /* Return code */
   477    478   
   478    479     rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi);
   479    480     return rc;
   480    481   }
   481    482   
   482    483   #endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */

Changes to ext/fts5/fts5_tokenize.c.

   233    233   struct Unicode61Tokenizer {
   234    234     unsigned char aTokenChar[128];  /* ASCII range token characters */
   235    235     char *aFold;                    /* Buffer to fold text into */
   236    236     int nFold;                      /* Size of aFold[] in bytes */
   237    237     int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   238    238     int nException;
   239    239     int *aiException;
          240  +
          241  +  unsigned char aCategory[32];    /* True for token char categories */
   240    242   };
   241    243   
   242    244   static int fts5UnicodeAddExceptions(
   243    245     Unicode61Tokenizer *p,          /* Tokenizer object */
   244    246     const char *z,                  /* Characters to treat as exceptions */
   245    247     int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
   246    248   ){
................................................................................
   257    259         while( zCsr<zTerm ){
   258    260           int iCode;
   259    261           int bToken;
   260    262           READ_UTF8(zCsr, zTerm, iCode);
   261    263           if( iCode<128 ){
   262    264             p->aTokenChar[iCode] = (unsigned char)bTokenChars;
   263    265           }else{
   264         -          bToken = sqlite3Fts5UnicodeIsalnum(iCode);
          266  +          bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
   265    267             assert( (bToken==0 || bToken==1) ); 
   266    268             assert( (bTokenChars==0 || bTokenChars==1) );
   267    269             if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
   268    270               int i;
   269    271               for(i=0; i<nNew; i++){
   270    272                 if( aNew[i]>iCode ) break;
   271    273               }
................................................................................
   317    319       Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
   318    320       sqlite3_free(p->aiException);
   319    321       sqlite3_free(p->aFold);
   320    322       sqlite3_free(p);
   321    323     }
   322    324     return;
   323    325   }
          326  +
          327  +static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
          328  +  const char *z = zCat;
          329  +
          330  +  while( *z ){
          331  +    while( *z==' ' || *z=='\t' ) z++;
          332  +    if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
          333  +      return SQLITE_ERROR;
          334  +    }
          335  +    while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
          336  +  }
          337  +
          338  +  sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
          339  +  return SQLITE_OK;
          340  +}
   324    341   
   325    342   /*
   326    343   ** Create a "unicode61" tokenizer.
   327    344   */
   328    345   static int fts5UnicodeCreate(
   329    346     void *pUnused, 
   330    347     const char **azArg, int nArg,
................................................................................
   336    353     UNUSED_PARAM(pUnused);
   337    354   
   338    355     if( nArg%2 ){
   339    356       rc = SQLITE_ERROR;
   340    357     }else{
   341    358       p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
   342    359       if( p ){
          360  +      const char *zCat = "L* N* Co";
   343    361         int i;
   344    362         memset(p, 0, sizeof(Unicode61Tokenizer));
   345         -      memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
          363  +
   346    364         p->bRemoveDiacritic = 1;
   347    365         p->nFold = 64;
   348    366         p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
   349    367         if( p->aFold==0 ){
   350    368           rc = SQLITE_NOMEM;
   351    369         }
          370  +
          371  +      /* Search for a "categories" argument */
          372  +      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
          373  +        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
          374  +          zCat = azArg[i+1];
          375  +        }
          376  +      }
          377  +
          378  +      if( rc==SQLITE_OK ){
          379  +        rc = unicodeSetCategories(p, zCat);
          380  +      }
          381  +
   352    382         for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
   353    383           const char *zArg = azArg[i+1];
   354    384           if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
   355    385             if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
   356    386               rc = SQLITE_ERROR;
   357    387             }
   358    388             p->bRemoveDiacritic = (zArg[0]=='1');
   359    389           }else
   360    390           if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
   361    391             rc = fts5UnicodeAddExceptions(p, zArg, 1);
   362    392           }else
   363    393           if( 0==sqlite3_stricmp(azArg[i], "separators") ){
   364    394             rc = fts5UnicodeAddExceptions(p, zArg, 0);
          395  +        }else
          396  +        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
          397  +          /* no-op */
   365    398           }else{
   366    399             rc = SQLITE_ERROR;
   367    400           }
   368    401         }
          402  +
   369    403       }else{
   370    404         rc = SQLITE_NOMEM;
   371    405       }
   372    406       if( rc!=SQLITE_OK ){
   373    407         fts5UnicodeDelete((Fts5Tokenizer*)p);
   374    408         p = 0;
   375    409       }
................................................................................
   380    414   
   381    415   /*
   382    416   ** Return true if, for the purposes of tokenizing with the tokenizer
   383    417   ** passed as the first argument, codepoint iCode is considered a token 
   384    418   ** character (not a separator).
   385    419   */
   386    420   static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
   387         -  assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
   388         -  return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
          421  +  return (
          422  +    p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
          423  +    ^ fts5UnicodeIsException(p, iCode)
          424  +  );
   389    425   }
   390    426   
   391    427   static int fts5UnicodeTokenize(
   392    428     Fts5Tokenizer *pTokenizer,
   393    429     void *pCtx,
   394    430     int iUnused,
   395    431     const char *pText, int nText,

Changes to ext/fts5/fts5_unicode2.c.

    14     14   /*
    15     15   ** DO NOT EDIT THIS MACHINE GENERATED FILE.
    16     16   */
    17     17   
    18     18   
    19     19   #include <assert.h>
    20     20   
    21         -/*
    22         -** Return true if the argument corresponds to a unicode codepoint
    23         -** classified as either a letter or a number. Otherwise false.
    24         -**
    25         -** The results are undefined if the value passed to this function
    26         -** is less than zero.
    27         -*/
    28         -int sqlite3Fts5UnicodeIsalnum(int c){
    29         -  /* Each unsigned integer in the following array corresponds to a contiguous
    30         -  ** range of unicode codepoints that are not either letters or numbers (i.e.
    31         -  ** codepoints for which this function should return 0).
    32         -  **
    33         -  ** The most significant 22 bits in each 32-bit value contain the first 
    34         -  ** codepoint in the range. The least significant 10 bits are used to store
    35         -  ** the size of the range (always at least 1). In other words, the value 
    36         -  ** ((C<<22) + N) represents a range of N codepoints starting with codepoint 
    37         -  ** C. It is not possible to represent a range larger than 1023 codepoints 
    38         -  ** using this format.
    39         -  */
    40         -  static const unsigned int aEntry[] = {
    41         -    0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
    42         -    0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
    43         -    0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
    44         -    0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
    45         -    0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
    46         -    0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
    47         -    0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
    48         -    0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
    49         -    0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
    50         -    0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
    51         -    0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
    52         -    0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
    53         -    0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
    54         -    0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
    55         -    0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
    56         -    0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
    57         -    0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
    58         -    0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
    59         -    0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
    60         -    0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
    61         -    0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
    62         -    0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
    63         -    0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
    64         -    0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
    65         -    0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
    66         -    0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
    67         -    0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
    68         -    0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
    69         -    0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
    70         -    0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
    71         -    0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
    72         -    0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
    73         -    0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
    74         -    0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
    75         -    0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
    76         -    0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
    77         -    0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
    78         -    0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
    79         -    0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
    80         -    0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
    81         -    0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
    82         -    0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
    83         -    0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
    84         -    0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
    85         -    0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
    86         -    0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
    87         -    0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
    88         -    0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
    89         -    0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
    90         -    0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
    91         -    0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
    92         -    0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
    93         -    0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
    94         -    0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
    95         -    0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
    96         -    0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
    97         -    0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
    98         -    0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
    99         -    0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
   100         -    0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
   101         -    0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
   102         -    0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
   103         -    0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
   104         -    0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
   105         -    0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
   106         -    0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
   107         -    0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
   108         -    0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
   109         -    0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
   110         -    0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
   111         -    0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
   112         -    0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
   113         -    0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
   114         -    0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
   115         -    0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
   116         -    0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
   117         -    0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
   118         -    0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
   119         -    0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
   120         -    0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
   121         -    0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
   122         -    0x380400F0,
   123         -  };
   124         -  static const unsigned int aAscii[4] = {
   125         -    0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
   126         -  };
   127         -
   128         -  if( (unsigned int)c<128 ){
   129         -    return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
   130         -  }else if( (unsigned int)c<(1<<22) ){
   131         -    unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
   132         -    int iRes = 0;
   133         -    int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
   134         -    int iLo = 0;
   135         -    while( iHi>=iLo ){
   136         -      int iTest = (iHi + iLo) / 2;
   137         -      if( key >= aEntry[iTest] ){
   138         -        iRes = iTest;
   139         -        iLo = iTest+1;
   140         -      }else{
   141         -        iHi = iTest-1;
   142         -      }
   143         -    }
   144         -    assert( aEntry[0]<key );
   145         -    assert( key>=aEntry[iRes] );
   146         -    return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
   147         -  }
   148         -  return 1;
   149         -}
   150     21   
   151     22   
   152     23   /*
   153     24   ** If the argument is a codepoint corresponding to a lowercase letter
   154     25   ** in the ASCII range with a diacritic added, return the codepoint
   155     26   ** of the ASCII letter only. For example, if passed 235 - "LATIN
   156     27   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
................................................................................
   354    225     
   355    226     else if( c>=66560 && c<66600 ){
   356    227       ret = c + 40;
   357    228     }
   358    229   
   359    230     return ret;
   360    231   }
          232  +
          233  +
          234  +#if 0
          235  +int sqlite3Fts5UnicodeNCat(void) { 
          236  +  return 32;
          237  +}
          238  +#endif
          239  +
          240  +int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
          241  +  aArray[0] = 1;
          242  +  switch( zCat[0] ){
          243  +    case 'C':
          244  +          switch( zCat[1] ){
          245  +            case 'c': aArray[1] = 1; break;
          246  +            case 'f': aArray[2] = 1; break;
          247  +            case 'n': aArray[3] = 1; break;
          248  +            case 's': aArray[4] = 1; break;
          249  +            case 'o': aArray[31] = 1; break;
          250  +            case '*': 
          251  +              aArray[1] = 1;
          252  +              aArray[2] = 1;
          253  +              aArray[3] = 1;
          254  +              aArray[4] = 1;
          255  +              aArray[31] = 1;
          256  +              break;
          257  +            default: return 1;          }
          258  +          break;
          259  +
          260  +    case 'L':
          261  +          switch( zCat[1] ){
          262  +            case 'l': aArray[5] = 1; break;
          263  +            case 'm': aArray[6] = 1; break;
          264  +            case 'o': aArray[7] = 1; break;
          265  +            case 't': aArray[8] = 1; break;
          266  +            case 'u': aArray[9] = 1; break;
          267  +            case 'C': aArray[30] = 1; break;
          268  +            case '*': 
          269  +              aArray[5] = 1;
          270  +              aArray[6] = 1;
          271  +              aArray[7] = 1;
          272  +              aArray[8] = 1;
          273  +              aArray[9] = 1;
          274  +              aArray[30] = 1;
          275  +              break;
          276  +            default: return 1;          }
          277  +          break;
          278  +
          279  +    case 'M':
          280  +          switch( zCat[1] ){
          281  +            case 'c': aArray[10] = 1; break;
          282  +            case 'e': aArray[11] = 1; break;
          283  +            case 'n': aArray[12] = 1; break;
          284  +            case '*': 
          285  +              aArray[10] = 1;
          286  +              aArray[11] = 1;
          287  +              aArray[12] = 1;
          288  +              break;
          289  +            default: return 1;          }
          290  +          break;
          291  +
          292  +    case 'N':
          293  +          switch( zCat[1] ){
          294  +            case 'd': aArray[13] = 1; break;
          295  +            case 'l': aArray[14] = 1; break;
          296  +            case 'o': aArray[15] = 1; break;
          297  +            case '*': 
          298  +              aArray[13] = 1;
          299  +              aArray[14] = 1;
          300  +              aArray[15] = 1;
          301  +              break;
          302  +            default: return 1;          }
          303  +          break;
          304  +
          305  +    case 'P':
          306  +          switch( zCat[1] ){
          307  +            case 'c': aArray[16] = 1; break;
          308  +            case 'd': aArray[17] = 1; break;
          309  +            case 'e': aArray[18] = 1; break;
          310  +            case 'f': aArray[19] = 1; break;
          311  +            case 'i': aArray[20] = 1; break;
          312  +            case 'o': aArray[21] = 1; break;
          313  +            case 's': aArray[22] = 1; break;
          314  +            case '*': 
          315  +              aArray[16] = 1;
          316  +              aArray[17] = 1;
          317  +              aArray[18] = 1;
          318  +              aArray[19] = 1;
          319  +              aArray[20] = 1;
          320  +              aArray[21] = 1;
          321  +              aArray[22] = 1;
          322  +              break;
          323  +            default: return 1;          }
          324  +          break;
          325  +
          326  +    case 'S':
          327  +          switch( zCat[1] ){
          328  +            case 'c': aArray[23] = 1; break;
          329  +            case 'k': aArray[24] = 1; break;
          330  +            case 'm': aArray[25] = 1; break;
          331  +            case 'o': aArray[26] = 1; break;
          332  +            case '*': 
          333  +              aArray[23] = 1;
          334  +              aArray[24] = 1;
          335  +              aArray[25] = 1;
          336  +              aArray[26] = 1;
          337  +              break;
          338  +            default: return 1;          }
          339  +          break;
          340  +
          341  +    case 'Z':
          342  +          switch( zCat[1] ){
          343  +            case 'l': aArray[27] = 1; break;
          344  +            case 'p': aArray[28] = 1; break;
          345  +            case 's': aArray[29] = 1; break;
          346  +            case '*': 
          347  +              aArray[27] = 1;
          348  +              aArray[28] = 1;
          349  +              aArray[29] = 1;
          350  +              break;
          351  +            default: return 1;          }
          352  +          break;
          353  +
          354  +  }
          355  +  return 0;
          356  +}
          357  +
          358  +static u16 aFts5UnicodeBlock[] = {
          359  +    0,     1471,  1753,  1760,  1760,  1760,  1760,  1760,  1760,  1760,  
          360  +    1760,  1760,  1760,  1760,  1760,  1763,  1765,  
          361  +  };
          362  +static u16 aFts5UnicodeMap[] = {
          363  +    0,     32,    33,    36,    37,    40,    41,    42,    43,    44,    
          364  +    45,    46,    48,    58,    60,    63,    65,    91,    92,    93,    
          365  +    94,    95,    96,    97,    123,   124,   125,   126,   127,   160,   
          366  +    161,   162,   166,   167,   168,   169,   170,   171,   172,   173,   
          367  +    174,   175,   176,   177,   178,   180,   181,   182,   184,   185,   
          368  +    186,   187,   188,   191,   192,   215,   216,   223,   247,   248,   
          369  +    256,   312,   313,   329,   330,   377,   383,   385,   387,   388,   
          370  +    391,   394,   396,   398,   402,   403,   405,   406,   409,   412,   
          371  +    414,   415,   417,   418,   423,   427,   428,   431,   434,   436,   
          372  +    437,   440,   442,   443,   444,   446,   448,   452,   453,   454,   
          373  +    455,   456,   457,   458,   459,   460,   461,   477,   478,   496,   
          374  +    497,   498,   499,   500,   503,   505,   506,   564,   570,   572,   
          375  +    573,   575,   577,   580,   583,   584,   592,   660,   661,   688,   
          376  +    706,   710,   722,   736,   741,   748,   749,   750,   751,   768,   
          377  +    880,   884,   885,   886,   890,   891,   894,   900,   902,   903,   
          378  +    904,   908,   910,   912,   913,   931,   940,   975,   977,   978,   
          379  +    981,   984,   1008,  1012,  1014,  1015,  1018,  1020,  1021,  1072,  
          380  +    1120,  1154,  1155,  1160,  1162,  1217,  1231,  1232,  1329,  1369,  
          381  +    1370,  1377,  1417,  1418,  1423,  1425,  1470,  1471,  1472,  1473,  
          382  +    1475,  1476,  1478,  1479,  1488,  1520,  1523,  1536,  1542,  1545,  
          383  +    1547,  1548,  1550,  1552,  1563,  1566,  1568,  1600,  1601,  1611,  
          384  +    1632,  1642,  1646,  1648,  1649,  1748,  1749,  1750,  1757,  1758,  
          385  +    1759,  1765,  1767,  1769,  1770,  1774,  1776,  1786,  1789,  1791,  
          386  +    1792,  1807,  1808,  1809,  1810,  1840,  1869,  1958,  1969,  1984,  
          387  +    1994,  2027,  2036,  2038,  2039,  2042,  2048,  2070,  2074,  2075,  
          388  +    2084,  2085,  2088,  2089,  2096,  2112,  2137,  2142,  2208,  2210,  
          389  +    2276,  2304,  2307,  2308,  2362,  2363,  2364,  2365,  2366,  2369,  
          390  +    2377,  2381,  2382,  2384,  2385,  2392,  2402,  2404,  2406,  2416,  
          391  +    2417,  2418,  2425,  2433,  2434,  2437,  2447,  2451,  2474,  2482,  
          392  +    2486,  2492,  2493,  2494,  2497,  2503,  2507,  2509,  2510,  2519,  
          393  +    2524,  2527,  2530,  2534,  2544,  2546,  2548,  2554,  2555,  2561,  
          394  +    2563,  2565,  2575,  2579,  2602,  2610,  2613,  2616,  2620,  2622,  
          395  +    2625,  2631,  2635,  2641,  2649,  2654,  2662,  2672,  2674,  2677,  
          396  +    2689,  2691,  2693,  2703,  2707,  2730,  2738,  2741,  2748,  2749,  
          397  +    2750,  2753,  2759,  2761,  2763,  2765,  2768,  2784,  2786,  2790,  
          398  +    2800,  2801,  2817,  2818,  2821,  2831,  2835,  2858,  2866,  2869,  
          399  +    2876,  2877,  2878,  2879,  2880,  2881,  2887,  2891,  2893,  2902,  
          400  +    2903,  2908,  2911,  2914,  2918,  2928,  2929,  2930,  2946,  2947,  
          401  +    2949,  2958,  2962,  2969,  2972,  2974,  2979,  2984,  2990,  3006,  
          402  +    3008,  3009,  3014,  3018,  3021,  3024,  3031,  3046,  3056,  3059,  
          403  +    3065,  3066,  3073,  3077,  3086,  3090,  3114,  3125,  3133,  3134,  
          404  +    3137,  3142,  3146,  3157,  3160,  3168,  3170,  3174,  3192,  3199,  
          405  +    3202,  3205,  3214,  3218,  3242,  3253,  3260,  3261,  3262,  3263,  
          406  +    3264,  3270,  3271,  3274,  3276,  3285,  3294,  3296,  3298,  3302,  
          407  +    3313,  3330,  3333,  3342,  3346,  3389,  3390,  3393,  3398,  3402,  
          408  +    3405,  3406,  3415,  3424,  3426,  3430,  3440,  3449,  3450,  3458,  
          409  +    3461,  3482,  3507,  3517,  3520,  3530,  3535,  3538,  3542,  3544,  
          410  +    3570,  3572,  3585,  3633,  3634,  3636,  3647,  3648,  3654,  3655,  
          411  +    3663,  3664,  3674,  3713,  3716,  3719,  3722,  3725,  3732,  3737,  
          412  +    3745,  3749,  3751,  3754,  3757,  3761,  3762,  3764,  3771,  3773,  
          413  +    3776,  3782,  3784,  3792,  3804,  3840,  3841,  3844,  3859,  3860,  
          414  +    3861,  3864,  3866,  3872,  3882,  3892,  3893,  3894,  3895,  3896,  
          415  +    3897,  3898,  3899,  3900,  3901,  3902,  3904,  3913,  3953,  3967,  
          416  +    3968,  3973,  3974,  3976,  3981,  3993,  4030,  4038,  4039,  4046,  
          417  +    4048,  4053,  4057,  4096,  4139,  4141,  4145,  4146,  4152,  4153,  
          418  +    4155,  4157,  4159,  4160,  4170,  4176,  4182,  4184,  4186,  4190,  
          419  +    4193,  4194,  4197,  4199,  4206,  4209,  4213,  4226,  4227,  4229,  
          420  +    4231,  4237,  4238,  4239,  4240,  4250,  4253,  4254,  4256,  4295,  
          421  +    4301,  4304,  4347,  4348,  4349,  4682,  4688,  4696,  4698,  4704,  
          422  +    4746,  4752,  4786,  4792,  4800,  4802,  4808,  4824,  4882,  4888,  
          423  +    4957,  4960,  4969,  4992,  5008,  5024,  5120,  5121,  5741,  5743,  
          424  +    5760,  5761,  5787,  5788,  5792,  5867,  5870,  5888,  5902,  5906,  
          425  +    5920,  5938,  5941,  5952,  5970,  5984,  5998,  6002,  6016,  6068,  
          426  +    6070,  6071,  6078,  6086,  6087,  6089,  6100,  6103,  6104,  6107,  
          427  +    6108,  6109,  6112,  6128,  6144,  6150,  6151,  6155,  6158,  6160,  
          428  +    6176,  6211,  6212,  6272,  6313,  6314,  6320,  6400,  6432,  6435,  
          429  +    6439,  6441,  6448,  6450,  6451,  6457,  6464,  6468,  6470,  6480,  
          430  +    6512,  6528,  6576,  6593,  6600,  6608,  6618,  6622,  6656,  6679,  
          431  +    6681,  6686,  6688,  6741,  6742,  6743,  6744,  6752,  6753,  6754,  
          432  +    6755,  6757,  6765,  6771,  6783,  6784,  6800,  6816,  6823,  6824,  
          433  +    6912,  6916,  6917,  6964,  6965,  6966,  6971,  6972,  6973,  6978,  
          434  +    6979,  6981,  6992,  7002,  7009,  7019,  7028,  7040,  7042,  7043,  
          435  +    7073,  7074,  7078,  7080,  7082,  7083,  7084,  7086,  7088,  7098,  
          436  +    7142,  7143,  7144,  7146,  7149,  7150,  7151,  7154,  7164,  7168,  
          437  +    7204,  7212,  7220,  7222,  7227,  7232,  7245,  7248,  7258,  7288,  
          438  +    7294,  7360,  7376,  7379,  7380,  7393,  7394,  7401,  7405,  7406,  
          439  +    7410,  7412,  7413,  7424,  7468,  7531,  7544,  7545,  7579,  7616,  
          440  +    7676,  7680,  7830,  7838,  7936,  7944,  7952,  7960,  7968,  7976,  
          441  +    7984,  7992,  8000,  8008,  8016,  8025,  8027,  8029,  8031,  8033,  
          442  +    8040,  8048,  8064,  8072,  8080,  8088,  8096,  8104,  8112,  8118,  
          443  +    8120,  8124,  8125,  8126,  8127,  8130,  8134,  8136,  8140,  8141,  
          444  +    8144,  8150,  8152,  8157,  8160,  8168,  8173,  8178,  8182,  8184,  
          445  +    8188,  8189,  8192,  8203,  8208,  8214,  8216,  8217,  8218,  8219,  
          446  +    8221,  8222,  8223,  8224,  8232,  8233,  8234,  8239,  8240,  8249,  
          447  +    8250,  8251,  8255,  8257,  8260,  8261,  8262,  8263,  8274,  8275,  
          448  +    8276,  8277,  8287,  8288,  8298,  8304,  8305,  8308,  8314,  8317,  
          449  +    8318,  8319,  8320,  8330,  8333,  8334,  8336,  8352,  8400,  8413,  
          450  +    8417,  8418,  8421,  8448,  8450,  8451,  8455,  8456,  8458,  8459,  
          451  +    8462,  8464,  8467,  8468,  8469,  8470,  8472,  8473,  8478,  8484,  
          452  +    8485,  8486,  8487,  8488,  8489,  8490,  8494,  8495,  8496,  8500,  
          453  +    8501,  8505,  8506,  8508,  8510,  8512,  8517,  8519,  8522,  8523,  
          454  +    8524,  8526,  8527,  8528,  8544,  8579,  8581,  8585,  8592,  8597,  
          455  +    8602,  8604,  8608,  8609,  8611,  8612,  8614,  8615,  8622,  8623,  
          456  +    8654,  8656,  8658,  8659,  8660,  8661,  8692,  8960,  8968,  8972,  
          457  +    8992,  8994,  9001,  9002,  9003,  9084,  9085,  9115,  9140,  9180,  
          458  +    9186,  9216,  9280,  9312,  9372,  9450,  9472,  9655,  9656,  9665,  
          459  +    9666,  9720,  9728,  9839,  9840,  9985,  10088, 10089, 10090, 10091, 
          460  +    10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 
          461  +    10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, 
          462  +    10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, 
          463  +    10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, 
          464  +    10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, 
          465  +    10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, 
          466  +    11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, 
          467  +    11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, 
          468  +    11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, 
          469  +    11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, 
          470  +    11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, 
          471  +    11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, 
          472  +    11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, 
          473  +    11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, 
          474  +    11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, 
          475  +    12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, 
          476  +    12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, 
          477  +    12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, 
          478  +    12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, 
          479  +    12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, 
          480  +    12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, 
          481  +    12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, 
          482  +    40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, 
          483  +    42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, 
          484  +    42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, 
          485  +    42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, 
          486  +    43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, 
          487  +    43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, 
          488  +    43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, 
          489  +    43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, 
          490  +    43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, 
          491  +    43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, 
          492  +    43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, 
          493  +    43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, 
          494  +    43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, 
          495  +    43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, 
          496  +    44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, 
          497  +    55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, 
          498  +    64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, 
          499  +    64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, 
          500  +    65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, 
          501  +    65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, 
          502  +    65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, 
          503  +    65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, 
          504  +    65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, 
          505  +    65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, 
          506  +    65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, 
          507  +    65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, 
          508  +    65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, 
          509  +    65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, 
          510  +    65532, 0,     13,    40,    60,    63,    80,    128,   256,   263,   
          511  +    311,   320,   373,   377,   394,   400,   464,   509,   640,   672,   
          512  +    768,   800,   816,   833,   834,   842,   896,   927,   928,   968,   
          513  +    976,   977,   1024,  1064,  1104,  1184,  2048,  2056,  2058,  2103,  
          514  +    2108,  2111,  2135,  2136,  2304,  2326,  2335,  2336,  2367,  2432,  
          515  +    2494,  2560,  2561,  2565,  2572,  2576,  2581,  2585,  2616,  2623,  
          516  +    2624,  2640,  2656,  2685,  2687,  2816,  2873,  2880,  2904,  2912,  
          517  +    2936,  3072,  3680,  4096,  4097,  4098,  4099,  4152,  4167,  4178,  
          518  +    4198,  4224,  4226,  4227,  4272,  4275,  4279,  4281,  4283,  4285,  
          519  +    4286,  4304,  4336,  4352,  4355,  4391,  4396,  4397,  4406,  4416,  
          520  +    4480,  4482,  4483,  4531,  4534,  4543,  4545,  4549,  4560,  5760,  
          521  +    5803,  5804,  5805,  5806,  5808,  5814,  5815,  5824,  8192,  9216,  
          522  +    9328,  12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, 
          523  +    53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, 
          524  +    53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, 
          525  +    54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, 
          526  +    54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, 
          527  +    54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, 
          528  +    54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, 
          529  +    54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, 
          530  +    55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, 
          531  +    55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, 
          532  +    60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, 
          533  +    61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, 
          534  +    61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, 
          535  +    61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, 
          536  +    61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, 
          537  +    62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, 
          538  +    63045, 63104, 63232, 0,     42710, 42752, 46900, 46912, 47133, 63488, 
          539  +    1,     32,    256,   0,     65533, 
          540  +  };
          541  +static u16 aFts5UnicodeData[] = {
          542  +    1025,  61,    117,   55,    117,   54,    50,    53,    57,    53,    
          543  +    49,    85,    333,   85,    121,   85,    841,   54,    53,    50,    
          544  +    56,    48,    56,    837,   54,    57,    50,    57,    1057,  61,    
          545  +    53,    151,   58,    53,    56,    58,    39,    52,    57,    34,    
          546  +    58,    56,    58,    57,    79,    56,    37,    85,    56,    47,    
          547  +    39,    51,    111,   53,    745,   57,    233,   773,   57,    261,   
          548  +    1822,  37,    542,   37,    1534,  222,   69,    73,    37,    126,   
          549  +    126,   73,    69,    137,   37,    73,    37,    105,   101,   73,    
          550  +    37,    73,    37,    190,   158,   37,    126,   126,   73,    37,    
          551  +    126,   94,    37,    39,    94,    69,    135,   41,    40,    37,    
          552  +    41,    40,    37,    41,    40,    37,    542,   37,    606,   37,    
          553  +    41,    40,    37,    126,   73,    37,    1886,  197,   73,    37,    
          554  +    73,    69,    126,   105,   37,    286,   2181,  39,    869,   582,   
          555  +    152,   390,   472,   166,   248,   38,    56,    38,    568,   3596,  
          556  +    158,   38,    56,    94,    38,    101,   53,    88,    41,    53,    
          557  +    105,   41,    73,    37,    553,   297,   1125,  94,    37,    105,   
          558  +    101,   798,   133,   94,    57,    126,   94,    37,    1641,  1541,  
          559  +    1118,  58,    172,   75,    1790,  478,   37,    2846,  1225,  38,    
          560  +    213,   1253,  53,    49,    55,    1452,  49,    44,    53,    76,    
          561  +    53,    76,    53,    44,    871,   103,   85,    162,   121,   85,    
          562  +    55,    85,    90,    364,   53,    85,    1031,  38,    327,   684,   
          563  +    333,   149,   71,    44,    3175,  53,    39,    236,   34,    58,    
          564  +    204,   70,    76,    58,    140,   71,    333,   103,   90,    39,    
          565  +    469,   34,    39,    44,    967,   876,   2855,  364,   39,    333,   
          566  +    1063,  300,   70,    58,    117,   38,    711,   140,   38,    300,   
          567  +    38,    108,   38,    172,   501,   807,   108,   53,    39,    359,   
          568  +    876,   108,   42,    1735,  44,    42,    44,    39,    106,   268,   
          569  +    138,   44,    74,    39,    236,   327,   76,    85,    333,   53,    
          570  +    38,    199,   231,   44,    74,    263,   71,    711,   231,   39,    
          571  +    135,   44,    39,    106,   140,   74,    74,    44,    39,    42,    
          572  +    71,    103,   76,    333,   71,    87,    207,   58,    55,    76,    
          573  +    42,    199,   71,    711,   231,   71,    71,    71,    44,    106,   
          574  +    76,    76,    108,   44,    135,   39,    333,   76,    103,   44,    
          575  +    76,    42,    295,   103,   711,   231,   71,    167,   44,    39,    
          576  +    106,   172,   76,    42,    74,    44,    39,    71,    76,    333,   
          577  +    53,    55,    44,    74,    263,   71,    711,   231,   71,    167,   
          578  +    44,    39,    42,    44,    42,    140,   74,    74,    44,    44,    
          579  +    42,    71,    103,   76,    333,   58,    39,    207,   44,    39,    
          580  +    199,   103,   135,   71,    39,    71,    71,    103,   391,   74,    
          581  +    44,    74,    106,   106,   44,    39,    42,    333,   111,   218,   
          582  +    55,    58,    106,   263,   103,   743,   327,   167,   39,    108,   
          583  +    138,   108,   140,   76,    71,    71,    76,    333,   239,   58,    
          584  +    74,    263,   103,   743,   327,   167,   44,    39,    42,    44,    
          585  +    170,   44,    74,    74,    76,    74,    39,    71,    76,    333,   
          586  +    71,    74,    263,   103,   1319,  39,    106,   140,   106,   106,   
          587  +    44,    39,    42,    71,    76,    333,   207,   58,    199,   74,    
          588  +    583,   775,   295,   39,    231,   44,    106,   108,   44,    266,   
          589  +    74,    53,    1543,  44,    71,    236,   55,    199,   38,    268,   
          590  +    53,    333,   85,    71,    39,    71,    39,    39,    135,   231,   
          591  +    103,   39,    39,    71,    135,   44,    71,    204,   76,    39,    
          592  +    167,   38,    204,   333,   135,   39,    122,   501,   58,    53,    
          593  +    122,   76,    218,   333,   335,   58,    44,    58,    44,    58,    
          594  +    44,    54,    50,    54,    50,    74,    263,   1159,  460,   42,    
          595  +    172,   53,    76,    167,   364,   1164,  282,   44,    218,   90,    
          596  +    181,   154,   85,    1383,  74,    140,   42,    204,   42,    76,    
          597  +    74,    76,    39,    333,   213,   199,   74,    76,    135,   108,   
          598  +    39,    106,   71,    234,   103,   140,   423,   44,    74,    76,    
          599  +    202,   44,    39,    42,    333,   106,   44,    90,    1225,  41,    
          600  +    41,    1383,  53,    38,    10631, 135,   231,   39,    135,   1319,  
          601  +    135,   1063,  135,   231,   39,    135,   487,   1831,  135,   2151,  
          602  +    108,   309,   655,   519,   346,   2727,  49,    19847, 85,    551,   
          603  +    61,    839,   54,    50,    2407,  117,   110,   423,   135,   108,   
          604  +    583,   108,   85,    583,   76,    423,   103,   76,    1671,  76,    
          605  +    42,    236,   266,   44,    74,    364,   117,   38,    117,   55,    
          606  +    39,    44,    333,   335,   213,   49,    149,   108,   61,    333,   
          607  +    1127,  38,    1671,  1319,  44,    39,    2247,  935,   108,   138,   
          608  +    76,    106,   74,    44,    202,   108,   58,    85,    333,   967,   
          609  +    167,   1415,  554,   231,   74,    333,   47,    1114,  743,   76,    
          610  +    106,   85,    1703,  42,    44,    42,    236,   44,    42,    44,    
          611  +    74,    268,   202,   332,   44,    333,   333,   245,   38,    213,   
          612  +    140,   42,    1511,  44,    42,    172,   42,    44,    170,   44,    
          613  +    74,    231,   333,   245,   346,   300,   314,   76,    42,    967,   
          614  +    42,    140,   74,    76,    42,    44,    74,    71,    333,   1415,  
          615  +    44,    42,    76,    106,   44,    42,    108,   74,    149,   1159,  
          616  +    266,   268,   74,    76,    181,   333,   103,   333,   967,   198,   
          617  +    85,    277,   108,   53,    428,   42,    236,   135,   44,    135,   
          618  +    74,    44,    71,    1413,  2022,  421,   38,    1093,  1190,  1260,  
          619  +    140,   4830,  261,   3166,  261,   265,   197,   201,   261,   265,   
          620  +    261,   265,   197,   201,   261,   41,    41,    41,    94,    229,   
          621  +    265,   453,   261,   264,   261,   264,   261,   264,   165,   69,    
          622  +    137,   40,    56,    37,    120,   101,   69,    137,   40,    120,   
          623  +    133,   69,    137,   120,   261,   169,   120,   101,   69,    137,   
          624  +    40,    88,    381,   162,   209,   85,    52,    51,    54,    84,    
          625  +    51,    54,    52,    277,   59,    60,    162,   61,    309,   52,    
          626  +    51,    149,   80,    117,   57,    54,    50,    373,   57,    53,    
          627  +    48,    341,   61,    162,   194,   47,    38,    207,   121,   54,    
          628  +    50,    38,    335,   121,   54,    50,    422,   855,   428,   139,   
          629  +    44,    107,   396,   90,    41,    154,   41,    90,    37,    105,   
          630  +    69,    105,   37,    58,    41,    90,    57,    169,   218,   41,    
          631  +    58,    41,    58,    41,    58,    137,   58,    37,    137,   37,    
          632  +    135,   37,    90,    69,    73,    185,   94,    101,   58,    57,    
          633  +    90,    37,    58,    527,   1134,  94,    142,   47,    185,   186,   
          634  +    89,    154,   57,    90,    57,    90,    57,    250,   57,    1018,  
          635  +    89,    90,    57,    58,    57,    1018,  8601,  282,   153,   666,   
          636  +    89,    250,   54,    50,    2618,  57,    986,   825,   1306,  217,   
          637  +    602,   1274,  378,   1935,  2522,  719,   5882,  57,    314,   57,    
          638  +    1754,  281,   3578,  57,    4634,  3322,  54,    50,    54,    50,    
          639  +    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
          640  +    975,   1434,  185,   54,    50,    1017,  54,    50,    54,    50,    
          641  +    54,    50,    54,    50,    54,    50,    537,   8218,  4217,  54,    
          642  +    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
          643  +    50,    54,    50,    54,    50,    54,    50,    54,    50,    54,    
          644  +    50,    2041,  54,    50,    54,    50,    1049,  54,    50,    8281,  
          645  +    1562,  697,   90,    217,   346,   1513,  1509,  126,   73,    69,    
          646  +    254,   105,   37,    94,    37,    94,    165,   70,    105,   37,    
          647  +    3166,  37,    218,   158,   108,   94,    149,   47,    85,    1221,  
          648  +    37,    37,    1799,  38,    53,    44,    743,   231,   231,   231,   
          649  +    231,   231,   231,   231,   231,   1036,  85,    52,    51,    52,    
          650  +    51,    117,   52,    51,    53,    52,    51,    309,   49,    85,    
          651  +    49,    53,    52,    51,    85,    52,    51,    54,    50,    54,    
          652  +    50,    54,    50,    54,    50,    181,   38,    341,   81,    858,   
          653  +    2874,  6874,  410,   61,    117,   58,    38,    39,    46,    54,    
          654  +    50,    54,    50,    54,    50,    54,    50,    54,    50,    90,    
          655  +    54,    50,    54,    50,    54,    50,    54,    50,    49,    54,    
          656  +    82,    58,    302,   140,   74,    49,    166,   90,    110,   38,    
          657  +    39,    53,    90,    2759,  76,    88,    70,    39,    49,    2887,  
          658  +    53,    102,   39,    1319,  3015,  90,    143,   346,   871,   1178,  
          659  +    519,   1018,  335,   986,   271,   58,    495,   1050,  335,   1274,  
          660  +    495,   2042,  8218,  39,    39,    2074,  39,    39,    679,   38,    
          661  +    36583, 1786,  1287,  198,   85,    8583,  38,    117,   519,   333,   
          662  +    71,    1502,  39,    44,    107,   53,    332,   53,    38,    798,   
          663  +    44,    2247,  334,   76,    213,   760,   294,   88,    478,   69,    
          664  +    2014,  38,    261,   190,   350,   38,    88,    158,   158,   382,   
          665  +    70,    37,    231,   44,    103,   44,    135,   44,    743,   74,    
          666  +    76,    42,    154,   207,   90,    55,    58,    1671,  149,   74,    
          667  +    1607,  522,   44,    85,    333,   588,   199,   117,   39,    333,   
          668  +    903,   268,   85,    743,   364,   74,    53,    935,   108,   42,    
          669  +    1511,  44,    74,    140,   74,    44,    138,   437,   38,    333,   
          670  +    85,    1319,  204,   74,    76,    74,    76,    103,   44,    263,   
          671  +    44,    42,    333,   149,   519,   38,    199,   122,   39,    42,    
          672  +    1543,  44,    39,    108,   71,    76,    167,   76,    39,    44,    
          673  +    39,    71,    38,    85,    359,   42,    76,    74,    85,    39,    
          674  +    70,    42,    44,    199,   199,   199,   231,   231,   1127,  74,    
          675  +    44,    74,    44,    74,    53,    42,    44,    333,   39,    39,    
          676  +    743,   1575,  36,    68,    68,    36,    63,    63,    11719, 3399,  
          677  +    229,   165,   39,    44,    327,   57,    423,   167,   39,    71,    
          678  +    71,    3463,  536,   11623, 54,    50,    2055,  1735,  391,   55,    
          679  +    58,    524,   245,   54,    50,    53,    236,   53,    81,    80,    
          680  +    54,    50,    54,    50,    54,    50,    54,    50,    54,    50,    
          681  +    54,    50,    54,    50,    54,    50,    85,    54,    50,    149,   
          682  +    112,   117,   149,   49,    54,    50,    54,    50,    54,    50,    
          683  +    117,   57,    49,    121,   53,    55,    85,    167,   4327,  34,    
          684  +    117,   55,    117,   54,    50,    53,    57,    53,    49,    85,    
          685  +    333,   85,    121,   85,    841,   54,    53,    50,    56,    48,    
          686  +    56,    837,   54,    57,    50,    57,    54,    50,    53,    54,    
          687  +    50,    85,    327,   38,    1447,  70,    999,   199,   199,   199,   
          688  +    103,   87,    57,    56,    58,    87,    58,    153,   90,    98,    
          689  +    90,    391,   839,   615,   71,    487,   455,   3943,  117,   1455,  
          690  +    314,   1710,  143,   570,   47,    410,   1466,  44,    935,   1575,  
          691  +    999,   143,   551,   46,    263,   46,    967,   53,    1159,  263,   
          692  +    53,    174,   1289,  1285,  2503,  333,   199,   39,    1415,  71,    
          693  +    39,    743,   53,    271,   711,   207,   53,    839,   53,    1799,  
          694  +    71,    39,    108,   76,    140,   135,   103,   871,   108,   44,    
          695  +    271,   309,   935,   79,    53,    1735,  245,   711,   271,   615,   
          696  +    271,   2343,  1007,  42,    44,    42,    1703,  492,   245,   655,   
          697  +    333,   76,    42,    1447,  106,   140,   74,    76,    85,    34,    
          698  +    149,   807,   333,   108,   1159,  172,   42,    268,   333,   149,   
          699  +    76,    42,    1543,  106,   300,   74,    135,   149,   333,   1383,  
          700  +    44,    42,    44,    74,    204,   42,    44,    333,   28135, 3182,  
          701  +    149,   34279, 18215, 2215,  39,    1482,  140,   422,   71,    7898,  
          702  +    1274,  1946,  74,    108,   122,   202,   258,   268,   90,    236,   
          703  +    986,   140,   1562,  2138,  108,   58,    2810,  591,   841,   837,   
          704  +    841,   229,   581,   841,   837,   41,    73,    41,    73,    137,   
          705  +    265,   133,   37,    229,   357,   841,   837,   73,    137,   265,   
          706  +    233,   837,   73,    137,   169,   41,    233,   837,   841,   837,   
          707  +    841,   837,   841,   837,   841,   837,   841,   837,   841,   901,   
          708  +    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
          709  +    809,   57,    805,   57,    197,   809,   57,    805,   57,    197,   
          710  +    809,   57,    805,   57,    197,   94,    1613,  135,   871,   71,    
          711  +    39,    39,    327,   135,   39,    39,    39,    39,    39,    39,    
          712  +    103,   71,    39,    39,    39,    39,    39,    39,    71,    39,    
          713  +    135,   231,   135,   135,   39,    327,   551,   103,   167,   551,   
          714  +    89,    1434,  3226,  506,   474,   506,   506,   367,   1018,  1946,  
          715  +    1402,  954,   1402,  314,   90,    1082,  218,   2266,  666,   1210,  
          716  +    186,   570,   2042,  58,    5850,  154,   2010,  154,   794,   2266,  
          717  +    378,   2266,  3738,  39,    39,    39,    39,    39,    39,    17351, 
          718  +    34,    3074,  7692,  63,    63,    
          719  +  };
          720  +
          721  +int sqlite3Fts5UnicodeCategory(int iCode) { 
          722  +  int iRes = -1;
          723  +  int iHi;
          724  +  int iLo;
          725  +  int ret;
          726  +  u16 iKey;
          727  +
          728  +  if( iCode>=(1<<20) ){
          729  +    return 0;
          730  +  }
          731  +  iLo = aFts5UnicodeBlock[(iCode>>16)];
          732  +  iHi = aFts5UnicodeBlock[1+(iCode>>16)];
          733  +  iKey = (iCode & 0xFFFF);
          734  +  while( iHi>iLo ){
          735  +    int iTest = (iHi + iLo) / 2;
          736  +    assert( iTest>=iLo && iTest<iHi );
          737  +    if( iKey>=aFts5UnicodeMap[iTest] ){
          738  +      iRes = iTest;
          739  +      iLo = iTest+1;
          740  +    }else{
          741  +      iHi = iTest;
          742  +    }
          743  +  }
          744  +
          745  +  if( iRes<0 ) return 0;
          746  +  if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
          747  +  ret = aFts5UnicodeData[iRes] & 0x1F;
          748  +  if( ret!=30 ) return ret;
          749  +  return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
          750  +}
          751  +
          752  +void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
          753  +  int i = 0;
          754  +  int iTbl = 0;
          755  +  while( i<128 ){
          756  +    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
          757  +    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
          758  +    for(; i<128 && i<n; i++){
          759  +      aAscii[i] = (u8)bToken;
          760  +    }
          761  +    iTbl++;
          762  +  }
          763  +}
          764  +

Changes to ext/fts5/fts5_vocab.c.

   427    427   static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
   428    428     int eDetail = pCsr->pConfig->eDetail;
   429    429     int rc = SQLITE_OK;
   430    430     Fts5IndexIter *pIter = pCsr->pIter;
   431    431     i64 *pp = &pCsr->iInstPos;
   432    432     int *po = &pCsr->iInstOff;
   433    433     
          434  +  assert( sqlite3Fts5IterEof(pIter)==0 );
          435  +  assert( pCsr->bEof==0 );
   434    436     while( eDetail==FTS5_DETAIL_NONE
   435    437         || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) 
   436    438     ){
   437    439       pCsr->iInstPos = 0;
   438    440       pCsr->iInstOff = 0;
   439    441   
   440    442       rc = sqlite3Fts5IterNextScan(pCsr->pIter);
   441    443       if( rc==SQLITE_OK ){
   442    444         rc = fts5VocabInstanceNewTerm(pCsr);
   443         -      if( eDetail==FTS5_DETAIL_NONE ) break;
          445  +      if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break;
   444    446       }
   445    447       if( rc ){
   446    448         pCsr->bEof = 1;
   447    449         break;
   448    450       }
   449    451     }
   450    452   
................................................................................
   751    753       /* xCommit       */ 0,
   752    754       /* xRollback     */ 0,
   753    755       /* xFindFunction */ 0,
   754    756       /* xRename       */ 0,
   755    757       /* xSavepoint    */ 0,
   756    758       /* xRelease      */ 0,
   757    759       /* xRollbackTo   */ 0,
          760  +    /* xShadowName   */ 0
   758    761     };
   759    762     void *p = (void*)pGlobal;
   760    763   
   761    764     return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
   762    765   }
   763         -
   764         -

Changes to ext/fts5/fts5parse.y.

   144    144   
   145    145   
   146    146   %type nearset     {Fts5ExprNearset*}
   147    147   %type nearphrases {Fts5ExprNearset*}
   148    148   %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
   149    149   %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
   150    150   
   151         -nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
          151  +nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
          152  +nearset(A) ::= CARET phrase(Y). { 
          153  +  sqlite3Fts5ParseSetCaret(Y);
          154  +  A = sqlite3Fts5ParseNearset(pParse, 0, Y); 
          155  +}
   152    156   nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
   153    157     sqlite3Fts5ParseNear(pParse, &X);
   154    158     sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
   155    159     A = Y;
   156    160   }
   157    161   
   158    162   nearphrases(A) ::= phrase(X). { 
................................................................................
   185    189     A = sqlite3Fts5ParseTerm(pParse, 0, &Y, Z);
   186    190   }
   187    191   
   188    192   /*
   189    193   ** Optional "*" character.
   190    194   */
   191    195   %type star_opt {int}
   192         -
   193    196   star_opt(A) ::= STAR. { A = 1; }
   194    197   star_opt(A) ::= . { A = 0; }

Changes to ext/fts5/test/fts5aa.test.

   405    405     }
   406    406     set nRow
   407    407   } {200}
   408    408   
   409    409   do_execsql_test 15.0 {
   410    410     INSERT INTO t1(t1) VALUES('integrity-check');
   411    411   }
          412  +sqlite3_db_config db DEFENSIVE 0
   412    413   do_execsql_test 15.1 {
   413    414     UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
   414    415   }
   415    416   do_catchsql_test 15.2 {
   416    417     INSERT INTO t1(t1) VALUES('integrity-check');
   417    418   } {1 {database disk image is malformed}}
   418    419   
................................................................................
   587    588     COMMIT;
   588    589   }
   589    590   
   590    591   do_execsql_test 22.1 {
   591    592     SELECT rowid FROM t9('a*')
   592    593   } {1}
   593    594   
          595  +#-------------------------------------------------------------------------
          596  +do_execsql_test 23.0 {
          597  +  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
          598  +  CREATE TABLE t11(x);
          599  +}
          600  +do_execsql_test 23.1 {
          601  +  SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
          602  +}
          603  +do_execsql_test 23.2 {
          604  +  SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
          605  +}
          606  +
   594    607   }
   595    608   
   596         -
          609  +expand_all_sql db
   597    610   finish_test

Changes to ext/fts5/test/fts5af.test.

   170    170       'x a a a a a a a a a a',
   171    171       'a a a a a a a a a a a a a a a a a a a x'
   172    172     );
   173    173   }
   174    174   do_execsql_test 5.1 {
   175    175     SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
   176    176   } {{[x] a a a a a...}}
          177  +
          178  +do_execsql_test 5.2 {
          179  +  SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
          180  +} {{[x] a a a a a}}
          181  +do_execsql_test 5.3 {
          182  +  SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
          183  +} {{x] a a a a a...}}
          184  +do_execsql_test 5.4 {
          185  +  SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
          186  +} {{[x a a a a a...}}
   177    187   
   178    188   } ;# foreach_detail_mode 
   179    189   
   180    190   finish_test

Added ext/fts5/test/fts5cat.test.

            1  +# 2016 Jan 15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +ifcapable !fts5 { finish_test ; return }
           15  +set ::testprefix fts5cat
           16  +
           17  +
           18  +do_execsql_test 1.0 {
           19  +  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'");
           20  +  INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names');
           21  +}
           22  +
           23  +do_execsql_test 1.1 {
           24  +  SELECT rowid FROM t1('option');
           25  +} {1}
           26  +
           27  +do_execsql_test 1.2 {
           28  +  CREATE VIRTUAL TABLE t2 USING fts5(x);
           29  +  CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row);
           30  +
           31  +  CREATE VIRTUAL TABLE t3 USING fts5(
           32  +    x, tokenize="unicode61 categories 'L* N* Co Mn'"
           33  +  );
           34  +  CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row);
           35  +
           36  +  CREATE VIRTUAL TABLE t4 USING fts5(
           37  +    x, tokenize="unicode61 categories 'L* N* Co M*'"
           38  +  );
           39  +  CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row);
           40  +
           41  +  INSERT INTO t2 VALUES ('สนามกีฬา');
           42  +  INSERT INTO t3 VALUES ('สนามกีฬา');
           43  +  INSERT INTO t4 VALUES ('สนามกีฬา');
           44  +}
           45  +
           46  +do_execsql_test 1.3 {
           47  +  SELECT * FROM t2t
           48  +} {สนามก 1 1 ฬา 1 1}
           49  +
           50  +do_execsql_test 1.4 {
           51  +  SELECT * FROM t3t
           52  +} {สนามกีฬา 1 1}
           53  +
           54  +do_execsql_test 1.5 {
           55  +  SELECT * FROM t4t
           56  +} {สนามกีฬา 1 1}
           57  +
           58  +
           59  +finish_test

Changes to ext/fts5/test/fts5connect.test.

   240    240   
   241    241     do_execsql_test 4.$tn.3 {
   242    242       INSERT INTO ft3(ft3) VALUES('integrity-check');
   243    243     }
   244    244   }
   245    245   
   246    246   finish_test
   247         -

Changes to ext/fts5/test/fts5corrupt.test.

    37     37     fts5_level_segs t1
    38     38   } {1}
    39     39   db_save
    40     40   
    41     41   do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
    42     42   set segid [lindex [fts5_level_segids t1] 0]
    43     43   
           44  +sqlite3_db_config db DEFENSIVE 0
    44     45   do_test 1.3 {
    45     46     execsql {
    46     47       DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
    47     48     }
    48     49     catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    49     50   } {1 {database disk image is malformed}}
    50     51   
    51     52   do_test 1.4 {
    52     53     db_restore_and_reopen
           54  +  sqlite3_db_config db DEFENSIVE 0
    53     55     execsql {
    54     56       UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
    55     57       rowid = fts5_rowid('segment', $segid, 4);
    56     58     }
    57     59     catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    58     60   } {1 {database disk image is malformed}}
    59     61   
................................................................................
    85     87     INSERT INTO t3 VALUES('three o');
    86     88     INSERT INTO t3 VALUES('four e');
    87     89     INSERT INTO t3 VALUES('five o');
    88     90   }
    89     91   do_execsql_test 3.1 {
    90     92     SELECT * FROM t3 WHERE t3 MATCH 'o'
    91     93   } {{one o} {three o} {five o}}
    92         -
           94  +sqlite3_db_config db DEFENSIVE 0
    93     95   do_catchsql_test 3.1 {
    94     96     DELETE FROM t3_content WHERE rowid = 3;
    95     97     SELECT * FROM t3 WHERE t3 MATCH 'o';
    96     98   } {1 {database disk image is malformed}}
    97     99   
    98    100   finish_test

Changes to ext/fts5/test/fts5corrupt2.test.

    95     95   # Also tested is that "MATCH 'x*'" does not crash and sometimes reports
    96     96   # corruption. It may not report the db as corrupt because truncating the
    97     97   # final leaf to some sizes may create a valid leaf page.
    98     98   #
    99     99   set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] 
   100    100   set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
   101    101   set all [db eval {SELECT rowid FROM t1}]
          102  +sqlite3_db_config db DEFENSIVE 0
   102    103   for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
   103    104     do_execsql_test 2.$i.1 {
   104    105       BEGIN;
   105    106         UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid;
   106    107     }
   107    108   
   108    109     do_catchsql_test 2.$i.2 {
................................................................................
   244    245   
   245    246       execsql ROLLBACK
   246    247     }
   247    248   }
   248    249   
   249    250   #--------------------------------------------------------------------
   250    251   reset_db
          252  +sqlite3_db_config db DEFENSIVE 0
   251    253   do_execsql_test 6.1 {
   252    254     CREATE VIRTUAL TABLE x5 USING fts5(tt);
   253    255     INSERT INTO x5 VALUES('a');
   254    256     INSERT INTO x5 VALUES('a a');
   255    257     INSERT INTO x5 VALUES('a a a');
   256    258     INSERT INTO x5 VALUES('a a a a');
   257    259   

Changes to ext/fts5/test/fts5corrupt3.test.

    47     47     set rowid [db one {
    48     48       SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
    49     49     }]
    50     50     set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
    51     51     set {} {}
    52     52   } {} 
    53     53   
           54  +sqlite3_db_config db DEFENSIVE 0
    54     55   for {set i 0} {$i < $L} {incr i} {
    55     56     do_test 1.2.$i {
    56     57       catchsql {
    57     58         BEGIN;
    58     59         UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid;
    59     60         INSERT INTO t1(t1) VALUES('integrity-check');
    60     61       }
................................................................................
    82     83   
    83     84   
    84     85   #-------------------------------------------------------------------------
    85     86   # Test that missing leaf pages are recognized as corruption.
    86     87   #
    87     88   reset_db
    88     89   do_test 3.0 { create_t1 } {}
           90  +sqlite3_db_config db DEFENSIVE 0
    89     91   
    90     92   do_execsql_test 3.1 {
    91     93     SELECT count(*) FROM t1_data;
    92     94   } {105}
    93     95   
    94     96   proc do_3_test {tn} {
    95     97     set i 0
................................................................................
   154    156   
   155    157   do_3_test 3.10
   156    158   
   157    159   #-------------------------------------------------------------------------
   158    160   # Test that segments that end unexpectedly are identified as corruption.
   159    161   #
   160    162   reset_db
          163  +sqlite3_db_config db DEFENSIVE 0
   161    164   do_test 4.0 {
   162    165     execsql { 
   163    166       CREATE VIRTUAL TABLE t1 USING fts5(x);
   164    167       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   165    168     }
   166    169     for {set i 0} {$i < 100} {incr i} {
   167    170       set rnd [expr int(rand() * 100)]
................................................................................
   178    181     set end [lindex $var end]
   179    182     if {$end<=$i} break
   180    183     lset var end [expr $end - $i]
   181    184     set struct [binary format c* $var]
   182    185   
   183    186     db close
   184    187     sqlite3 db test.db
          188  +  sqlite3_db_config db DEFENSIVE 0
   185    189   
   186    190     db eval {
   187    191       BEGIN;
   188    192       UPDATE t1_data SET block = $struct WHERE id=10;
   189    193     }
   190    194     do_test 4.1.$i {
   191    195       incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
................................................................................
   253    257       catch { db eval ROLLBACK }
   254    258     }
   255    259   }
   256    260   
   257    261   #------------------------------------------------------------------------
   258    262   #
   259    263   reset_db
          264  +sqlite3_db_config db DEFENSIVE 0
   260    265   do_execsql_test 6.1.0 {
   261    266     CREATE VIRTUAL TABLE t1 USING fts5(a);
   262    267     INSERT INTO t1 VALUES('bbbbb ccccc');
   263    268     SELECT quote(block) FROM t1_data WHERE rowid>100;
   264    269   } {X'000000180630626262626201020201056363636363010203040A'}
   265    270   do_execsql_test 6.1.1 {
   266    271     UPDATE t1_data SET block = 
................................................................................
   269    274   }
   270    275   do_catchsql_test 6.1.2 {
   271    276     INSERT INTO t1(t1) VALUES('integrity-check');
   272    277   } {1 {database disk image is malformed}}
   273    278   
   274    279   #-------
   275    280   reset_db
          281  +sqlite3_db_config db DEFENSIVE 0
   276    282   do_execsql_test 6.2.0 {
   277    283     CREATE VIRTUAL TABLE t1 USING fts5(a);
   278    284     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   279    285     INSERT INTO t1 VALUES('aa bb cc dd ee');
   280    286     SELECT pgno, quote(term) FROM t1_idx;
   281    287   } {2 X'' 4 X'3064'}
   282    288   do_execsql_test 6.2.1 {
................................................................................
   284    290   }
   285    291   do_catchsql_test 6.2.2 {
   286    292     INSERT INTO t1(t1) VALUES('integrity-check');
   287    293   } {1 {database disk image is malformed}}
   288    294   
   289    295   #-------
   290    296   reset_db
          297  +sqlite3_db_config db DEFENSIVE 0
   291    298   do_execsql_test 6.3.0 {
   292    299     CREATE VIRTUAL TABLE t1 USING fts5(a);
   293    300     INSERT INTO t1 VALUES('abc abcdef abcdefghi');
   294    301     SELECT quote(block) FROM t1_data WHERE id>100;
   295    302   }    {X'0000001C043061626301020204036465660102030703676869010204040808'}
   296    303   do_execsql_test 6.3.1 {
   297    304     BEGIN;
................................................................................
   358    365       INSERT INTO t5 VALUES( rnddoc(10000) );
   359    366       INSERT INTO t5 VALUES( rnddoc(10000) );
   360    367       INSERT INTO t5 VALUES( rnddoc(10000) );
   361    368       INSERT INTO t5(t5) VALUES('optimize');
   362    369     }
   363    370   } {}
   364    371   
          372  +sqlite3_db_config db DEFENSIVE 0
   365    373   do_test 7.1 {
   366    374     foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
   367    375       db eval BEGIN  
   368    376       db eval {DELETE FROM t5_data WHERE rowid = $i}
   369    377       set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ]
   370    378       if {$r != "1 {database disk image is malformed}"} { error $r }
   371    379       db eval ROLLBACK  
................................................................................
   379    387   #
   380    388   reset_db
   381    389   do_execsql_test 8.1 {
   382    390     CREATE VIRTUAL TABLE t1 USING fts5(x, y);
   383    391     INSERT INTO t1 VALUES('one', 'two');
   384    392   }
   385    393   
          394  +sqlite3_db_config db DEFENSIVE 0
   386    395   do_test 9.1.1 {
   387    396     set    blob "12345678"    ;# cookie
   388    397     append blob "0105"        ;# 1 level, total of 5 segments
   389    398     append blob "06"          ;# write counter
   390    399     append blob "0002"        ;# first level has 0 segments merging, 2 other.
   391    400     append blob "450108"      ;# first segment
   392    401     execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"

Changes to ext/fts5/test/fts5fault6.test.

   249    249     sqlite3_fts5_register_matchinfo db
   250    250     db func mit mit
   251    251   } -body {
   252    252     db eval { 
   253    253       SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
   254    254     }
   255    255   } -test {
   256         -  faultsim_test_result [list 0 $::res]
          256  +  faultsim_test_result [list 0 $::res] {1 {SQL logic error}}
   257    257   }
   258    258   
   259    259   do_faultsim_test 5.3 -faults oom* -prep {
   260    260     faultsim_restore_and_reopen
   261    261     sqlite3_fts5_create_tokenizer db tcl tcl_create
   262    262   } -body {
   263    263     db eval { 
   264    264       SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
   265    265     }
   266    266   } -test {
   267         -  faultsim_test_result {0 29}
          267  +  faultsim_test_result {0 29} {1 {SQL logic error}}
   268    268   }
   269    269   
   270    270   do_faultsim_test 5.4 -faults oom* -prep {
   271    271     faultsim_restore_and_reopen
   272    272     sqlite3_fts5_create_tokenizer db tcl tcl_create
   273    273   } -body {
   274    274     db eval { 
   275    275       SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
   276    276     }
   277    277   } -test {
   278         -  faultsim_test_result {0 1}
          278  +  faultsim_test_result {0 1} {1 {SQL logic error}}
   279    279   }
   280    280   
   281    281   #-------------------------------------------------------------------------
   282    282   catch { db close }
   283    283   do_faultsim_test 6 -faults oom* -prep {
   284    284     sqlite_orig db test.db
   285    285     sqlite3_db_config_lookaside db 0 0 0

Changes to ext/fts5/test/fts5fault9.test.

    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   foreach_detail_mode $testprefix {
    26     26   
           27  +if {"%DETAIL%" != "none"} continue
           28  +
    27     29   fts5_aux_test_functions db
    28     30   
    29     31   do_execsql_test 1.0 {
    30     32     CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    31     33     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    32     34     WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
    33     35     INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
................................................................................
    94     96     INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
    95     97   }
    96     98   
    97     99   do_faultsim_test 4.1 -faults oom-t* -body {
    98    100     execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
    99    101   } -test {
   100    102     faultsim_test_result \
   101         -      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
          103  +      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} \
          104  +      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
   102    105   }
   103    106   
   104    107   do_faultsim_test 4.2 -faults oom-t* -body {
   105    108     execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
   106    109   } -test {
   107    110     faultsim_test_result \
   108         -      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
          111  +      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} \
          112  +      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
   109    113   }
   110    114   
   111    115   
   112    116   #-------------------------------------------------------------------------
   113    117   # An OOM within an "ORDER BY rank" query.
   114    118   #
   115    119   db func rnddoc fts5_rnddoc 

Changes to ext/fts5/test/fts5faultB.test.

   125    125   }
   126    126   
   127    127   do_faultsim_test 4.2 -faults oom* -body {
   128    128     execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
   129    129   } -test {
   130    130     faultsim_test_result {0 {2 3}}
   131    131   }
          132  +
          133  +#-------------------------------------------------------------------------
          134  +# Test OOM injection while parsing a CARET expression
          135  +#
          136  +reset_db
          137  +do_execsql_test 5.0 {
          138  +  CREATE VIRTUAL TABLE t1 USING fts5(a);
          139  +  INSERT INTO t1 VALUES('a b c d');  -- 1
          140  +  INSERT INTO t1 VALUES('d a b c');  -- 2
          141  +  INSERT INTO t1 VALUES('c d a b');  -- 3
          142  +  INSERT INTO t1 VALUES('b c d a');  -- 4
          143  +}
          144  +do_faultsim_test 5.1 -faults oom* -body {
          145  +  execsql { SELECT rowid FROM t1('^a OR ^b') }
          146  +} -test {
          147  +  faultsim_test_result {0 {1 4}}
          148  +}
   132    149   
   133    150   
   134    151   finish_test

Added ext/fts5/test/fts5first.test.

            1  +# 2017 November 25
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +
           12  +source [file join [file dirname [info script]] fts5_common.tcl]
           13  +set testprefix fts5first
           14  +
           15  +ifcapable !fts5 {
           16  +  finish_test
           17  +  return
           18  +}
           19  +
           20  +
           21  +do_execsql_test 1.0 {
           22  +  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
           23  +}
           24  +
           25  +foreach {tn expr ok} {
           26  +  1 {^abc}           1
           27  +  2 {^abc + def}     1
           28  +  3 {^ "abc def"}    1
           29  +  4 {^"abc def"}     1
           30  +  5 {abc ^def}       1
           31  +  6 {abc + ^def}     0
           32  +  7 {abc ^+ def}     0
           33  +  8 {"^abc"}         1
           34  +  9 {NEAR(^abc def)} 0
           35  +} {
           36  +  set res(0) {/1 {fts5: syntax error near .*}/}
           37  +  set res(1) {0 {}}
           38  +
           39  +  do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok)
           40  +}
           41  +
           42  +#-------------------------------------------------------------------------
           43  +# 
           44  +do_execsql_test 2.0 {
           45  +  INSERT INTO x1 VALUES('a b c', 'b c a');
           46  +}
           47  +
           48  +foreach {tn expr match} {
           49  +  1 {^a} 1
           50  +  2 {^b} 1
           51  +  3 {^c} 0
           52  +  4 {^a + b} 1
           53  +  5 {^b + c} 1
           54  +  6 {^c + a} 0
           55  +  7 {^"c a"} 0
           56  +  8 {a:^a} 1
           57  +  9 {a:^b} 0
           58  +  10 {a:^"a b"} 1
           59  +} {
           60  +  do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match
           61  +}
           62  +
           63  +#-------------------------------------------------------------------------
           64  +# 
           65  +do_execsql_test 3.0 {
           66  +  DELETE FROM x1;
           67  +  INSERT INTO x1 VALUES('b a', 'c a');
           68  +  INSERT INTO x1 VALUES('a a', 'c c');
           69  +  INSERT INTO x1 VALUES('a b', 'a a');
           70  +}
           71  +fts5_aux_test_functions db
           72  +
           73  +foreach {tn expr expect} {
           74  +  1 {^a} {{2 1}}
           75  +  2 {^c AND ^b} {{0 2} {1 0}}
           76  +} {
           77  +  do_execsql_test 3.$tn {
           78  +    SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1
           79  +  } [list $expect]
           80  +}
           81  +
           82  +#-------------------------------------------------------------------------
           83  +# 
           84  +do_execsql_test 3.1 {
           85  +  CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column);
           86  +}
           87  +
           88  +do_catchsql_test 3.2 {
           89  +  SELECT * FROM x2('a + b');
           90  +} {1 {fts5: phrase queries are not supported (detail!=full)}}
           91  +
           92  +do_catchsql_test 3.3 {
           93  +  SELECT * FROM x2('^a');
           94  +} {1 {fts5: phrase queries are not supported (detail!=full)}}
           95  +finish_test

Changes to ext/fts5/test/fts5integrity.test.

    67     67     INSERT INTO aa(zz) VALUES('a');
    68     68     SELECT length(sz) FROM aa_docsize;
    69     69   } {1 1 1 1 1}
    70     70   do_execsql_test 4.1 { 
    71     71     INSERT INTO aa(aa) VALUES('integrity-check'); 
    72     72   }
    73     73   
           74  +sqlite3_db_config db DEFENSIVE 0
    74     75   do_catchsql_test 4.2 { 
    75     76     BEGIN;
    76     77       UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
    77     78       INSERT INTO aa(aa) VALUES('integrity-check'); 
    78     79   } {1 {database disk image is malformed}}
    79     80   
    80     81   do_catchsql_test 4.3 { 

Changes to ext/fts5/test/fts5plan.test.

    25     25     CREATE TABLE t1(x, y);
    26     26     CREATE VIRTUAL TABLE f1 USING fts5(ff);
    27     27   }
    28     28   
    29     29   do_eqp_test 1.1 {
    30     30     SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
    31     31   } {
    32         -  0 0 0 {SCAN TABLE t1} 
    33         -  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
           32  +  QUERY PLAN
           33  +  |--SCAN TABLE t1
           34  +  `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
    34     35   }
    35     36   
    36     37   do_eqp_test 1.2 {
    37     38     SELECT * FROM t1, f1 WHERE f1 > t1.x
    38     39   } {
    39         -  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
    40         -  0 1 0 {SCAN TABLE t1} 
           40  +  QUERY PLAN
           41  +  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
           42  +  `--SCAN TABLE t1
    41     43   }
    42     44   
    43     45   do_eqp_test 1.3 {
    44     46     SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
    45     47   } {
    46         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
    47         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
           48  +  QUERY PLAN
           49  +  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
           50  +  `--USE TEMP B-TREE FOR ORDER BY
    48     51   }
    49     52   
    50     53   do_eqp_test 1.4 {
    51     54     SELECT * FROM f1 ORDER BY rank
    52     55   } {
    53         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
    54         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
           56  +  QUERY PLAN
           57  +  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
           58  +  `--USE TEMP B-TREE FOR ORDER BY
    55     59   }
    56     60   
    57     61   do_eqp_test 1.5 {
    58     62     SELECT * FROM f1 WHERE rank MATCH ?
    59         -} {
    60         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
    61         -}
    62         -
    63         -
    64         -
           63  +} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
    65     64   
    66     65   finish_test

Changes to ext/fts5/test/fts5query.test.

    60     60       foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] {
    61     61         set doc [string repeat "$x " 30]
    62     62         execsql { INSERT INTO t1 VALUES($doc) }
    63     63       }
    64     64       execsql COMMIT
    65     65     } {}
    66     66   
    67         -  do_execsql_test 1.$tn.2 {
           67  +  do_execsql_test 2.$tn.2 {
    68     68       INSERT INTO t1(t1) VALUES('integrity-check');
    69     69     }
    70     70   
    71     71     set ret 1
    72     72     foreach x [list a c e g i k m o q s u] {
    73     73       do_execsql_test 2.$tn.3.$ret {
    74     74         SELECT rowid FROM t1 WHERE t1 MATCH $x || '*';
    75     75       } {}
    76     76       incr ret
    77     77     }
    78     78   }
    79     79   
           80  +reset_db
           81  +do_execsql_test 3.0 {
           82  +  CREATE VIRTUAL TABLE x1 USING fts5(a);
           83  +  INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota');
           84  +  INSERT INTO x1(rowid, a) VALUES(1, 'tarago');
           85  +}
           86  +do_execsql_test 3.1 {
           87  +  SELECT rowid FROM x1('t*');
           88  +} {-1000000000000 1}
           89  +
    80     90   
    81     91   finish_test

Changes to ext/fts5/test/fts5rank.test.

   144    144     );
   145    145     INSERT INTO VTest (Title, Author) VALUES ('wrinkle in time', 'Bill Smith');
   146    146   
   147    147     SELECT * FROM VTest WHERE 
   148    148     VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
   149    149   } {{wrinkle in time} {Bill Smith}}
   150    150   
   151         -
          151  +#-------------------------------------------------------------------------
          152  +reset_db
          153  +do_execsql_test 5.0 {
          154  +  CREATE VIRTUAL TABLE ttt USING fts5(a);
          155  +  WITH s(i) AS (
          156  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
          157  +  )
          158  +  INSERT INTO ttt SELECT 'word ' || i FROM s;
          159  +}
   152    160   
          161  +do_execsql_test 5.1 {
          162  +  SELECT rowid FROM ttt('word') WHERE rowid BETWEEN 30 AND 40 ORDER BY rank;
          163  +} {30 31 32 33 34 35 36 37 38 39 40}
   153    164   
   154    165   finish_test

Changes to ext/fts5/test/fts5rebuild.test.

    35     35     INSERT INTO f1(f1) VALUES('rebuild');
    36     36   } {}
    37     37   
    38     38   do_execsql_test 1.4 {
    39     39     INSERT INTO f1(f1) VALUES('integrity-check');
    40     40   } {}
    41     41   
           42  +sqlite3_db_config db DEFENSIVE 0
    42     43   do_execsql_test 1.5 {
    43     44     DELETE FROM f1_data;
    44     45   } {}
    45     46   
    46     47   do_catchsql_test 1.6 {
    47     48     INSERT INTO f1(f1) VALUES('integrity-check');
    48     49   } {1 {database disk image is malformed}}

Changes to ext/fts5/test/fts5rowid.test.

    66     66     DELETE FROM x1 WHERE (rowid%2);
    67     67   }
    68     68   
    69     69   set res [db one {SELECT count(*) FROM x1_data}]
    70     70   do_execsql_test 2.3 {
    71     71     SELECT count(fts5_decode(rowid, block)) FROM x1_data;
    72     72   } $res
           73  +sqlite3_db_config db DEFENSIVE 0
    73     74   do_execsql_test 2.4 {
    74     75     UPDATE x1_data SET block = X'';
    75     76     SELECT count(fts5_decode(rowid, block)) FROM x1_data;
    76     77   } $res
    77     78   
    78     79   do_execsql_test 2.5 {
    79     80     INSERT INTO x1(x1, rank) VALUES('pgsz', 1024);

Changes to ext/fts5/test/fts5unicode.test.

    37     37     tokenize_test 1.$tn.2 $t {..May...you.shAre.freely} {may you share freely}
    38     38     tokenize_test 1.$tn.3 $t {} {}
    39     39   }
    40     40   
    41     41   #-------------------------------------------------------------------------
    42     42   # Check that "unicode61" really is the default tokenizer.
    43     43   #
    44         -
    45     44   do_execsql_test 2.0 "
    46     45     CREATE VIRTUAL TABLE t1 USING fts5(x);
    47     46     CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
    48     47     CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii);
    49     48     INSERT INTO t1 VALUES('\xC0\xC8\xCC');
    50     49     INSERT INTO t2 VALUES('\xC0\xC8\xCC');
    51     50     INSERT INTO t3 VALUES('\xC0\xC8\xCC');
    52     51   "
    53     52   do_execsql_test 2.1 "
    54     53     SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC';
    55     54     SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC';
    56     55     SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
    57     56   " {t1 t2}
           57  +
           58  +#-------------------------------------------------------------------------
           59  +# Check that codepoints that require 4 bytes to store in utf-8 (those that
           60  +# require 17 or more bits to store).
           61  +#
           62  +
           63  +set A [db one {SELECT char(0x1F75E)}]    ;# Type So
           64  +set B [db one {SELECT char(0x1F5FD)}]    ;# Type So
           65  +set C [db one {SELECT char(0x2F802)}]    ;# Type Lo
           66  +set D [db one {SELECT char(0x2F808)}]    ;# Type Lo
           67  +
           68  +do_execsql_test 3.0 "
           69  +  CREATE VIRTUAL TABLE xyz USING fts5(x,
           70  +    tokenize = \"unicode61 separators '$C' tokenchars '$A'\"
           71  +  );
           72  +  CREATE VIRTUAL TABLE xyz_v USING fts5vocab(xyz, row);
           73  +
           74  +  INSERT INTO xyz VALUES('$A$B$C$D');
           75  +"
           76  +
           77  +do_execsql_test 3.1 {
           78  +  SELECT * FROM xyz_v;
           79  +} [list $A 1 1 $D 1 1]
           80  +  
           81  +
           82  +
    58     83   
    59     84   
    60     85   finish_test

Added ext/fts5/test/fts5unicode4.test.

            1  +# 2018 July 25
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +#
           13  +
           14  +source [file join [file dirname [info script]] fts5_common.tcl]
           15  +set testprefix fts5unicode4
           16  +
           17  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           18  +ifcapable !fts5 {
           19  +  finish_test
           20  +  return
           21  +}
           22  +
           23  +do_execsql_test 1.0 {
           24  +  CREATE VIRTUAL TABLE sss USING fts5(a, prefix=3); 
           25  +}
           26  +
           27  +do_execsql_test 1.1 {
           28  +  INSERT INTO sss VALUES('まりや');
           29  +}
           30  +
           31  +finish_test

Changes to ext/fts5/test/fts5version.test.

    32     32     SELECT * FROM t1_config WHERE k='version'
    33     33   } {version 4}
    34     34   
    35     35   do_execsql_test 1.3 {
    36     36     SELECT rowid FROM t1 WHERE t1 MATCH 'a';
    37     37   } {1}
    38     38   
           39  +sqlite3_db_config db DEFENSIVE 0
    39     40   do_execsql_test 1.4 {
    40     41     UPDATE t1_config set v=5 WHERE k='version';
    41         -} 
           42  +}
    42     43   
    43     44   do_test 1.5 {
    44     45     db close
    45     46     sqlite3 db test.db
    46     47     catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
    47     48   } {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
    48     49   
................................................................................
    49     50   do_test 1.6 {
    50     51     db close
    51     52     sqlite3 db test.db
    52     53     catchsql { INSERT INTO t1 VALUES('x y z') }
    53     54   } {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
    54     55   
    55     56   do_test 1.7 {
           57  +  sqlite3_db_config db DEFENSIVE 0
    56     58     execsql { DELETE FROM t1_config WHERE k='version' }
    57     59     db close
    58     60     sqlite3 db test.db
    59     61     catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
    60     62   } {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
    61     63   
    62     64   
    63     65   finish_test

Changes to ext/fts5/test/fts5vocab.test.

   416    416     i a 1 1 i b 1 1 i c 1 1
   417    417   }]
   418    418   if {[detail_is_none]} { set resc [row_to_col $resr] }
   419    419   
   420    420   do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
   421    421   do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc
   422    422   
          423  +sqlite3_db_config db DEFENSIVE 0
   423    424   do_execsql_test 8.2 {
   424    425     PRAGMA writable_schema = 1;
   425    426     UPDATE sqlite_master 
   426    427     SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
   427    428     WHERE name = 'x1';
   428    429   }
   429    430   db close
................................................................................
   477    478     set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }]
   478    479     expr [lsearch $e2 SorterSort]<0
   479    480   } 0
   480    481   
   481    482   
   482    483   
   483    484   finish_test
   484         -

Changes to ext/fts5/test/fts5vocab2.test.

     9      9   #
    10     10   #***********************************************************************
    11     11   #
    12     12   # The tests in this file focus on testing the fts5vocab module.
    13     13   #
    14     14   
    15     15   source [file join [file dirname [info script]] fts5_common.tcl]
    16         -set testprefix fts5vocab
           16  +set testprefix fts5vocab2
    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
    23     23   
................................................................................
   202    202   do_execsql_test 3.5 {
   203    203     DELETE FROM t1;
   204    204     SELECT * FROM v1;
   205    205   } {
   206    206   }
   207    207   
   208    208   finish_test
   209         -

Changes to ext/icu/README.txt.

    35     35   
    36     36          http://www.icu-project.org/userguide/caseMappings.html
    37     37          http://www.icu-project.org/userguide/posix.html#case_mappings
    38     38   
    39     39       To utilise "general" case mapping, the upper() or lower() scalar 
    40     40       functions are invoked with one argument:
    41     41   
    42         -        upper('ABC') -> 'abc'
    43         -        lower('abc') -> 'ABC'
           42  +        upper('abc') -> 'ABC'
           43  +        lower('ABC') -> 'abc'
    44     44   
    45     45       To access ICU "language specific" case mapping, upper() or lower()
    46     46       should be invoked with two arguments. The second argument is the name
    47     47       of the locale to use. Passing an empty string ("") or SQL NULL value
    48     48       as the second argument is the same as invoking the 1 argument version
    49     49       of upper() or lower():
    50     50   

Changes to ext/icu/icu.c.

    24     24   **
    25     25   **   * Integration of ICU and SQLite collation sequences.
    26     26   **
    27     27   **   * An implementation of the LIKE operator that uses ICU to 
    28     28   **     provide case-independent matching.
    29     29   */
    30     30   
    31         -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
           31  +#if !defined(SQLITE_CORE)                  \
           32  + || defined(SQLITE_ENABLE_ICU)             \
           33  + || defined(SQLITE_ENABLE_ICU_COLLATIONS)
    32     34   
    33     35   /* Include ICU headers */
    34     36   #include <unicode/utypes.h>
    35     37   #include <unicode/uregex.h>
    36     38   #include <unicode/ustring.h>
    37     39   #include <unicode/ucol.h>
    38     40   
................................................................................
    41     43   #ifndef SQLITE_CORE
    42     44     #include "sqlite3ext.h"
    43     45     SQLITE_EXTENSION_INIT1
    44     46   #else
    45     47     #include "sqlite3.h"
    46     48   #endif
    47     49   
           50  +/*
           51  +** This function is called when an ICU function called from within
           52  +** the implementation of an SQL scalar function returns an error.
           53  +**
           54  +** The scalar function context passed as the first argument is 
           55  +** loaded with an error message based on the following two args.
           56  +*/
           57  +static void icuFunctionError(
           58  +  sqlite3_context *pCtx,       /* SQLite scalar function context */
           59  +  const char *zName,           /* Name of ICU function that failed */
           60  +  UErrorCode e                 /* Error code returned by ICU function */
           61  +){
           62  +  char zBuf[128];
           63  +  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
           64  +  zBuf[127] = '\0';
           65  +  sqlite3_result_error(pCtx, zBuf, -1);
           66  +}
           67  +
           68  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
           69  +
    48     70   /*
    49     71   ** Maximum length (in bytes) of the pattern in a LIKE or GLOB
    50     72   ** operator.
    51     73   */
    52     74   #ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
    53     75   # define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000
    54     76   #endif
................................................................................
   220    242     }
   221    243   
   222    244     if( zA && zB ){
   223    245       sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc));
   224    246     }
   225    247   }
   226    248   
   227         -/*
   228         -** This function is called when an ICU function called from within
   229         -** the implementation of an SQL scalar function returns an error.
   230         -**
   231         -** The scalar function context passed as the first argument is 
   232         -** loaded with an error message based on the following two args.
   233         -*/
   234         -static void icuFunctionError(
   235         -  sqlite3_context *pCtx,       /* SQLite scalar function context */
   236         -  const char *zName,           /* Name of ICU function that failed */
   237         -  UErrorCode e                 /* Error code returned by ICU function */
   238         -){
   239         -  char zBuf[128];
   240         -  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
   241         -  zBuf[127] = '\0';
   242         -  sqlite3_result_error(pCtx, zBuf, -1);
   243         -}
   244         -
   245    249   /*
   246    250   ** Function to delete compiled regexp objects. Registered as
   247    251   ** a destructor function with sqlite3_set_auxdata().
   248    252   */
   249    253   static void icuRegexpDelete(void *p){
   250    254     URegularExpression *pExpr = (URegularExpression *)p;
   251    255     uregex_close(pExpr);
................................................................................
   403    407         icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status);
   404    408       }
   405    409       return;
   406    410     }
   407    411     assert( 0 );     /* Unreachable */
   408    412   }
   409    413   
          414  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
          415  +
   410    416   /*
   411    417   ** Collation sequence destructor function. The pCtx argument points to
   412    418   ** a UCollator structure previously allocated using ucol_open().
   413    419   */
   414    420   static void icuCollationDel(void *pCtx){
   415    421     UCollator *p = (UCollator *)pCtx;
   416    422     ucol_close(p);
................................................................................
   497    503       const char *zName;                        /* Function name */
   498    504       unsigned char nArg;                       /* Number of arguments */
   499    505       unsigned short enc;                       /* Optimal text encoding */
   500    506       unsigned char iContext;                   /* sqlite3_user_data() context */
   501    507       void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   502    508     } scalars[] = {
   503    509       {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
          510  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
   504    511       {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
   505    512       {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
   506    513       {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
   507    514       {"upper",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
   508    515       {"upper",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
   509    516       {"lower",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
   510    517       {"lower",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
   511    518       {"upper",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
   512    519       {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
   513    520       {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
   514    521       {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
          522  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
   515    523     };
   516    524     int rc = SQLITE_OK;
   517    525     int i;
   518         -
   519    526     
   520    527     for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
   521    528       const struct IcuScalar *p = &scalars[i];
   522    529       rc = sqlite3_create_function(
   523    530           db, p->zName, p->nArg, p->enc, 
   524    531           p->iContext ? (void*)db : (void*)0,
   525    532           p->xFunc, 0, 0

Changes to ext/lsm1/Makefile.

    39     39                $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \
    40     40                $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \
    41     41                $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c
    42     42   
    43     43   
    44     44   # all: lsm.so
    45     45   
    46         -LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
           46  +LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB
    47     47   
    48     48   lsm.so:	$(LSMOBJ)
    49     49   	$(TCCX) -shared -o lsm.so $(LSMOBJ)
    50     50   
    51     51   %.o:	$(LSMDIR)/%.c $(LSMHDR) sqlite3.h
    52     52   	$(TCCX) $(LSMOPTS) -c $<
    53     53   	
    54     54   lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
    55     55   	# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
    56         -	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)
           56  +	$(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz

Changes to ext/lsm1/lsm-test/lsmtest.h.

   117    117   ** Functions in wrapper3.c. This file contains the tdb wrapper for lsm.
   118    118   ** The wrapper for lsm is a bit more involved than the others, as it 
   119    119   ** includes code for a couple of different lsm configurations, and for
   120    120   ** various types of fault injection and robustness testing.
   121    121   */
   122    122   int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
   123    123   int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
          124  +int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
   124    125   int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
   125    126   int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
   126    127   int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
   127    128   int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
   128    129   
   129    130   int tdb_lsm_configure(lsm_db *, const char *);
   130    131   

Changes to ext/lsm1/lsm-test/lsmtest1.c.

   270    270     int *pRc                        /* OUT: Error code */
   271    271   ){
   272    272     int i;
   273    273     int iDot;
   274    274     int rc = LSM_OK;
   275    275     Datasource *pData;
   276    276     TestDb *pDb;
          277  +  int iToggle = 0;
   277    278   
   278    279     /* Start the test case, open a database and allocate the datasource. */
   279    280     pDb = testOpen(zSystem, 1, &rc);
   280    281     pData = testDatasourceNew(&p->defn);
   281    282   
   282    283     i = 0;
   283    284     iDot = 0;
   284    285     while( rc==LSM_OK && i<p->nRow ){
   285    286   
   286    287       /* Insert some data */
   287    288       testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
   288    289       i += p->nVerify;
   289    290   
          291  +    if( iToggle ) testBegin(pDb, 1, &rc);
   290    292       /* Check that the db content is correct. */
   291    293       testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
          294  +    if( iToggle ) testCommit(pDb, 0, &rc);
          295  +    iToggle = (iToggle+1)%2;
   292    296   
   293    297       if( bRecover ){
   294    298         testReopenRecover(&pDb, &rc);
   295    299       }else{
   296    300         testReopen(&pDb, &rc);
   297    301       }
   298    302   

Changes to ext/lsm1/lsm-test/lsmtest_tdb.c.

   717    717     const char *zName;
   718    718     const char *zDefaultDb;
   719    719     int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb);
   720    720   } aLib[] = {
   721    721     { "sqlite3",      "testdb.sqlite",    sql_open },
   722    722     { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   723    723     { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
          724  +  { "lsm_lomem2",   "testdb.lsm_lomem2", test_lsm_lomem2_open },
   724    725   #ifdef HAVE_ZLIB
   725    726     { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
   726    727   #endif
   727    728     { "lsm",          "testdb.lsm",       test_lsm_open },
   728    729   #ifdef LSM_MUTEX_PTHREADS
   729    730     { "lsm_mt2",      "testdb.lsm_mt2",   test_lsm_mt2 },
   730    731     { "lsm_mt3",      "testdb.lsm_mt3",   test_lsm_mt3 },

Changes to ext/lsm1/lsm-test/lsmtest_tdb3.c.

   613    613   ){
   614    614     int rc;
   615    615     LsmDb *pDb = (LsmDb *)pTestDb;
   616    616     lsm_cursor *csr;
   617    617   
   618    618     if( pKey==0 ) return LSM_OK;
   619    619   
   620         -  rc = lsm_csr_open(pDb->db, &csr);
   621         -  if( rc!=LSM_OK ) return rc;
          620  +  if( pDb->pCsr==0 ){
          621  +    rc = lsm_csr_open(pDb->db, &csr);
          622  +    if( rc!=LSM_OK ) return rc;
          623  +  }else{
          624  +    csr = pDb->pCsr;
          625  +  }
   622    626   
   623    627     rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ);
   624    628     if( rc==LSM_OK ){
   625    629       if( lsm_csr_valid(csr) ){
   626    630         const void *pVal; int nVal;
   627    631         rc = lsm_csr_value(csr, &pVal, &nVal);
   628    632         if( nVal>pDb->nBuf ){
................................................................................
   634    638         *ppVal = pDb->pBuf;
   635    639         *pnVal = nVal;
   636    640       }else{
   637    641         *ppVal = 0;
   638    642         *pnVal = -1;
   639    643       }
   640    644     }
   641         -  lsm_csr_close(csr);
          645  +  if( pDb->pCsr==0 ){
          646  +    lsm_csr_close(csr);
          647  +  }
   642    648     return rc;
   643    649   }
   644    650   
   645    651   static int test_lsm_scan(
   646    652     TestDb *pTestDb,
   647    653     void *pCtx,
   648    654     int bReverse,
   649    655     void *pFirst, int nFirst,
   650    656     void *pLast, int nLast,
   651    657     void (*xCallback)(void *, void *, int , void *, int)
   652    658   ){
   653    659     LsmDb *pDb = (LsmDb *)pTestDb;
   654    660     lsm_cursor *csr;
          661  +  lsm_cursor *csr2 = 0;
   655    662     int rc;
   656    663   
   657         -  rc = lsm_csr_open(pDb->db, &csr);
   658         -  if( rc!=LSM_OK ) return rc;
          664  +  if( pDb->pCsr==0 ){
          665  +    rc = lsm_csr_open(pDb->db, &csr);
          666  +    if( rc!=LSM_OK ) return rc;
          667  +  }else{
          668  +    rc = LSM_OK;
          669  +    csr = pDb->pCsr;
          670  +  }
          671  +
          672  +  /* To enhance testing, if both pLast and pFirst are defined, seek the
          673  +  ** cursor to the "end" boundary here. Then the next block seeks it to
          674  +  ** the "start" ready for the scan. The point is to test that cursors
          675  +  ** can be reused.  */
          676  +  if( pLast && pFirst ){
          677  +    if( bReverse ){
          678  +      rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE);
          679  +    }else{
          680  +      rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE);
          681  +    }
          682  +  }
   659    683   
   660    684     if( bReverse ){
   661    685       if( pLast ){
   662    686         rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_LE);
   663    687       }else{
   664    688         rc = lsm_csr_last(csr);
   665    689       }
................................................................................
   692    716       if( bReverse ){
   693    717         rc = lsm_csr_prev(csr);
   694    718       }else{
   695    719         rc = lsm_csr_next(csr);
   696    720       }
   697    721     }
   698    722   
   699         -  lsm_csr_close(csr);
          723  +  if( pDb->pCsr==0 ){
          724  +    lsm_csr_close(csr);
          725  +  }
   700    726     return rc;
   701    727   }
   702    728   
   703    729   static int test_lsm_begin(TestDb *pTestDb, int iLevel){
   704    730     int rc = LSM_OK;
   705    731     LsmDb *pDb = (LsmDb *)pTestDb;
   706    732   
................................................................................
   757    783   
   758    784   #define TEST_NO_RECOVERY -1
   759    785   #define TEST_COMPRESSION -3
   760    786   
   761    787   #define TEST_MT_MODE     -2
   762    788   #define TEST_MT_MIN_CKPT -4
   763    789   #define TEST_MT_MAX_CKPT -5
          790  +
   764    791   
   765    792   int test_lsm_config_str(
   766    793     LsmDb *pLsm,
   767    794     lsm_db *db, 
   768    795     int bWorker,
   769    796     const char *zStr,
   770    797     int *pnThread
................................................................................
  1028   1055     const char *zCfg = 
  1029   1056       "page_size=256 block_size=64 autoflush=16 "
  1030   1057       "autocheckpoint=32"
  1031   1058       "mmap=0 "
  1032   1059     ;
  1033   1060     return testLsmOpen(zCfg, zFilename, bClear, ppDb);
  1034   1061   }
         1062  +
         1063  +int test_lsm_lomem2_open(
         1064  +  const char *zSpec, 
         1065  +  const char *zFilename, 
         1066  +  int bClear, 
         1067  +  TestDb **ppDb
         1068  +){
         1069  +    /* "max_freelist=4 autocheckpoint=32" */
         1070  +  const char *zCfg = 
         1071  +    "page_size=512 block_size=64 autoflush=0 mmap=0 "
         1072  +  ;
         1073  +  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
         1074  +}
  1035   1075   
  1036   1076   int test_lsm_zip_open(
  1037   1077     const char *zSpec, 
  1038   1078     const char *zFilename, 
  1039   1079     int bClear, 
  1040   1080     TestDb **ppDb
  1041   1081   ){

Changes to ext/lsm1/lsmInt.h.

   106    106   typedef unsigned short int u16;
   107    107   typedef unsigned int u32;
   108    108   typedef lsm_i64 i64;
   109    109   typedef unsigned long long int u64;
   110    110   #endif
   111    111   
   112    112   /* A page number is a 64-bit integer. */
   113         -typedef i64 Pgno;
          113  +typedef i64 LsmPgno;
   114    114   
   115    115   #ifdef LSM_DEBUG
   116    116   int lsmErrorBkpt(int);
   117    117   #else
   118    118   # define lsmErrorBkpt(x) (x)
   119    119   #endif
   120    120   
................................................................................
   398    398     void **apShm;                   /* Shared memory chunks */
   399    399     ShmHeader *pShmhdr;             /* Live shared-memory header */
   400    400     TreeHeader treehdr;             /* Local copy of tree-header */
   401    401     u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)];
   402    402   };
   403    403   
   404    404   struct Segment {
   405         -  Pgno iFirst;                     /* First page of this run */
   406         -  Pgno iLastPg;                    /* Last page of this run */
   407         -  Pgno iRoot;                      /* Root page number (if any) */
          405  +  LsmPgno iFirst;                  /* First page of this run */
          406  +  LsmPgno iLastPg;                 /* Last page of this run */
          407  +  LsmPgno iRoot;                   /* Root page number (if any) */
   408    408     int nSize;                       /* Size of this run in pages */
   409    409   
   410    410     Redirect *pRedirect;             /* Block redirects (or NULL) */
   411    411   };
   412    412   
   413    413   /*
   414    414   ** iSplitTopic/pSplitKey/nSplitKey:
................................................................................
   452    452   ** access to the associated Level struct.
   453    453   **
   454    454   ** iOutputOff:
   455    455   **   The byte offset to write to next within the last page of the 
   456    456   **   output segment.
   457    457   */
   458    458   struct MergeInput {
   459         -  Pgno iPg;                       /* Page on which next input is stored */
          459  +  LsmPgno iPg;                    /* Page on which next input is stored */
   460    460     int iCell;                      /* Cell containing next input to merge */
   461    461   };
   462    462   struct Merge {
   463    463     int nInput;                     /* Number of input runs being merged */
   464    464     MergeInput *aInput;             /* Array nInput entries in size */
   465    465     MergeInput splitkey;            /* Location in file of current splitkey */
   466    466     int nSkip;                      /* Number of separators entries to skip */
   467    467     int iOutputOff;                 /* Write offset on output page */
   468         -  Pgno iCurrentPtr;               /* Current pointer value */
          468  +  LsmPgno iCurrentPtr;            /* Current pointer value */
   469    469   };
   470    470   
   471    471   /* 
   472    472   ** The first argument to this macro is a pointer to a Segment structure.
   473    473   ** Returns true if the structure instance indicates that the separators
   474    474   ** array is valid.
   475    475   */
................................................................................
   575    575     u32 iCmpId;                     /* Id of compression scheme */
   576    576     Level *pLevel;                  /* Pointer to level 0 of snapshot (or NULL) */
   577    577     i64 iId;                        /* Snapshot id */
   578    578     i64 iLogOff;                    /* Log file offset */
   579    579     Redirect redirect;              /* Block redirection array */
   580    580   
   581    581     /* Used by worker snapshots only */
   582         -  int nBlock;                     /* Number of blocks in database file */
   583         -  Pgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
   584         -  Freelist freelist;              /* Free block list */
   585         -  u32 nWrite;                     /* Total number of pages written to disk */
          582  +  int nBlock;                        /* Number of blocks in database file */
          583  +  LsmPgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
          584  +  Freelist freelist;                 /* Free block list */
          585  +  u32 nWrite;                        /* Total number of pages written to disk */
   586    586   };
   587    587   #define LSM_INITIAL_SNAPSHOT_ID 11
   588    588   
   589    589   /*
   590    590   ** Functions from file "lsm_ckpt.c".
   591    591   */
   592    592   int lsmCheckpointWrite(lsm_db *, u32 *);
................................................................................
   706    706   
   707    707   int lsmFsPageSize(FileSystem *);
   708    708   void lsmFsSetPageSize(FileSystem *, int);
   709    709   
   710    710   int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId);
   711    711   
   712    712   /* Creating, populating, gobbling and deleting sorted runs. */
   713         -void lsmFsGobble(lsm_db *, Segment *, Pgno *, int);
          713  +void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int);
   714    714   int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *);
   715    715   int lsmFsSortedFinish(FileSystem *, Segment *);
   716    716   int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **);
   717    717   int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *);
   718    718   
   719    719   /* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */
   720    720   lsm_env *lsmFsEnv(FileSystem *);
................................................................................
   723    723   
   724    724   int lsmFsSectorSize(FileSystem *);
   725    725   
   726    726   void lsmSortedSplitkey(lsm_db *, Level *, int *);
   727    727   
   728    728   /* Reading sorted run content. */
   729    729   int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg);
   730         -int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **);
          730  +int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **);
   731    731   int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **);
   732    732   
   733    733   u8 *lsmFsPageData(Page *, int *);
   734    734   int lsmFsPageRelease(Page *);
   735    735   int lsmFsPagePersist(Page *);
   736    736   void lsmFsPageRef(Page *);
   737         -Pgno lsmFsPageNumber(Page *);
          737  +LsmPgno lsmFsPageNumber(Page *);
   738    738   
   739    739   int lsmFsNRead(FileSystem *);
   740    740   int lsmFsNWrite(FileSystem *);
   741    741   
   742    742   int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **);
   743    743   int lsmFsMetaPageRelease(MetaPage *);
   744    744   u8 *lsmFsMetaPageData(MetaPage *, int *);
   745    745   
   746    746   #ifdef LSM_DEBUG
   747    747   int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg);
   748    748   int lsmFsIntegrityCheck(lsm_db *);
   749    749   #endif
   750    750   
   751         -Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);
          751  +LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno);
   752    752   
   753    753   int lsmFsPageWritable(Page *);
   754    754   
   755    755   /* Functions to read, write and sync the log file. */
   756    756   int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr);
   757    757   int lsmFsSyncLog(FileSystem *pFS);
   758    758   int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr);
................................................................................
   764    764   
   765    765   /* And to sync the db file */
   766    766   int lsmFsSyncDb(FileSystem *, int);
   767    767   
   768    768   void lsmFsFlushWaiting(FileSystem *, int *);
   769    769   
   770    770   /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */
   771         -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut);
   772         -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut);
          771  +int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz);
          772  +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut);
   773    773   int lsmConfigMmap(lsm_db *pDb, int *piParam);
   774    774   
   775    775   int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **);
   776    776   int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile);
   777    777   int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock);
   778    778   int lsmEnvTestLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int nLock, int);
   779    779   
................................................................................
   781    781   void lsmEnvShmBarrier(lsm_env *);
   782    782   void lsmEnvShmUnmap(lsm_env *, lsm_file *, int);
   783    783   
   784    784   void lsmEnvSleep(lsm_env *, int);
   785    785   
   786    786   int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);
   787    787   
   788         -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
          788  +int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *);
   789    789   
   790    790   void lsmFsPurgeCache(FileSystem *);
   791    791   
   792    792   /*
   793    793   ** End of functions from "lsm_file.c".
   794    794   **************************************************************************/
   795    795   
   796    796   /* 
   797    797   ** Functions from file "lsm_sorted.c".
   798    798   */
   799         -int lsmInfoPageDump(lsm_db *, Pgno, int, char **);
          799  +int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **);
   800    800   void lsmSortedCleanup(lsm_db *);
   801    801   int lsmSortedAutoWork(lsm_db *, int nUnit);
   802    802   
   803    803   int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *);
   804    804   
   805    805   int lsmSaveWorker(lsm_db *, int);
   806    806   

Changes to ext/lsm1/lsm_ckpt.c.

   385    385   static void ckptExportAppendlist(
   386    386     lsm_db *db,                     /* Database connection */
   387    387     CkptBuffer *p,                  /* Checkpoint buffer to write to */
   388    388     int *piOut,                     /* IN/OUT: Offset within checkpoint buffer */
   389    389     int *pRc                        /* IN/OUT: Error code */
   390    390   ){
   391    391     int i;
   392         -  Pgno *aiAppend = db->pWorker->aiAppend;
          392  +  LsmPgno *aiAppend = db->pWorker->aiAppend;
   393    393   
   394    394     for(i=0; i<LSM_APPLIST_SZ; i++){
   395    395       ckptAppend64(p, piOut, aiAppend[i], pRc);
   396    396     }
   397    397   };
   398    398   
   399    399   static int ckptExportSnapshot( 

Changes to ext/lsm1/lsm_file.c.

   265    265   **   The lsmFsSortedAppend() function sets the pSeg pointer to point to the
   266    266   **   segment that the new page will be a part of. It is unset by
   267    267   **   lsmFsPagePersist() after the page is written to disk.
   268    268   */
   269    269   struct Page {
   270    270     u8 *aData;                      /* Buffer containing page data */
   271    271     int nData;                      /* Bytes of usable data at aData[] */
   272         -  Pgno iPg;                       /* Page number */
          272  +  LsmPgno iPg;                    /* Page number */
   273    273     int nRef;                       /* Number of outstanding references */
   274    274     int flags;                      /* Combination of PAGE_XXX flags */
   275    275     Page *pHashNext;                /* Next page in hash table slot */
   276    276     Page *pLruNext;                 /* Next page in LRU list */
   277    277     Page *pLruPrev;                 /* Previous page in LRU list */
   278    278     FileSystem *pFS;                /* File system that owns this page */
   279    279   
................................................................................
   328    328   #else
   329    329   # define IOERR_WRAPPER(rc) (rc)
   330    330   #endif
   331    331   
   332    332   #ifdef NDEBUG
   333    333   # define assert_lists_are_ok(x)
   334    334   #else
   335         -static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash);
          335  +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash);
   336    336   
   337    337   static void assert_lists_are_ok(FileSystem *pFS){
   338    338   #if 0
   339    339     Page *p;
   340    340   
   341    341     assert( pFS->nMapLimit>=0 );
   342    342   
................................................................................
   528    528     return LSM_OK;
   529    529   }
   530    530   
   531    531   /*
   532    532   ** Return true if page iReal of the database should be accessed using mmap.
   533    533   ** False otherwise.
   534    534   */
   535         -static int fsMmapPage(FileSystem *pFS, Pgno iReal){
          535  +static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){
   536    536     return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit);
   537    537   }
   538    538   
   539    539   /*
   540    540   ** Given that there are currently nHash slots in the hash table, return 
   541    541   ** the hash key for file iFile, page iPg.
   542    542   */
   543         -static int fsHashKey(int nHash, Pgno iPg){
          543  +static int fsHashKey(int nHash, LsmPgno iPg){
   544    544     return (iPg % nHash);
   545    545   }
   546    546   
   547    547   /*
   548    548   ** This is a helper function for lsmFsOpen(). It opens a single file on
   549    549   ** disk (either the database or log file).
   550    550   */
................................................................................
   876    876   ** Return the page number of the first page on block iBlock. Blocks are
   877    877   ** numbered starting from 1.
   878    878   **
   879    879   ** For a compressed database, page numbers are byte offsets. The first
   880    880   ** page on each block is the byte offset immediately following the 4-byte
   881    881   ** "previous block" pointer at the start of each block.
   882    882   */
   883         -static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
   884         -  Pgno iPg;
          883  +static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
          884  +  LsmPgno iPg;
   885    885     if( pFS->pCompress ){
   886    886       if( iBlock==1 ){
   887    887         iPg = pFS->nMetasize * 2 + 4;
   888    888       }else{
   889         -      iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4;
          889  +      iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
   890    890       }
   891    891     }else{
   892    892       const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   893    893       if( iBlock==1 ){
   894    894         iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize);
   895    895       }else{
   896    896         iPg = 1 + (iBlock-1) * nPagePerBlock;
................................................................................
   903    903   ** Return the page number of the last page on block iBlock. Blocks are
   904    904   ** numbered starting from 1.
   905    905   **
   906    906   ** For a compressed database, page numbers are byte offsets. The first
   907    907   ** page on each block is the byte offset of the byte immediately before 
   908    908   ** the 4-byte "next block" pointer at the end of each block.
   909    909   */
   910         -static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
          910  +static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
   911    911     if( pFS->pCompress ){
   912         -    return pFS->nBlocksize * (Pgno)iBlock - 1 - 4;
          912  +    return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4;
   913    913     }else{
   914    914       const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   915    915       return iBlock * nPagePerBlock;
   916    916     }
   917    917   }
   918    918   
   919    919   /*
   920    920   ** Return the block number of the block that page iPg is located on. 
   921    921   ** Blocks are numbered starting from 1.
   922    922   */
   923         -static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
          923  +static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){
   924    924     if( pFS->pCompress ){
   925    925       return (int)((iPg / pFS->nBlocksize) + 1);
   926    926     }else{
   927    927       return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize)));
   928    928     }
   929    929   }
   930    930   
   931    931   /*
   932    932   ** Return true if page iPg is the last page on its block.
   933    933   **
   934    934   ** This function is only called in non-compressed database mode.
   935    935   */
   936         -static int fsIsLast(FileSystem *pFS, Pgno iPg){
          936  +static int fsIsLast(FileSystem *pFS, LsmPgno iPg){
   937    937     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   938    938     assert( !pFS->pCompress );
   939    939     return ( iPg && (iPg % nPagePerBlock)==0 );
   940    940   }
   941    941   
   942    942   /*
   943    943   ** Return true if page iPg is the first page on its block.
   944    944   **
   945    945   ** This function is only called in non-compressed database mode.
   946    946   */
   947         -static int fsIsFirst(FileSystem *pFS, Pgno iPg){
          947  +static int