SQLite: Check-in [36801eff]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Merge latest trunk changes into this branch.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| begin-concurrent
Files:	files \| file ages \| folders
SHA3-256:	36801effa9ec67b551f58972e21794466420f10cd0420701fcd87695e6cd11ee
User & Date:	dan 2018-03-02 17:40:23

Context

2018-03-02
17:59		Update this branch with recent checkpoint related changes from trunk. (check-in: fb6b7938 user: dan tags: begin-concurrent)
17:40		Merge latest trunk changes into this branch. (check-in: 36801eff user: dan tags: begin-concurrent)
2018-03-01
18:09		Typo fixes in comments. No changes to code. (check-in: 1293d4f6 user: mistachkin tags: trunk)
2017-12-11
14:02		Abort on an invalid paramater to sqlite3BitvecSet(). (check-in: 163c8709 user: drh tags: begin-concurrent)

Changes

Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Added .fossil-settings/empty-dirs.

            1  +compat

Added .fossil-settings/ignore-glob.

            1  +compat/*

Changes to Makefile.in.

    84     84   OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
    85     85   
    86     86   TCC += $(OPT_FEATURE_FLAGS)
    87     87   
    88     88   # Add in any optional parameters specified on the make commane line
    89     89   # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
    90     90   TCC += $(OPTS)
           91  +
           92  +# Add in compile-time options for some libraries used by extensions
           93  +TCC += @HAVE_ZLIB@
    91     94   
    92     95   # Version numbers and release number for the SQLite being compiled.
    93     96   #
    94     97   VERSION = @VERSION@
    95     98   VERSION_NUMBER = @VERSION_NUMBER@
    96     99   RELEASE = @RELEASE@
    97    100   
................................................................................
   418    421     $(TOP)/ext/fts3/fts3_test.c  \
   419    422     $(TOP)/ext/session/test_session.c \
   420    423     $(TOP)/ext/rbu/test_rbu.c 
   421    424   
   422    425   # Statically linked extensions
   423    426   #
   424    427   TESTSRC += \
          428  +  $(TOP)/ext/expert/sqlite3expert.c \
          429  +  $(TOP)/ext/expert/test_expert.c \
   425    430     $(TOP)/ext/misc/amatch.c \
   426    431     $(TOP)/ext/misc/carray.c \
   427    432     $(TOP)/ext/misc/closure.c \
   428    433     $(TOP)/ext/misc/csv.c \
   429    434     $(TOP)/ext/misc/eval.c \
   430    435     $(TOP)/ext/misc/fileio.c \
   431    436     $(TOP)/ext/misc/fuzzer.c \
   432    437     $(TOP)/ext/fts5/fts5_tcl.c \
   433    438     $(TOP)/ext/fts5/fts5_test_mi.c \
   434    439     $(TOP)/ext/fts5/fts5_test_tok.c \
   435    440     $(TOP)/ext/misc/ieee754.c \
   436    441     $(TOP)/ext/misc/mmapwarm.c \
   437    442     $(TOP)/ext/misc/nextchar.c \
          443  +  $(TOP)/ext/misc/normalize.c \
   438    444     $(TOP)/ext/misc/percentile.c \
   439    445     $(TOP)/ext/misc/regexp.c \
   440    446     $(TOP)/ext/misc/remember.c \
   441    447     $(TOP)/ext/misc/series.c \
   442    448     $(TOP)/ext/misc/spellfix.c \
   443    449     $(TOP)/ext/misc/totype.c \
   444    450     $(TOP)/ext/misc/unionvtab.c \
   445         -  $(TOP)/ext/misc/wholenumber.c
          451  +  $(TOP)/ext/misc/wholenumber.c \
          452  +  $(TOP)/ext/misc/zipfile.c
   446    453   
   447    454   # Source code to the library files needed by the test fixture
   448    455   #
   449    456   TESTSRC2 = \
   450    457     $(TOP)/src/attach.c \
   451    458     $(TOP)/src/backup.c \
   452    459     $(TOP)/src/bitvec.c \
................................................................................
   549    556   # executables needed for testing
   550    557   #
   551    558   TESTPROGS = \
   552    559     testfixture$(TEXE) \
   553    560     sqlite3$(TEXE) \
   554    561     sqlite3_analyzer$(TEXE) \
   555    562     sqldiff$(TEXE) \
   556         -  dbhash$(TEXE)
          563  +  dbhash$(TEXE) \
          564  +  sqltclsh$(TEXE)
   557    565   
   558    566   # Databases containing fuzzer test cases
   559    567   #
   560    568   FUZZDATA = \
   561    569     $(TOP)/test/fuzzdata1.db \
   562    570     $(TOP)/test/fuzzdata2.db \
   563    571     $(TOP)/test/fuzzdata3.db \
................................................................................
   567    575   # Standard options to testfixture
   568    576   #
   569    577   TESTOPTS = --verbose=file --output=test-out.txt
   570    578   
   571    579   # Extra compiler options for various shell tools
   572    580   #
   573    581   SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
   574         -# SHELL_OPT += -DSQLITE_ENABLE_FTS5
          582  +#SHELL_OPT += -DSQLITE_ENABLE_FTS5
          583  +SHELL_OPT += -DSQLITE_ENABLE_RTREE
   575    584   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   576    585   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   577    586   SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   578    587   SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
   579    588   SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
          589  +SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
          590  +SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
   580    591   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   581    592   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
   582    593   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
          594  +FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
   583    595   FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
   584    596   DBFUZZ_OPT = 
   585    597   
   586    598   # This is the default Makefile target.  The objects listed here
   587    599   # are what get build when you type just "make" with no arguments.
   588    600   #
   589    601   all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
................................................................................
   981    993   keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
   982    994   	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
   983    995   	./mkkeywordhash$(BEXE) >keywordhash.h
   984    996   
   985    997   # Source files that go into making shell.c
   986    998   SHELL_SRC = \
   987    999   	$(TOP)/src/shell.c.in \
         1000  +        $(TOP)/ext/misc/appendvfs.c \
   988   1001   	$(TOP)/ext/misc/shathree.c \
   989   1002   	$(TOP)/ext/misc/fileio.c \
   990         -	$(TOP)/ext/misc/completion.c
         1003  +	$(TOP)/ext/misc/completion.c \
         1004  +	$(TOP)/ext/misc/sqlar.c \
         1005  +	$(TOP)/ext/expert/sqlite3expert.c \
         1006  +	$(TOP)/ext/expert/sqlite3expert.h \
         1007  +	$(TOP)/ext/misc/zipfile.c \
         1008  +        $(TOP)/src/test_windirent.c
   991   1009   
   992   1010   shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
   993   1011   	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
   994   1012   
   995   1013   
   996   1014   
   997   1015   
................................................................................
  1182   1200   	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
  1183   1201   
  1184   1202   sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
  1185   1203   	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
  1186   1204   
  1187   1205   sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
  1188   1206   	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
         1207  +
         1208  +sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
         1209  +	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
         1210  +
         1211  +sqltclsh$(TEXE): sqltclsh.c
         1212  +	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
         1213  +
         1214  +sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
         1215  +	$(LTLINK)	$(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS)
  1189   1216   
  1190   1217   CHECKER_DEPS =\
  1191   1218     $(TOP)/tool/mkccode.tcl \
  1192   1219     sqlite3.c \
  1193   1220     $(TOP)/src/tclsqlite.c \
  1194   1221     $(TOP)/ext/repair/sqlite3_checker.tcl \
  1195   1222     $(TOP)/ext/repair/checkindex.c \

Changes to Makefile.msc.

    88     88   # be used for debugging with Visual Studio.
    89     89   #
    90     90   !IFNDEF SPLIT_AMALGAMATION
    91     91   SPLIT_AMALGAMATION = 0
    92     92   !ENDIF
    93     93   
    94     94   # <<mark>>
           95  +# Set this non-0 to have this makefile assume the Tcl shell executable
           96  +# (tclsh*.exe) is available in the PATH.  By default, this is disabled
           97  +# for compatibility with older build environments.  This setting only
           98  +# applies if TCLSH_CMD is not set manually.
           99  +#
          100  +!IFNDEF USE_TCLSH_IN_PATH
          101  +USE_TCLSH_IN_PATH = 0
          102  +!ENDIF
          103  +
          104  +# Set this non-0 to use zlib, possibly compiling it from source code.
          105  +#
          106  +!IFNDEF USE_ZLIB
          107  +USE_ZLIB = 0
          108  +!ENDIF
          109  +
          110  +# Set this non-0 to build zlib from source code.  This is enabled by
          111  +# default and in that case it will be assumed that the ZLIBDIR macro
          112  +# points to the top-level source code directory for zlib.
          113  +#
          114  +!IFNDEF BUILD_ZLIB
          115  +BUILD_ZLIB = 1
          116  +!ENDIF
          117  +
    95    118   # Set this non-0 to use the International Components for Unicode (ICU).
    96    119   #
    97    120   !IFNDEF USE_ICU
    98    121   USE_ICU = 0
    99    122   !ENDIF
   100    123   # <</mark>>
   101    124   
................................................................................
   608    631   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   609    632   SHELL_CORE_DEP = $(SQLITE3DLL)
   610    633   !ELSE
   611    634   SHELL_CORE_DEP =
   612    635   !ENDIF
   613    636   !ENDIF
   614    637   
          638  +# <<mark>>
          639  +# If zlib support is enabled, add the dependencies for it.
          640  +#
          641  +!IF $(USE_ZLIB)!=0 && $(BUILD_ZLIB)!=0
          642  +SHELL_CORE_DEP = zlib $(SHELL_CORE_DEP)
          643  +TESTFIXTURE_DEP = zlib $(TESTFIXTURE_DEP)
          644  +!ENDIF
          645  +# <</mark>>
          646  +
   615    647   # This is the core library that the shell executable should link with.
   616    648   #
   617    649   !IFNDEF SHELL_CORE_LIB
   618    650   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   619    651   SHELL_CORE_LIB = $(SQLITE3LIB)
   620    652   !ELSE
   621    653   SHELL_CORE_LIB =
................................................................................
   798    830   # <<mark>>
   799    831   # The locations of the Tcl header and library files.  Also, the library that
   800    832   # non-stubs enabled programs using Tcl must link against.  These variables
   801    833   # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment
   802    834   # prior to running nmake in order to match the actual installed location and
   803    835   # version on this machine.
   804    836   #
          837  +!IFNDEF TCLDIR
          838  +TCLDIR = $(TOP)\compat\tcl
          839  +!ENDIF
          840  +
   805    841   !IFNDEF TCLINCDIR
   806         -TCLINCDIR = c:\tcl\include
          842  +TCLINCDIR = $(TCLDIR)\include
   807    843   !ENDIF
   808    844   
   809    845   !IFNDEF TCLLIBDIR
   810         -TCLLIBDIR = c:\tcl\lib
          846  +TCLLIBDIR = $(TCLDIR)\lib
   811    847   !ENDIF
   812    848   
   813    849   !IFNDEF LIBTCL
   814    850   LIBTCL = tcl86.lib
   815    851   !ENDIF
   816    852   
   817    853   !IFNDEF LIBTCLSTUB
   818    854   LIBTCLSTUB = tclstub86.lib
   819    855   !ENDIF
   820    856   
   821    857   !IFNDEF LIBTCLPATH
   822         -LIBTCLPATH = c:\tcl\bin
          858  +LIBTCLPATH = $(TCLDIR)\bin
          859  +!ENDIF
          860  +
          861  +# The locations of the zlib header and library files.  These variables
          862  +# (ZLIBINCDIR, ZLIBLIBDIR, and ZLIBLIB) may be overridden via the environment
          863  +# prior to running nmake in order to match the actual installed (or source
          864  +# code) location on this machine.
          865  +#
          866  +!IFNDEF ZLIBDIR
          867  +ZLIBDIR = $(TOP)\compat\zlib
          868  +!ENDIF
          869  +
          870  +!IFNDEF ZLIBINCDIR
          871  +ZLIBINCDIR = $(ZLIBDIR)
          872  +!ENDIF
          873  +
          874  +!IFNDEF ZLIBLIBDIR
          875  +ZLIBLIBDIR = $(ZLIBDIR)
          876  +!ENDIF
          877  +
          878  +!IFNDEF ZLIBLIB
          879  +!IF $(DYNAMIC_SHELL)!=0
          880  +ZLIBLIB = zdll.lib
          881  +!ELSE
          882  +ZLIBLIB = zlib.lib
          883  +!ENDIF
   823    884   !ENDIF
   824    885   
   825    886   # The locations of the ICU header and library files.  These variables
   826    887   # (ICUINCDIR, ICULIBDIR, and LIBICU) may be overridden via the environment
   827    888   # prior to running nmake in order to match the actual installed location on
   828    889   # this machine.
   829    890   #
          891  +!IFNDEF ICUDIR
          892  +ICUDIR = $(TOP)\compat\icu
          893  +!ENDIF
          894  +
   830    895   !IFNDEF ICUINCDIR
   831         -ICUINCDIR = c:\icu\include
          896  +ICUINCDIR = $(ICUDIR)\include
   832    897   !ENDIF
   833    898   
   834    899   !IFNDEF ICULIBDIR
   835         -ICULIBDIR = c:\icu\lib
          900  +ICULIBDIR = $(ICUDIR)\lib
   836    901   !ENDIF
   837    902   
   838    903   !IFNDEF LIBICU
   839    904   LIBICU = icuuc.lib icuin.lib
   840    905   !ENDIF
   841    906   
   842    907   # This is the command to use for tclsh - normally just "tclsh", but we may
   843    908   # know the specific version we want to use.  This variable (TCLSH_CMD) may be
   844    909   # overridden via the environment prior to running nmake in order to select a
   845    910   # specific Tcl shell to use.
   846    911   #
   847    912   !IFNDEF TCLSH_CMD
          913  +!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe")
   848    914   TCLSH_CMD = tclsh
          915  +!ELSE
          916  +TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
          917  +!ENDIF
   849    918   !ENDIF
   850    919   # <</mark>>
   851    920   
   852    921   # Compiler options needed for programs that use the readline() library.
   853    922   #
   854    923   !IFNDEF READLINE_FLAGS
   855    924   READLINE_FLAGS = -DHAVE_READLINE=0
................................................................................
   947   1016   #
   948   1017   !IF $(DEBUG)>1 || $(SYMBOLS)!=0
   949   1018   TCC = $(TCC) -Zi
   950   1019   BCC = $(BCC) -Zi
   951   1020   !ENDIF
   952   1021   
   953   1022   # <<mark>>
         1023  +# If zlib support is enabled, add the compiler options for it.
         1024  +#
         1025  +!IF $(USE_ZLIB)!=0
         1026  +TCC = $(TCC) -DSQLITE_HAVE_ZLIB=1
         1027  +RCC = $(RCC) -DSQLITE_HAVE_ZLIB=1
         1028  +TCC = $(TCC) -I$(ZLIBINCDIR)
         1029  +RCC = $(RCC) -I$(ZLIBINCDIR)
         1030  +!ENDIF
         1031  +
   954   1032   # If ICU support is enabled, add the compiler options for it.
   955   1033   #
   956   1034   !IF $(USE_ICU)!=0
   957   1035   TCC = $(TCC) -DSQLITE_ENABLE_ICU=1
   958   1036   RCC = $(RCC) -DSQLITE_ENABLE_ICU=1
   959   1037   TCC = $(TCC) -I$(TOP)\ext\icu
   960   1038   RCC = $(RCC) -I$(TOP)\ext\icu
................................................................................
   970   1048   LTRCOMPILE = $(RCC) -r
   971   1049   LTLIB = lib.exe
   972   1050   LTLINK = $(TCC) -Fe$@
   973   1051   
   974   1052   # If requested, link to the RPCRT4 library.
   975   1053   #
   976   1054   !IF $(USE_RPCRT4_LIB)!=0
   977         -LTLINK = $(LTLINK) rpcrt4.lib
         1055  +LTLIBS = $(LTLIBS) rpcrt4.lib
   978   1056   !ENDIF
   979   1057   
   980   1058   # If a platform was set, force the linker to target that.
   981   1059   # Note that the vcvars*.bat family of batch files typically
   982   1060   # set this for you.  Otherwise, the linker will attempt
   983   1061   # to deduce the binary type based on the object files.
   984   1062   !IFDEF PLATFORM
................................................................................
  1067   1145   LDFLAGS = $(LDOPTS)
  1068   1146   !ENDIF
  1069   1147   
  1070   1148   # <<mark>>
  1071   1149   # Start with the Tcl related linker options.
  1072   1150   #
  1073   1151   !IF $(NO_TCL)==0
  1074         -LTLIBPATHS = /LIBPATH:$(TCLLIBDIR)
  1075         -LTLIBS = $(LIBTCL)
         1152  +TCLLIBPATHS = $(TCLLIBPATHS) /LIBPATH:$(TCLLIBDIR)
         1153  +TCLLIBS = $(TCLLIBS) $(LIBTCL)
         1154  +!ENDIF
         1155  +
         1156  +# If zlib support is enabled, add the linker options for it.
         1157  +#
         1158  +!IF $(USE_ZLIB)!=0
         1159  +LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ZLIBLIBDIR)
         1160  +LTLIBS = $(LTLIBS) $(ZLIBLIB)
  1076   1161   !ENDIF
  1077   1162   
  1078   1163   # If ICU support is enabled, add the linker options for it.
  1079   1164   #
  1080   1165   !IF $(USE_ICU)!=0
  1081   1166   LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR)
  1082   1167   LTLIBS = $(LTLIBS) $(LIBICU)
................................................................................
  1396   1481     $(TOP)\ext\fts3\fts3_test.c \
  1397   1482     $(TOP)\ext\rbu\test_rbu.c \
  1398   1483     $(TOP)\ext\session\test_session.c
  1399   1484   
  1400   1485   # Statically linked extensions.
  1401   1486   #
  1402   1487   TESTEXT = \
         1488  +  $(TOP)\ext\expert\sqlite3expert.c \
         1489  +  $(TOP)\ext\expert\test_expert.c \
  1403   1490     $(TOP)\ext\misc\amatch.c \
  1404   1491     $(TOP)\ext\misc\carray.c \
  1405   1492     $(TOP)\ext\misc\closure.c \
  1406   1493     $(TOP)\ext\misc\csv.c \
  1407   1494     $(TOP)\ext\misc\eval.c \
  1408   1495     $(TOP)\ext\misc\fileio.c \
  1409   1496     $(TOP)\ext\misc\fuzzer.c \
  1410   1497     $(TOP)\ext\fts5\fts5_tcl.c \
  1411   1498     $(TOP)\ext\fts5\fts5_test_mi.c \
  1412   1499     $(TOP)\ext\fts5\fts5_test_tok.c \
  1413   1500     $(TOP)\ext\misc\ieee754.c \
  1414   1501     $(TOP)\ext\misc\mmapwarm.c \
  1415   1502     $(TOP)\ext\misc\nextchar.c \
         1503  +  $(TOP)\ext\misc\normalize.c \
  1416   1504     $(TOP)\ext\misc\percentile.c \
  1417   1505     $(TOP)\ext\misc\regexp.c \
  1418   1506     $(TOP)\ext\misc\remember.c \
  1419   1507     $(TOP)\ext\misc\series.c \
  1420   1508     $(TOP)\ext\misc\spellfix.c \
  1421   1509     $(TOP)\ext\misc\totype.c \
  1422   1510     $(TOP)\ext\misc\unionvtab.c \
  1423   1511     $(TOP)\ext\misc\wholenumber.c
         1512  +
         1513  +# If use of zlib is enabled, add the "zipfile.c" source file.
         1514  +#
         1515  +!IF $(USE_ZLIB)!=0
         1516  +TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
         1517  +!ENDIF
  1424   1518   
  1425   1519   # Source code to the library files needed by the test fixture
  1426   1520   # (non-amalgamation)
  1427   1521   #
  1428   1522   TESTSRC2 = \
  1429   1523     $(SRC00) \
  1430   1524     $(SRC01) \
................................................................................
  1489   1583   #
  1490   1584   TESTPROGS = \
  1491   1585     testfixture.exe \
  1492   1586     $(SQLITE3EXE) \
  1493   1587     sqlite3_analyzer.exe \
  1494   1588     sqlite3_checker.exe \
  1495   1589     sqldiff.exe \
  1496         -  dbhash.exe
         1590  +  dbhash.exe \
         1591  +  sqltclsh.exe
  1497   1592   
  1498   1593   # Databases containing fuzzer test cases
  1499   1594   #
  1500   1595   FUZZDATA = \
  1501   1596     $(TOP)\test\fuzzdata1.db \
  1502   1597     $(TOP)\test\fuzzdata2.db \
  1503   1598     $(TOP)\test\fuzzdata3.db \
................................................................................
  1507   1602   
  1508   1603   # Additional compiler options for the shell.  These are only effective
  1509   1604   # when the shell is not being dynamically linked.
  1510   1605   #
  1511   1606   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1512   1607   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
  1513   1608   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
         1609  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
         1610  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE
  1514   1611   !ENDIF
  1515   1612   
  1516   1613   # <<mark>>
  1517   1614   # Extra compiler options for various test tools.
  1518   1615   #
  1519   1616   MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
  1520   1617   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
  1521         -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
         1618  +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
  1522   1619   FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
  1523   1620   OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
  1524   1621   DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
  1525   1622   KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
  1526   1623   DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
  1527   1624   ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0
  1528   1625   
................................................................................
  1538   1635   ALL_TCL_TARGETS =
  1539   1636   !ENDIF
  1540   1637   # <</mark>>
  1541   1638   
  1542   1639   # This is the default Makefile target.  The objects listed here
  1543   1640   # are what get build when you type just "make" with no arguments.
  1544   1641   #
  1545         -all:	dll libsqlite3.lib shell $(ALL_TCL_TARGETS)
         1642  +core:	dll libsqlite3.lib shell
         1643  +
         1644  +# Targets that require the Tcl library.
         1645  +#
         1646  +tcl:	$(ALL_TCL_TARGETS)
         1647  +
         1648  +# This Makefile target builds all of the standard binaries.
         1649  +#
         1650  +all:	core tcl
  1546   1651   
  1547   1652   # Dynamic link library section.
  1548   1653   #
  1549   1654   dll:	$(SQLITE3DLL)
  1550   1655   
  1551   1656   # Shell executable.
  1552   1657   #
................................................................................
  1933   2038   tclsqlite.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1934   2039   	$(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1935   2040   
  1936   2041   tclsqlite-shell.lo:	$(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP)
  1937   2042   	$(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c
  1938   2043   
  1939   2044   tclsqlite3.exe:	tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
  1940         -	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2045  +	$(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  1941   2046   
  1942   2047   # Rules to build opcodes.c and opcodes.h
  1943   2048   #
  1944   2049   opcodes.c:	opcodes.h $(TOP)\tool\mkopcodec.tcl
  1945   2050   	$(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c
  1946   2051   
  1947   2052   opcodes.h:	parse.h $(TOP)\src\vdbe.c $(TOP)\tool\mkopcodeh.tcl
................................................................................
  1976   2081   
  1977   2082   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
  1978   2083   	.\mkkeywordhash.exe > keywordhash.h
  1979   2084   
  1980   2085   # Source files that go into making shell.c
  1981   2086   SHELL_SRC = \
  1982   2087   	$(TOP)\src\shell.c.in \
         2088  +        $(TOP)\ext\misc\appendvfs.c \
  1983   2089   	$(TOP)\ext\misc\shathree.c \
  1984   2090   	$(TOP)\ext\misc\fileio.c \
  1985         -	$(TOP)\ext\misc\completion.c
         2091  +	$(TOP)\ext\misc\completion.c \
         2092  +	$(TOP)\ext\expert\sqlite3expert.c \
         2093  +	$(TOP)\ext\expert\sqlite3expert.h \
         2094  +	$(TOP)\src\test_windirent.c
         2095  +
         2096  +# If use of zlib is enabled, add the "zipfile.c" source file.
         2097  +#
         2098  +!IF $(USE_ZLIB)!=0
         2099  +SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
         2100  +SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
         2101  +!ENDIF
  1986   2102   
  1987   2103   shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
  1988   2104   	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
  1989   2105   
         2106  +zlib:
         2107  +	pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd
  1990   2108   
  1991   2109   # Rules to build the extension objects.
  1992   2110   #
  1993   2111   icu.lo:	$(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
  1994   2112   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c
  1995   2113   
  1996   2114   fts2.lo:	$(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR)
................................................................................
  2071   2189      fts5parse.c fts5parse.h \
  2072   2190      $(TOP)\ext\fts5\fts5_storage.c \
  2073   2191      $(TOP)\ext\fts5\fts5_tokenize.c \
  2074   2192      $(TOP)\ext\fts5\fts5_unicode2.c \
  2075   2193      $(TOP)\ext\fts5\fts5_varint.c \
  2076   2194      $(TOP)\ext\fts5\fts5_vocab.c
  2077   2195   
         2196  +LSM1_SRC = \
         2197  +   $(TOP)\ext\lsm1\lsm.h \
         2198  +   $(TOP)\ext\lsm1\lsmInt.h \
         2199  +   $(TOP)\ext\lsm1\lsm_ckpt.c \
         2200  +   $(TOP)\ext\lsm1\lsm_file.c \
         2201  +   $(TOP)\ext\lsm1\lsm_log.c \
         2202  +   $(TOP)\ext\lsm1\lsm_main.c \
         2203  +   $(TOP)\ext\lsm1\lsm_mem.c \
         2204  +   $(TOP)\ext\lsm1\lsm_mutex.c \
         2205  +   $(TOP)\ext\lsm1\lsm_shared.c \
         2206  +   $(TOP)\ext\lsm1\lsm_sorted.c \
         2207  +   $(TOP)\ext\lsm1\lsm_str.c \
         2208  +   $(TOP)\ext\lsm1\lsm_tree.c \
         2209  +   $(TOP)\ext\lsm1\lsm_unix.c \
         2210  +   $(TOP)\ext\lsm1\lsm_varint.c \
         2211  +   $(TOP)\ext\lsm1\lsm_vtab.c \
         2212  +   $(TOP)\ext\lsm1\lsm_win32.c
         2213  +
  2078   2214   fts5parse.c:	$(TOP)\ext\fts5\fts5parse.y lemon.exe
  2079   2215   	copy $(TOP)\ext\fts5\fts5parse.y .
  2080   2216   	del /Q fts5parse.h 2>NUL
  2081   2217   	.\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y
  2082   2218   
  2083   2219   fts5parse.h:	fts5parse.c
  2084   2220   
  2085   2221   fts5.c:	$(FTS5_SRC)
  2086   2222   	$(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl
  2087   2223   	copy $(TOP)\ext\fts5\fts5.h .
  2088   2224   
         2225  +lsm1.c:	$(LSM1_SRC)
         2226  +	$(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl
         2227  +	copy $(TOP)\ext\lsm1\lsm.h .
         2228  +
  2089   2229   fts5.lo:	fts5.c $(HDR) $(EXTHDR)
  2090   2230   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c
  2091   2231   
  2092   2232   fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
  2093   2233   	$(LTCOMPILE) $(NO_WARN) -c fts5.c
  2094   2234   
  2095   2235   fts5.dll:	fts5_ext.lo
................................................................................
  2108   2248   TESTFIXTURE_FLAGS = -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
  2109   2249   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE=""
  2110   2250   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN)
  2111   2251   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
  2112   2252   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
  2113   2253   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB
  2114   2254   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB
         2255  +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1
  2115   2256   TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
  2116   2257   
  2117   2258   TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
  2118   2259   TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
  2119   2260   !IF $(USE_AMALGAMATION)==0
  2120   2261   TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0)
  2121   2262   !ELSE
................................................................................
  2138   2279   	type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \
  2139   2280   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \
  2140   2281   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \
  2141   2282   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \
  2142   2283   		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH)
  2143   2284   !ENDIF
  2144   2285   
  2145         -testfixture.exe:	$(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
         2286  +testfixture.exe:	$(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP)
  2146   2287   	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \
  2147   2288   		-DBUILD_sqlite -I$(TCLINCDIR) \
  2148   2289   		$(TESTFIXTURE_SRC) \
  2149         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2290  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  2150   2291   
  2151   2292   extensiontest:	testfixture.exe testloadext.dll
  2152   2293   	@set PATH=$(LIBTCLPATH);$(PATH)
  2153   2294   	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
  2154   2295   
  2155   2296   fulltest:	$(TESTPROGS) fuzztest
  2156   2297   	@set PATH=$(LIBTCLPATH);$(PATH)
................................................................................
  2192   2333   	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
  2193   2334   
  2194   2335   sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
  2195   2336   	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
  2196   2337   
  2197   2338   sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
  2198   2339   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \
  2199         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2340  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
         2341  +
         2342  +sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in
         2343  +	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c
         2344  +
         2345  +sqltclsh.exe: sqltclsh.c  $(SHELL_CORE_DEP) $(LIBRESOBJS)
         2346  +	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqltclsh.c \
         2347  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
         2348  +
         2349  +sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c
         2350  +	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
  2200   2351   
  2201   2352   CHECKER_DEPS =\
  2202   2353     $(TOP)/tool/mkccode.tcl \
  2203   2354     sqlite3.c \
  2204   2355     $(TOP)/src/tclsqlite.c \
  2205   2356     $(TOP)/ext/repair/sqlite3_checker.tcl \
  2206   2357     $(TOP)/ext/repair/checkindex.c \
................................................................................
  2209   2360     $(TOP)/ext/repair/sqlite3_checker.c.in
  2210   2361   
  2211   2362   sqlite3_checker.c:	$(CHECKER_DEPS)
  2212   2363   	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
  2213   2364   
  2214   2365   sqlite3_checker.exe:	sqlite3_checker.c $(LIBRESOBJS)
  2215   2366   	$(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \
  2216         -		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
         2367  +		/link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS)
  2217   2368   
  2218   2369   dbdump.exe:	$(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H)
  2219   2370   	$(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \
  2220   2371   		/link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS)
  2221   2372   
  2222   2373   testloadext.lo:	$(TOP)\src\test_loadext.c
  2223   2374   	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c
................................................................................
  2310   2461   	del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL
  2311   2462   	del /Q sqlite3.c sqlite3-*.c 2>NUL
  2312   2463   	del /Q sqlite3rc.h 2>NUL
  2313   2464   	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
  2314   2465   	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
  2315   2466   	del /Q sqlite-*-output.vsix 2>NUL
  2316   2467   	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
         2468  +	del /Q sqltclsh.exe 2>NUL
  2317   2469   	del /Q fts5.* fts5parse.* 2>NUL
         2470  +	del /Q lsm.h lsm1.c 2>NUL
  2318   2471   # <</mark>>

Changes to README.md.

      2      3      4       5      6      7      8      9     10     11     12     13     14     15     16     17      18     19     20     21         22     23     24     25         26     27     28     29     30     31     32 ......    101    102    103    104    105    106    107    108    109    110    111    112    113    114    115    116    117    118    119     120    121    122    123    124    125    126 ......    160    161    162    163    164    165    166    167      168    169    170    171    172    173      174    175    176    177    178    179    180 ......    204    205    206    207    208    209    210    211      212    213    214    215    216    217    218 ......    234    235    236    237    238    239    240      241    242    243    244    245    246    247    248    249    250    251    252      253    254    255    256    257    258       259    260    261      262    263    264    265    266    267     268    269      270    271    272    273              274    275    276    277    278    279    280    281    282    283    284

class="diffln"> 1 1 <h1 align="center">SQLite Source Repository</h1> 2 3 This repository contains the complete source code for the SQLite database -engine. Some test scripts are also include. However, many other test scripts 4 +engine. Some test scripts are also included. However, many other test scripts 5 and most of the documentation are managed separately. 6 7 If you are reading this on a Git mirror someplace, you are doing it wrong. 8 The [official repository](https://www.sqlite.org/src/) is better. Go there 9 now. 10 11 ## Obtaining The Code 12 13 SQLite sources are managed using the 14 [Fossil](https://www.fossil-scm.org/), a distributed version control system 15 that was specifically designed to support SQLite development. 16 If you do not want to use Fossil, you can download tarballs or ZIP -archives as follows: 17 +archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: 18 - * Lastest trunk check-in: - <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or - <https://www.sqlite.org/src/zip/sqlite.zip>. 19 + * Lastest trunk check-in as 20 + [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz), 21 + [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or 22 + [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar). 23 - * Latest release: - <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or - <https://www.sqlite.org/src/zip/sqlite.zip?r=release>. 24 + * Latest release as 25 + [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release), 26 + [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or 27 + [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release). 28 29 * For other check-ins, substitute an appropriate branch name or 30 tag or hash prefix for "release" in the URLs of the previous 31 bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline) 32 to locate the check-in desired, click on its information page link, 33 then click on the "Tarball" or "ZIP Archive" links on the information 34 page. .......................................................................... class="diffln"> 100 102 to the "sqlite3.dll" command line above. When debugging into the SQLite 103 code, adding the "DEBUG=1" argument to one of the above command lines is 104 recommended. 105 106 SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation 107 is required by the makefiles (including those for MSVC). SQLite contains 108 a lot of generated code and Tcl is used to do much of that code generation. -The makefiles also require AWK. 109 110 ## Source Code Tour 111 112 Most of the core source files are in the **src/** subdirectory. The 113 **src/** folder also contains files used to build the "testfixture" test 114 harness. The names of the source files used by "testfixture" all begin 115 with "test". 116 The **src/** also contains the "shell.c" file 117 which is the main program for the "sqlite3.exe" 118 [command-line shell](https://sqlite.org/cli.html) and 119 the "tclsqlite.c" file which implements the -[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite. 120 +[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite. 121 (Historical note: SQLite began as a Tcl 122 extension and only later escaped to the wild as an independent library.) 123 124 Test scripts and programs are found in the **test/** subdirectory. 125 Addtional test code is found in other source repositories. 126 See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for 127 additional information. .......................................................................... class="diffln"> 159 160 the src/parse.y file. The conversion of "parse.y" into "parse.c" is done 161 by the [lemon](./doc/lemon.html) LALR(1) parser generator. The source code 162 for lemon is at tool/lemon.c. Lemon uses the tool/lempar.c file as a 163 template for generating its parser. 164 165 Lemon also generates the **parse.h** header file, at the same time it 166 generates parse.c. But the parse.h header file is -modified further (to add additional symbols) using the ./addopcodes.awk -AWK script. 167 +modified further (to add additional symbols) using the ./addopcodes.tcl 168 +Tcl script. 169 170 The **opcodes.h** header file contains macros that define the numbers 171 corresponding to opcodes in the "VDBE" virtual machine. The opcodes.h 172 file is generated by the scanning the src/vdbe.c source file. The -AWK script at ./mkopcodeh.awk does this scan and generates opcodes.h. -A second AWK script, ./mkopcodec.awk, then scans opcodes.h to generate 173 +Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h. 174 +A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate 175 the **opcodes.c** source file, which contains a reverse mapping from 176 opcode-number to opcode-name that is used for EXPLAIN output. 177 178 The **keywordhash.h** header file contains the definition of a hash table 179 that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into 180 the numeric codes used by the parse.c parser. The keywordhash.h file is 181 generated by a C-language program at tool mkkeywordhash.c. .......................................................................... class="diffln"> 203 204 tool/mksqlite3c.tcl script is run to copy them all together in just the 205 right order while resolving internal "#include" references. 206 207 The amalgamation source file is more than 200K lines long. Some symbolic 208 debuggers (most notably MSVC) are unable to deal with files longer than 64K 209 lines. To work around this, a separate Tcl script, tool/split-sqlite3c.tcl, 210 can be run on the amalgamation to break it up into a single small C file -called **sqlite3-all.c** that does #include on about five other files -named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**. In this way, 211 +called **sqlite3-all.c** that does #include on about seven other files 212 +named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**. In this way, 213 all of the source code is contained within a single translation unit so 214 that the compiler can do extra cross-procedure optimization, but no 215 individual source file exceeds 32K lines in length. 216 217 ## How It All Fits Together 218 219 SQLite is modular in design. .......................................................................... class="diffln"> 233 234 Key files: 235 236 * **sqlite.h.in** - This file defines the public interface to the SQLite 237 library. Readers will need to be familiar with this interface before 238 trying to understand how the library works internally. 239 240 * **sqliteInt.h** - this header file defines many of the data objects - used internally by SQLite. 241 + used internally by SQLite. In addition to "sqliteInt.h", some 242 + subsystems have their own header files. 243 244 * **parse.y** - This file describes the LALR(1) grammar that SQLite uses 245 to parse SQL statements, and the actions that are taken at each step 246 in the parsing process. 247 248 * **vdbe.c** - This file implements the virtual machine that runs 249 prepared statements. There are various helper files whose names 250 begin with "vdbe". The VDBE has access to the vdbeInt.h header file 251 which defines internal data objects. The rest of SQLite interacts 252 with the VDBE through an interface defined by vdbe.h. 253 - * **where.c** - This file analyzes the WHERE clause and generates 254 + * **where.c** - This file (together with its helper files named 255 + by "where*.c") analyzes the WHERE clause and generates 256 virtual machine code to run queries efficiently. This file is 257 sometimes called the "query optimizer". It has its own private 258 header file, whereInt.h, that defines data objects used internally. 259 260 * **btree.c** - This file contains the implementation of the B-Tree - storage engine used by SQLite. 261 + storage engine used by SQLite. The interface to the rest of the system 262 + is defined by "btree.h". The "btreeInt.h" header defines objects 263 + used internally by btree.c and not published to the rest of the system. 264 265 * **pager.c** - This file contains the "pager" implementation, the - module that implements transactions. 266 + module that implements transactions. The "pager.h" header file 267 + defines the interface between pager.c and the rest of the system. 268 269 * **os_unix.c** and **os_win.c** - These two files implement the interface 270 between SQLite and the underlying operating system using the run-time 271 pluggable VFS interface. 272 - * **shell.c** - This file is not part of the core SQLite library. This 273 + * **shell.c.in** - This file is not part of the core SQLite library. This 274 is the file that, when linked against sqlite3.a, generates the - "sqlite3.exe" command-line shell. 275 + "sqlite3.exe" command-line shell. The "shell.c.in" file is transformed 276 + into "shell.c" as part of the build process. 277 278 * **tclsqlite.c** - This file implements the Tcl bindings for SQLite. It 279 is not part of the core SQLite library. But as most of the tests in this 280 repository are written in Tcl, the Tcl language bindings are important. 281 + 282 + * **test*.c** - Files in the src/ folder that begin with "test" go into 283 + building the "testfixture.exe" program. The testfixture.exe program is 284 + an enhanced Tcl shell. The testfixture.exe program runs scripts in the 285 + test/ folder to validate the core SQLite code. The testfixture program 286 + (and some other test programs too) is build and run when you type 287 + "make test". 288 + 289 + * **ext/misc/json1.c** - This file implements the various JSON functions 290 + that are build into SQLite. 291 292 There are many other source files. Each has a succinct header comment that 293 describes its purpose and role within the larger system. 294 295 296 ## Contacts 297 298 The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/) 299 with geographically distributed backups at 300 [http://www2.sqlite.org/](http://www2.sqlite.org) and 301 [http://www3.sqlite.org/](http://www3.sqlite.org).

Changes to VERSION.

     1         -3.22.0
            1  +3.23.0

Changes to autoconf/Makefile.am.

     1      1   
     2         -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
            2  +AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @ZLIB_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
     3      3   
     4      4   lib_LTLIBRARIES = libsqlite3.la
     5      5   libsqlite3_la_SOURCES = sqlite3.c
     6      6   libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
     7      7   
     8      8   bin_PROGRAMS = sqlite3
     9      9   sqlite3_SOURCES = shell.c sqlite3.h
    10     10   EXTRA_sqlite3_SOURCES = sqlite3.c
    11     11   sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
    12     12   sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
    13         -sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB
           13  +sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
    14     14   
    15     15   include_HEADERS = sqlite3.h sqlite3ext.h
    16     16   
    17     17   EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
    18     18   pkgconfigdir = ${libdir}/pkgconfig
    19     19   pkgconfig_DATA = sqlite3.pc
    20     20   
    21     21   man_MANS = sqlite3.1

Changes to autoconf/Makefile.msc.

   556    556   !IFNDEF SHELL_CORE_DEP
   557    557   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   558    558   SHELL_CORE_DEP = $(SQLITE3DLL)
   559    559   !ELSE
   560    560   SHELL_CORE_DEP =
   561    561   !ENDIF
   562    562   !ENDIF
          563  +
   563    564   
   564    565   # This is the core library that the shell executable should link with.
   565    566   #
   566    567   !IFNDEF SHELL_CORE_LIB
   567    568   !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
   568    569   SHELL_CORE_LIB = $(SQLITE3LIB)
   569    570   !ELSE
................................................................................
   804    805   LTRCOMPILE = $(RCC) -r
   805    806   LTLIB = lib.exe
   806    807   LTLINK = $(TCC) -Fe$@
   807    808   
   808    809   # If requested, link to the RPCRT4 library.
   809    810   #
   810    811   !IF $(USE_RPCRT4_LIB)!=0
   811         -LTLINK = $(LTLINK) rpcrt4.lib
          812  +LTLIBS = $(LTLIBS) rpcrt4.lib
   812    813   !ENDIF
   813    814   
   814    815   # If a platform was set, force the linker to target that.
   815    816   # Note that the vcvars*.bat family of batch files typically
   816    817   # set this for you.  Otherwise, the linker will attempt
   817    818   # to deduce the binary type based on the object files.
   818    819   !IFDEF PLATFORM
................................................................................
   925    926   
   926    927   # Additional compiler options for the shell.  These are only effective
   927    928   # when the shell is not being dynamically linked.
   928    929   #
   929    930   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   930    931   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB
   931    932   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_DBSTAT_VTAB
          933  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC -DSQLITE_INTROSPECTION_PRAGMAS
   932    934   !ENDIF
   933    935   
   934    936   
   935    937   # This is the default Makefile target.  The objects listed here
   936    938   # are what get build when you type just "make" with no arguments.
   937    939   #
   938         -all:	dll shell
          940  +core:	dll shell
          941  +
          942  +# Targets that require the Tcl library.
          943  +#
          944  +tcl:	$(ALL_TCL_TARGETS)
          945  +
          946  +# This Makefile target builds all of the standard binaries.
          947  +#
          948  +all:	core tcl
   939    949   
   940    950   # Dynamic link library section.
   941    951   #
   942    952   dll:	$(SQLITE3DLL)
   943    953   
   944    954   # Shell executable.
   945    955   #

Changes to autoconf/configure.ac.

   160    160   else
   161    161     EXTRA_SHELL_OBJ=libsqlite3.la
   162    162   fi
   163    163   AC_SUBST(EXTRA_SHELL_OBJ)
   164    164   #-----------------------------------------------------------------------
   165    165   
   166    166   AC_CHECK_FUNCS(posix_fallocate)
          167  +AC_CHECK_HEADERS(zlib.h,[
          168  +  AC_SEARCH_LIBS(deflate,z,[ZLIB_FLAGS="-DSQLITE_HAVE_ZLIB"])
          169  +])
          170  +AC_SUBST(ZLIB_FLAGS)
          171  +
          172  +AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"])
          173  +AC_SUBST(SHELL_CFLAGS)
   167    174   
   168    175   #-----------------------------------------------------------------------
   169    176   # UPDATE: Maybe it's better if users just set CFLAGS before invoking
   170    177   # configure. This option doesn't really add much...
   171    178   #
   172    179   #   --enable-tempstore
   173    180   #

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.22.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.23.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.22.0'
   730         -PACKAGE_STRING='sqlite 3.22.0'
          729  +PACKAGE_VERSION='3.23.0'
          730  +PACKAGE_STRING='sqlite 3.23.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
   768    768   #endif"
   769    769   
   770    770   ac_subst_vars='LTLIBOBJS
   771    771   LIBOBJS
   772    772   BUILD_CFLAGS
   773    773   USE_GCOV
   774    774   OPT_FEATURE_FLAGS
          775  +HAVE_ZLIB
   775    776   USE_AMALGAMATION
   776    777   TARGET_DEBUG
   777    778   TARGET_HAVE_EDITLINE
   778    779   TARGET_HAVE_READLINE
   779    780   TARGET_READLINE_INC
   780    781   TARGET_READLINE_LIBS
   781    782   HAVE_TCL
................................................................................
  1460   1461   #
  1461   1462   # Report the --help message.
  1462   1463   #
  1463   1464   if test "$ac_init_help" = "long"; then
  1464   1465     # Omit some internal or obsolete options to make the list less imposing.
  1465   1466     # This message is too long to be a string in the A/UX 3.1 sh.
  1466   1467     cat <<_ACEOF
  1467         -\`configure' configures sqlite 3.22.0 to adapt to many kinds of systems.
         1468  +\`configure' configures sqlite 3.23.0 to adapt to many kinds of systems.
  1468   1469   
  1469   1470   Usage: $0 [OPTION]... [VAR=VALUE]...
  1470   1471   
  1471   1472   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1472   1473   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1473   1474   
  1474   1475   Defaults for the options are specified in brackets.
................................................................................
  1525   1526     --build=BUILD     configure for building on BUILD [guessed]
  1526   1527     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1527   1528   _ACEOF
  1528   1529   fi
  1529   1530   
  1530   1531   if test -n "$ac_init_help"; then
  1531   1532     case $ac_init_help in
  1532         -     short | recursive ) echo "Configuration of sqlite 3.22.0:";;
         1533  +     short | recursive ) echo "Configuration of sqlite 3.23.0:";;
  1533   1534      esac
  1534   1535     cat <<\_ACEOF
  1535   1536   
  1536   1537   Optional Features:
  1537   1538     --disable-option-checking  ignore unrecognized --enable/--with options
  1538   1539     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1539   1540     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1650   1651       cd "$ac_pwd" || { ac_status=$?; break; }
  1651   1652     done
  1652   1653   fi
  1653   1654   
  1654   1655   test -n "$ac_init_help" && exit $ac_status
  1655   1656   if $ac_init_version; then
  1656   1657     cat <<\_ACEOF
  1657         -sqlite configure 3.22.0
         1658  +sqlite configure 3.23.0
  1658   1659   generated by GNU Autoconf 2.69
  1659   1660   
  1660   1661   Copyright (C) 2012 Free Software Foundation, Inc.
  1661   1662   This configure script is free software; the Free Software Foundation
  1662   1663   gives unlimited permission to copy, distribute and modify it.
  1663   1664   _ACEOF
  1664   1665     exit
................................................................................
  2069   2070     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2070   2071   
  2071   2072   } # ac_fn_c_check_header_mongrel
  2072   2073   cat >config.log <<_ACEOF
  2073   2074   This file contains any messages produced by compilers while
  2074   2075   running configure, to aid debugging if configure makes a mistake.
  2075   2076   
  2076         -It was created by sqlite $as_me 3.22.0, which was
         2077  +It was created by sqlite $as_me 3.23.0, which was
  2077   2078   generated by GNU Autoconf 2.69.  Invocation command line was
  2078   2079   
  2079   2080     $ $0 $@
  2080   2081   
  2081   2082   _ACEOF
  2082   2083   exec 5>>config.log
  2083   2084   {
................................................................................
  3927   3928   { $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5
  3928   3929   $as_echo_n "checking the name lister ($NM) interface... " >&6; }
  3929   3930   if ${lt_cv_nm_interface+:} false; then :
  3930   3931     $as_echo_n "(cached) " >&6
  3931   3932   else
  3932   3933     lt_cv_nm_interface="BSD nm"
  3933   3934     echo "int some_variable = 0;" > conftest.$ac_ext
  3934         -  (eval echo "\"\$as_me:3934: $ac_compile\"" >&5)
         3935  +  (eval echo "\"\$as_me:3935: $ac_compile\"" >&5)
  3935   3936     (eval "$ac_compile" 2>conftest.err)
  3936   3937     cat conftest.err >&5
  3937         -  (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
         3938  +  (eval echo "\"\$as_me:3938: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  3938   3939     (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  3939   3940     cat conftest.err >&5
  3940         -  (eval echo "\"\$as_me:3940: output\"" >&5)
         3941  +  (eval echo "\"\$as_me:3941: output\"" >&5)
  3941   3942     cat conftest.out >&5
  3942   3943     if $GREP 'External.*some_variable' conftest.out > /dev/null; then
  3943   3944       lt_cv_nm_interface="MS dumpbin"
  3944   3945     fi
  3945   3946     rm -f conftest*
  3946   3947   fi
  3947   3948   { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5
................................................................................
  5139   5140   	;;
  5140   5141       esac
  5141   5142     fi
  5142   5143     rm -rf conftest*
  5143   5144     ;;
  5144   5145   *-*-irix6*)
  5145   5146     # Find out which ABI we are using.
  5146         -  echo '#line 5146 "configure"' > conftest.$ac_ext
         5147  +  echo '#line 5147 "configure"' > conftest.$ac_ext
  5147   5148     if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  5148   5149     (eval $ac_compile) 2>&5
  5149   5150     ac_status=$?
  5150   5151     $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
  5151   5152     test $ac_status = 0; }; then
  5152   5153       if test "$lt_cv_prog_gnu_ld" = yes; then
  5153   5154         case `/usr/bin/file conftest.$ac_objext` in
................................................................................
  6664   6665      # Note that $ac_compile itself does not contain backslashes and begins
  6665   6666      # with a dollar sign (not a hyphen), so the echo should work correctly.
  6666   6667      # The option is referenced via a variable to avoid confusing sed.
  6667   6668      lt_compile=`echo "$ac_compile" | $SED \
  6668   6669      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  6669   6670      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  6670   6671      -e 's:$: $lt_compiler_flag:'`
  6671         -   (eval echo "\"\$as_me:6671: $lt_compile\"" >&5)
         6672  +   (eval echo "\"\$as_me:6672: $lt_compile\"" >&5)
  6672   6673      (eval "$lt_compile" 2>conftest.err)
  6673   6674      ac_status=$?
  6674   6675      cat conftest.err >&5
  6675         -   echo "$as_me:6675: \$? = $ac_status" >&5
         6676  +   echo "$as_me:6676: \$? = $ac_status" >&5
  6676   6677      if (exit $ac_status) && test -s "$ac_outfile"; then
  6677   6678        # The compiler can only warn and ignore the option if not recognized
  6678   6679        # So say no if there are warnings other than the usual output.
  6679   6680        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  6680   6681        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  6681   6682        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  6682   6683          lt_cv_prog_compiler_rtti_exceptions=yes
................................................................................
  7003   7004      # Note that $ac_compile itself does not contain backslashes and begins
  7004   7005      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7005   7006      # The option is referenced via a variable to avoid confusing sed.
  7006   7007      lt_compile=`echo "$ac_compile" | $SED \
  7007   7008      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7008   7009      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7009   7010      -e 's:$: $lt_compiler_flag:'`
  7010         -   (eval echo "\"\$as_me:7010: $lt_compile\"" >&5)
         7011  +   (eval echo "\"\$as_me:7011: $lt_compile\"" >&5)
  7011   7012      (eval "$lt_compile" 2>conftest.err)
  7012   7013      ac_status=$?
  7013   7014      cat conftest.err >&5
  7014         -   echo "$as_me:7014: \$? = $ac_status" >&5
         7015  +   echo "$as_me:7015: \$? = $ac_status" >&5
  7015   7016      if (exit $ac_status) && test -s "$ac_outfile"; then
  7016   7017        # The compiler can only warn and ignore the option if not recognized
  7017   7018        # So say no if there are warnings other than the usual output.
  7018   7019        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
  7019   7020        $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
  7020   7021        if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
  7021   7022          lt_cv_prog_compiler_pic_works=yes
................................................................................
  7108   7109      # (2) before a word containing "conftest.", or (3) at the end.
  7109   7110      # Note that $ac_compile itself does not contain backslashes and begins
  7110   7111      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7111   7112      lt_compile=`echo "$ac_compile" | $SED \
  7112   7113      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7113   7114      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7114   7115      -e 's:$: $lt_compiler_flag:'`
  7115         -   (eval echo "\"\$as_me:7115: $lt_compile\"" >&5)
         7116  +   (eval echo "\"\$as_me:7116: $lt_compile\"" >&5)
  7116   7117      (eval "$lt_compile" 2>out/conftest.err)
  7117   7118      ac_status=$?
  7118   7119      cat out/conftest.err >&5
  7119         -   echo "$as_me:7119: \$? = $ac_status" >&5
         7120  +   echo "$as_me:7120: \$? = $ac_status" >&5
  7120   7121      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7121   7122      then
  7122   7123        # The compiler can only warn and ignore the option if not recognized
  7123   7124        # So say no if there are warnings
  7124   7125        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7125   7126        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7126   7127        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  7163   7164      # (2) before a word containing "conftest.", or (3) at the end.
  7164   7165      # Note that $ac_compile itself does not contain backslashes and begins
  7165   7166      # with a dollar sign (not a hyphen), so the echo should work correctly.
  7166   7167      lt_compile=`echo "$ac_compile" | $SED \
  7167   7168      -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
  7168   7169      -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
  7169   7170      -e 's:$: $lt_compiler_flag:'`
  7170         -   (eval echo "\"\$as_me:7170: $lt_compile\"" >&5)
         7171  +   (eval echo "\"\$as_me:7171: $lt_compile\"" >&5)
  7171   7172      (eval "$lt_compile" 2>out/conftest.err)
  7172   7173      ac_status=$?
  7173   7174      cat out/conftest.err >&5
  7174         -   echo "$as_me:7174: \$? = $ac_status" >&5
         7175  +   echo "$as_me:7175: \$? = $ac_status" >&5
  7175   7176      if (exit $ac_status) && test -s out/conftest2.$ac_objext
  7176   7177      then
  7177   7178        # The compiler can only warn and ignore the option if not recognized
  7178   7179        # So say no if there are warnings
  7179   7180        $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
  7180   7181        $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
  7181   7182        if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
................................................................................
  9543   9544   else
  9544   9545     	  if test "$cross_compiling" = yes; then :
  9545   9546     lt_cv_dlopen_self=cross
  9546   9547   else
  9547   9548     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9548   9549     lt_status=$lt_dlunknown
  9549   9550     cat > conftest.$ac_ext <<_LT_EOF
  9550         -#line 9550 "configure"
         9551  +#line 9551 "configure"
  9551   9552   #include "confdefs.h"
  9552   9553   
  9553   9554   #if HAVE_DLFCN_H
  9554   9555   #include <dlfcn.h>
  9555   9556   #endif
  9556   9557   
  9557   9558   #include <stdio.h>
................................................................................
  9639   9640   else
  9640   9641     	  if test "$cross_compiling" = yes; then :
  9641   9642     lt_cv_dlopen_self_static=cross
  9642   9643   else
  9643   9644     lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  9644   9645     lt_status=$lt_dlunknown
  9645   9646     cat > conftest.$ac_ext <<_LT_EOF
  9646         -#line 9646 "configure"
         9647  +#line 9647 "configure"
  9647   9648   #include "confdefs.h"
  9648   9649   
  9649   9650   #if HAVE_DLFCN_H
  9650   9651   #include <dlfcn.h>
  9651   9652   #endif
  9652   9653   
  9653   9654   #include <stdio.h>
................................................................................
 11269  11270     use_amalgamation=yes
 11270  11271   fi
 11271  11272   
 11272  11273   if test "${use_amalgamation}" != "yes" ; then
 11273  11274     USE_AMALGAMATION=0
 11274  11275   fi
 11275  11276   
        11277  +
        11278  +#########
        11279  +# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
        11280  +for ac_header in zlib.h
        11281  +do :
        11282  +  ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default"
        11283  +if test "x$ac_cv_header_zlib_h" = xyes; then :
        11284  +  cat >>confdefs.h <<_ACEOF
        11285  +#define HAVE_ZLIB_H 1
        11286  +_ACEOF
        11287  +
        11288  +fi
        11289  +
        11290  +done
        11291  +
        11292  +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5
        11293  +$as_echo_n "checking for library containing deflate... " >&6; }
        11294  +if ${ac_cv_search_deflate+:} false; then :
        11295  +  $as_echo_n "(cached) " >&6
        11296  +else
        11297  +  ac_func_search_save_LIBS=$LIBS
        11298  +cat confdefs.h - <<_ACEOF >conftest.$ac_ext
        11299  +/* end confdefs.h.  */
        11300  +
        11301  +/* Override any GCC internal prototype to avoid an error.
        11302  +   Use char because int might match the return type of a GCC
        11303  +   builtin and then its argument prototype would still apply.  */
        11304  +#ifdef __cplusplus
        11305  +extern "C"
        11306  +#endif
        11307  +char deflate ();
        11308  +int
        11309  +main ()
        11310  +{
        11311  +return deflate ();
        11312  +  ;
        11313  +  return 0;
        11314  +}
        11315  +_ACEOF
        11316  +for ac_lib in '' z; do
        11317  +  if test -z "$ac_lib"; then
        11318  +    ac_res="none required"
        11319  +  else
        11320  +    ac_res=-l$ac_lib
        11321  +    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
        11322  +  fi
        11323  +  if ac_fn_c_try_link "$LINENO"; then :
        11324  +  ac_cv_search_deflate=$ac_res
        11325  +fi
        11326  +rm -f core conftest.err conftest.$ac_objext \
        11327  +    conftest$ac_exeext
        11328  +  if ${ac_cv_search_deflate+:} false; then :
        11329  +  break
        11330  +fi
        11331  +done
        11332  +if ${ac_cv_search_deflate+:} false; then :
        11333  +
        11334  +else
        11335  +  ac_cv_search_deflate=no
        11336  +fi
        11337  +rm conftest.$ac_ext
        11338  +LIBS=$ac_func_search_save_LIBS
        11339  +fi
        11340  +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5
        11341  +$as_echo "$ac_cv_search_deflate" >&6; }
        11342  +ac_res=$ac_cv_search_deflate
        11343  +if test "$ac_res" != no; then :
        11344  +  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
        11345  +  HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"
        11346  +else
        11347  +  HAVE_ZLIB=""
        11348  +fi
        11349  +
        11350  +
 11276  11351   
 11277  11352   #########
 11278  11353   # See whether we should allow loadable extensions
 11279  11354   # Check whether --enable-load-extension was given.
 11280  11355   if test "${enable_load_extension+set}" = set; then :
 11281  11356     enableval=$enable_load_extension; use_loadextension=$enableval
 11282  11357   else
................................................................................
 12163  12238   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12164  12239   
 12165  12240   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12166  12241   # Save the log message, to keep $0 and so on meaningful, and to
 12167  12242   # report actual input values of CONFIG_FILES etc. instead of their
 12168  12243   # values after options handling.
 12169  12244   ac_log="
 12170         -This file was extended by sqlite $as_me 3.22.0, which was
        12245  +This file was extended by sqlite $as_me 3.23.0, which was
 12171  12246   generated by GNU Autoconf 2.69.  Invocation command line was
 12172  12247   
 12173  12248     CONFIG_FILES    = $CONFIG_FILES
 12174  12249     CONFIG_HEADERS  = $CONFIG_HEADERS
 12175  12250     CONFIG_LINKS    = $CONFIG_LINKS
 12176  12251     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12177  12252     $ $0 $@
................................................................................
 12229  12304   
 12230  12305   Report bugs to the package provider."
 12231  12306   
 12232  12307   _ACEOF
 12233  12308   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12234  12309   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12235  12310   ac_cs_version="\\
 12236         -sqlite config.status 3.22.0
        12311  +sqlite config.status 3.23.0
 12237  12312   configured by $0, generated by GNU Autoconf 2.69,
 12238  12313     with options \\"\$ac_cs_config\\"
 12239  12314   
 12240  12315   Copyright (C) 2012 Free Software Foundation, Inc.
 12241  12316   This config.status script is free software; the Free Software Foundation
 12242  12317   gives unlimited permission to copy, distribute and modify it."
 12243  12318

Changes to configure.ac.

   572    572         [Disable the amalgamation and instead build all files separately]),
   573    573         [use_amalgamation=$enableval],[use_amalgamation=yes])
   574    574   if test "${use_amalgamation}" != "yes" ; then
   575    575     USE_AMALGAMATION=0
   576    576   fi
   577    577   AC_SUBST(USE_AMALGAMATION)
   578    578   
          579  +#########
          580  +# Look for zlib.  Only needed by extensions and by the sqlite3.exe shell
          581  +AC_CHECK_HEADERS(zlib.h)
          582  +AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""])
          583  +AC_SUBST(HAVE_ZLIB)
          584  +
   579    585   #########
   580    586   # See whether we should allow loadable extensions
   581    587   AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
   582    588         [Disable loading of external extensions]),
   583    589         [use_loadextension=$enableval],[use_loadextension=yes])
   584    590   if test "${use_loadextension}" = "yes" ; then
   585    591     OPT_FEATURE_FLAGS=""

Added ext/expert/README.md.

            1  +## SQLite Expert Extension
            2  +
            3  +This folder contains code for a simple system to propose useful indexes
            4  +given a database and a set of SQL queries. It works as follows:
            5  +
            6  +  1. The user database schema is copied to a temporary database.
            7  +
            8  +  1. All SQL queries are prepared against the temporary database.
            9  +     Information regarding the WHERE and ORDER BY clauses, and other query
           10  +     features that affect index selection are recorded.
           11  +
           12  +  1. The information gathered in step 2 is used to create candidate 
           13  +     indexes - indexes that the planner might have made use of in the previous
           14  +     step, had they been available.
           15  +
           16  +  1. A subset of the data in the user database is used to generate statistics
           17  +     for all existing indexes and the candidate indexes generated in step 3
           18  +     above.
           19  +
           20  +  1. The SQL queries are prepared a second time. If the planner uses any
           21  +     of the indexes created in step 3, they are recommended to the user.
           22  +
           23  +# C API
           24  +
           25  +The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
           26  +as follows:
           27  +
           28  +  1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
           29  +     A database handle opened by the user is passed as an argument.
           30  +
           31  +  1. The sqlite3expert object is configured with one or more SQL statements
           32  +     by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
           33  +     specify a single SQL statement, or multiple statements separated by
           34  +     semi-colons.
           35  +  
           36  +  1. Optionally, the **sqlite3\_expert\_config()** API may be used to 
           37  +     configure the size of the data subset used to generate index statistics.
           38  +     Using a smaller subset of the data can speed up the analysis.
           39  +
           40  +  1. **sqlite3\_expert\_analyze()** is called to run the analysis.
           41  +
           42  +  1. One or more calls are made to **sqlite3\_expert\_report()** to extract
           43  +     components of the results of the analysis.
           44  +
           45  +  1. **sqlite3\_expert\_destroy()** is called to free all resources.
           46  +
           47  +Refer to comments in sqlite3expert.h for further details.
           48  +
           49  +# sqlite3_expert application
           50  +
           51  +The file "expert.c" contains the code for a command line application that
           52  +uses the API described above. It can be compiled with (for example):
           53  +
           54  +<pre>
           55  +  gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
           56  +</pre>
           57  +
           58  +Assuming the database is named "test.db", it can then be run to analyze a
           59  +single query:
           60  +
           61  +<pre>
           62  +  ./sqlite3_expert -sql &lt;sql-query&gt; test.db
           63  +</pre>
           64  +
           65  +Or an entire text file worth of queries with:
           66  +
           67  +<pre>
           68  +  ./sqlite3_expert -file &lt;text-file&gt; test.db
           69  +</pre>
           70  +
           71  +By default, sqlite3\_expert generates index statistics using all the data in
           72  +the user database. For a large database, this may be prohibitively time
           73  +consuming. The "-sample" option may be used to configure sqlite3\_expert to
           74  +generate statistics based on an integer percentage of the user database as
           75  +follows:
           76  +
           77  +<pre>
           78  +  # Generate statistics based on 25% of the user database rows:
           79  +  ./sqlite3_expert -sample 25 -sql &lt;sql-query&gt; test.db
           80  +
           81  +  # Do not generate any statistics at all:
           82  +  ./sqlite3_expert -sample 0 -sql &lt;sql-query&gt; test.db
           83  +</pre>

Added ext/expert/expert.c.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +
           15  +#include <sqlite3.h>
           16  +#include <stdio.h>
           17  +#include <stdlib.h>
           18  +#include <string.h>
           19  +#include "sqlite3expert.h"
           20  +
           21  +
           22  +static void option_requires_argument(const char *zOpt){
           23  +  fprintf(stderr, "Option requires an argument: %s\n", zOpt);
           24  +  exit(-3);
           25  +}
           26  +
           27  +static int option_integer_arg(const char *zVal){
           28  +  return atoi(zVal);
           29  +}
           30  +
           31  +static void usage(char **argv){
           32  +  fprintf(stderr, "\n");
           33  +  fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
           34  +  fprintf(stderr, "\n");
           35  +  fprintf(stderr, "Options are:\n");
           36  +  fprintf(stderr, "  -sql SQL   (analyze SQL statements passed as argument)\n");
           37  +  fprintf(stderr, "  -file FILE (read SQL statements from file FILE)\n");
           38  +  fprintf(stderr, "  -verbose LEVEL (integer verbosity level. default 1)\n");
           39  +  fprintf(stderr, "  -sample PERCENT (percent of db to sample. default 100)\n");
           40  +  exit(-1);
           41  +}
           42  +
           43  +static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
           44  +  FILE *in = fopen(zFile, "rb");
           45  +  long nIn;
           46  +  size_t nRead;
           47  +  char *pBuf;
           48  +  int rc;
           49  +  if( in==0 ){
           50  +    *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
           51  +    return SQLITE_ERROR;
           52  +  }
           53  +  fseek(in, 0, SEEK_END);
           54  +  nIn = ftell(in);
           55  +  rewind(in);
           56  +  pBuf = sqlite3_malloc64( nIn+1 );
           57  +  nRead = fread(pBuf, nIn, 1, in);
           58  +  fclose(in);
           59  +  if( nRead!=1 ){
           60  +    sqlite3_free(pBuf);
           61  +    *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
           62  +    return SQLITE_ERROR;
           63  +  }
           64  +  pBuf[nIn] = 0;
           65  +  rc = sqlite3_expert_sql(p, pBuf, pzErr);
           66  +  sqlite3_free(pBuf);
           67  +  return rc;
           68  +}
           69  +
           70  +int main(int argc, char **argv){
           71  +  const char *zDb;
           72  +  int rc = 0;
           73  +  char *zErr = 0;
           74  +  int i;
           75  +  int iVerbose = 1;               /* -verbose option */
           76  +
           77  +  sqlite3 *db = 0;
           78  +  sqlite3expert *p = 0;
           79  +
           80  +  if( argc<2 ) usage(argv);
           81  +  zDb = argv[argc-1];
           82  +  if( zDb[0]=='-' ) usage(argv);
           83  +  rc = sqlite3_open(zDb, &db);
           84  +  if( rc!=SQLITE_OK ){
           85  +    fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
           86  +    exit(-2);
           87  +  }
           88  +
           89  +  p = sqlite3_expert_new(db, &zErr);
           90  +  if( p==0 ){
           91  +    fprintf(stderr, "Cannot run analysis: %s\n", zErr);
           92  +    rc = 1;
           93  +  }else{
           94  +    for(i=1; i<(argc-1); i++){
           95  +      char *zArg = argv[i];
           96  +      if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
           97  +      int nArg = (int)strlen(zArg);
           98  +      if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
           99  +        if( ++i==(argc-1) ) option_requires_argument("-file");
          100  +        rc = readSqlFromFile(p, argv[i], &zErr);
          101  +      }
          102  +
          103  +      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
          104  +        if( ++i==(argc-1) ) option_requires_argument("-sql");
          105  +        rc = sqlite3_expert_sql(p, argv[i], &zErr);
          106  +      }
          107  +
          108  +      else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
          109  +        int iSample;
          110  +        if( ++i==(argc-1) ) option_requires_argument("-sample");
          111  +        iSample = option_integer_arg(argv[i]);
          112  +        sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
          113  +      }
          114  +
          115  +      else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
          116  +        if( ++i==(argc-1) ) option_requires_argument("-verbose");
          117  +        iVerbose = option_integer_arg(argv[i]);
          118  +      }
          119  +
          120  +      else{
          121  +        usage(argv);
          122  +      }
          123  +    }
          124  +  }
          125  +
          126  +  if( rc==SQLITE_OK ){
          127  +    rc = sqlite3_expert_analyze(p, &zErr);
          128  +  }
          129  +
          130  +  if( rc==SQLITE_OK ){
          131  +    int nQuery = sqlite3_expert_count(p);
          132  +    if( iVerbose>0 ){
          133  +      const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
          134  +      fprintf(stdout, "-- Candidates -------------------------------\n");
          135  +      fprintf(stdout, "%s\n", zCand);
          136  +    }
          137  +    for(i=0; i<nQuery; i++){
          138  +      const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL);
          139  +      const char *zIdx = sqlite3_expert_report(p, i, EXPERT_REPORT_INDEXES);
          140  +      const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN);
          141  +      if( zIdx==0 ) zIdx = "(no new indexes)\n";
          142  +      if( iVerbose>0 ){
          143  +        fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
          144  +        fprintf(stdout, "%s\n\n", zSql);
          145  +      }
          146  +      fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
          147  +    }
          148  +  }else{
          149  +    fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
          150  +  }
          151  +
          152  +  sqlite3_expert_destroy(p);
          153  +  sqlite3_free(zErr);
          154  +  return rc;
          155  +}

Added ext/expert/expert1.test.

            1  +# 2009 Nov 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# The focus of this file is testing the CLI shell tool. Specifically,
           13  +# the ".recommend" command.
           14  +#
           15  +#
           16  +
           17  +# Test plan:
           18  +#
           19  +#
           20  +if {![info exists testdir]} {
           21  +  set testdir [file join [file dirname [info script]] .. .. test]
           22  +}
           23  +source $testdir/tester.tcl
           24  +set testprefix expert1
           25  +
           26  +if {[info commands sqlite3_expert_new]==""} {
           27  +  finish_test
           28  +  return
           29  +}
           30  +
           31  +set CLI [test_binary_name sqlite3]
           32  +set CMD [test_binary_name sqlite3_expert]
           33  +
           34  +proc squish {txt} {
           35  +  regsub -all {[[:space:]]+} $txt { }
           36  +}
           37  +
           38  +proc do_setup_rec_test {tn setup sql res} {
           39  +  reset_db
           40  +  db eval $setup
           41  +  uplevel [list do_rec_test $tn $sql $res]
           42  +}
           43  +
           44  +foreach {tn setup} {
           45  +  1 {
           46  +    if {![file executable $CMD]} { continue }
           47  +
           48  +    proc do_rec_test {tn sql res} {
           49  +      set res [squish [string trim $res]]
           50  +      set tst [subst -nocommands { 
           51  +        squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
           52  +      }]
           53  +      uplevel [list do_test $tn $tst $res]
           54  +    }
           55  +  }
           56  +  2 {
           57  +    if {[info commands sqlite3_expert_new]==""} { continue }
           58  +
           59  +    proc do_rec_test {tn sql res} {
           60  +      set expert [sqlite3_expert_new db]
           61  +      $expert sql $sql
           62  +      $expert analyze
           63  +
           64  +      set result [list]
           65  +      for {set i 0} {$i < [$expert count]} {incr i} {
           66  +        set idx [string trim [$expert report $i indexes]]
           67  +        if {$idx==""} {set idx "(no new indexes)"}
           68  +        lappend result $idx
           69  +        lappend result [string trim [$expert report $i plan]]
           70  +      }
           71  +
           72  +      $expert destroy
           73  +
           74  +      set tst [subst -nocommands {set {} [squish [join {$result}]]}]
           75  +      uplevel [list do_test $tn $tst [string trim [squish $res]]]
           76  +    }
           77  +  }
           78  +  3 {
           79  +    if {![file executable $CLI]} { continue }
           80  +
           81  +    proc do_rec_test {tn sql res} {
           82  +      set res [squish [string trim $res]]
           83  +      set tst [subst -nocommands { 
           84  +        squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
           85  +      }]
           86  +      uplevel [list do_test $tn $tst $res]
           87  +    }
           88  +  }
           89  +} {
           90  +
           91  +  eval $setup
           92  +
           93  +
           94  +do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
           95  +  SELECT * FROM t1
           96  +} {
           97  +  (no new indexes)
           98  +  0|0|0|SCAN TABLE t1
           99  +}
          100  +
          101  +do_setup_rec_test $tn.2 {
          102  +  CREATE TABLE t1(a, b, c);
          103  +} {
          104  +  SELECT * FROM t1 WHERE b>?;
          105  +} {
          106  +  CREATE INDEX t1_idx_00000062 ON t1(b);
          107  +  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
          108  +}
          109  +
          110  +do_setup_rec_test $tn.3 {
          111  +  CREATE TABLE t1(a, b, c);
          112  +} {
          113  +  SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
          114  +} {
          115  +  CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
          116  +  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
          117  +}
          118  +
          119  +do_setup_rec_test $tn.4 {
          120  +  CREATE TABLE t1(a, b, c);
          121  +} {
          122  +  SELECT a FROM t1 ORDER BY b;
          123  +} {
          124  +  CREATE INDEX t1_idx_00000062 ON t1(b);
          125  +  0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
          126  +}
          127  +
          128  +do_setup_rec_test $tn.5 {
          129  +  CREATE TABLE t1(a, b, c);
          130  +} {
          131  +  SELECT a FROM t1 WHERE a=? ORDER BY b;
          132  +} {
          133  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b);
          134  +  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
          135  +}
          136  +
          137  +do_setup_rec_test $tn.6 {
          138  +  CREATE TABLE t1(a, b, c);
          139  +} {
          140  +  SELECT min(a) FROM t1
          141  +} {
          142  +  CREATE INDEX t1_idx_00000061 ON t1(a);
          143  +  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
          144  +}
          145  +
          146  +do_setup_rec_test $tn.7 {
          147  +  CREATE TABLE t1(a, b, c);
          148  +} {
          149  +  SELECT * FROM t1 ORDER BY a, b, c;
          150  +} {
          151  +  CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
          152  +  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
          153  +}
          154  +
          155  +#do_setup_rec_test $tn.1.8 {
          156  +#  CREATE TABLE t1(a, b, c);
          157  +#} {
          158  +#  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
          159  +#} {
          160  +#  CREATE INDEX t1_idx_5be6e222 ON t1(a, b COLLATE NOCASE DESC, c);
          161  +#  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5be6e222
          162  +#}
          163  +
          164  +do_setup_rec_test $tn.8.1 {
          165  +  CREATE TABLE t1(a COLLATE NOCase, b, c);
          166  +} {
          167  +  SELECT * FROM t1 WHERE a=?
          168  +} {
          169  +  CREATE INDEX t1_idx_00000061 ON t1(a);
          170  +  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
          171  +}
          172  +do_setup_rec_test $tn.8.2 {
          173  +  CREATE TABLE t1(a, b COLLATE nocase, c);
          174  +} {
          175  +  SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
          176  +} {
          177  +  CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
          178  +  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
          179  +}
          180  +
          181  +
          182  +# Tables with names that require quotes.
          183  +#
          184  +do_setup_rec_test $tn.9.1 {
          185  +  CREATE TABLE "t t"(a, b, c);
          186  +} {
          187  +  SELECT * FROM "t t" WHERE a=?
          188  +} {
          189  +  CREATE INDEX 't t_idx_00000061' ON 't t'(a);
          190  +  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
          191  +}
          192  +
          193  +do_setup_rec_test $tn.9.2 {
          194  +  CREATE TABLE "t t"(a, b, c);
          195  +} {
          196  +  SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
          197  +} {
          198  +  CREATE INDEX 't t_idx_00000062' ON 't t'(b);
          199  +  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
          200  +}
          201  +
          202  +# Columns with names that require quotes.
          203  +#
          204  +do_setup_rec_test $tn.10.1 {
          205  +  CREATE TABLE t3(a, "b b", c);
          206  +} {
          207  +  SELECT * FROM t3 WHERE "b b" = ?
          208  +} {
          209  +  CREATE INDEX t3_idx_00050c52 ON t3('b b');
          210  +  0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
          211  +}
          212  +
          213  +do_setup_rec_test $tn.10.2 {
          214  +  CREATE TABLE t3(a, "b b", c);
          215  +} {
          216  +  SELECT * FROM t3 ORDER BY "b b"
          217  +} {
          218  +  CREATE INDEX t3_idx_00050c52 ON t3('b b');
          219  +  0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
          220  +}
          221  +
          222  +# Transitive constraints
          223  +#
          224  +do_setup_rec_test $tn.11.1 {
          225  +  CREATE TABLE t5(a, b);
          226  +  CREATE TABLE t6(c, d);
          227  +} {
          228  +  SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
          229  +} {
          230  +  CREATE INDEX t5_idx_000123a7 ON t5(a, b);
          231  +  CREATE INDEX t6_idx_00000063 ON t6(c);
          232  +  0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
          233  +  0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
          234  +}
          235  +
          236  +# OR terms.
          237  +#
          238  +do_setup_rec_test $tn.12.1 {
          239  +  CREATE TABLE t7(a, b);
          240  +} {
          241  +  SELECT * FROM t7 WHERE a=? OR b=?
          242  +} {
          243  +  CREATE INDEX t7_idx_00000062 ON t7(b);
          244  +  CREATE INDEX t7_idx_00000061 ON t7(a);
          245  +  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
          246  +  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
          247  +}
          248  +
          249  +# rowid terms.
          250  +#
          251  +do_setup_rec_test $tn.13.1 {
          252  +  CREATE TABLE t8(a, b);
          253  +} {
          254  +  SELECT * FROM t8 WHERE rowid=?
          255  +} {
          256  +  (no new indexes)
          257  +  0|0|0|SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
          258  +}
          259  +do_setup_rec_test $tn.13.2 {
          260  +  CREATE TABLE t8(a, b);
          261  +} {
          262  +  SELECT * FROM t8 ORDER BY rowid
          263  +} {
          264  +  (no new indexes)
          265  +  0|0|0|SCAN TABLE t8
          266  +}
          267  +do_setup_rec_test $tn.13.3 {
          268  +  CREATE TABLE t8(a, b);
          269  +} {
          270  +  SELECT * FROM t8 WHERE a=? ORDER BY rowid
          271  +} {
          272  +  CREATE INDEX t8_idx_00000061 ON t8(a); 
          273  +  0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
          274  +}
          275  +
          276  +# Triggers
          277  +#
          278  +do_setup_rec_test $tn.14 {
          279  +  CREATE TABLE t9(a, b, c);
          280  +  CREATE TABLE t10(a, b, c);
          281  +  CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
          282  +    UPDATE t10 SET a=new.a WHERE b = new.b;
          283  +  END;
          284  +} {
          285  +  INSERT INTO t9 VALUES(?, ?, ?);
          286  +} {
          287  +  CREATE INDEX t10_idx_00000062 ON t10(b); 
          288  +  0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
          289  +}
          290  +
          291  +do_setup_rec_test $tn.15 {
          292  +  CREATE TABLE t1(a, b);
          293  +  CREATE TABLE t2(c, d);
          294  +
          295  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          296  +  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
          297  +
          298  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          299  +  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
          300  +} {
          301  +  SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
          302  +} {
          303  +  CREATE INDEX t2_idx_00000064 ON t2(d);
          304  +  0|0|0|SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
          305  +  0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
          306  +}
          307  +
          308  +do_setup_rec_test $tn.16 {
          309  +  CREATE TABLE t1(a, b);
          310  +} {
          311  +  SELECT * FROM t1 WHERE b IS NOT NULL;
          312  +} {
          313  +  (no new indexes)
          314  +  0|0|0|SCAN TABLE t1
          315  +}
          316  +
          317  +}
          318  +
          319  +proc do_candidates_test {tn sql res} {
          320  +  set res [squish [string trim $res]]
          321  +
          322  +  set expert [sqlite3_expert_new db]
          323  +  $expert sql $sql
          324  +  $expert analyze
          325  +
          326  +  set candidates [squish [string trim [$expert report 0 candidates]]]
          327  +  $expert destroy
          328  +
          329  +  uplevel [list do_test $tn [list set {} $candidates] $res]
          330  +}
          331  +
          332  +
          333  +reset_db
          334  +do_execsql_test 3.0 {
          335  +  CREATE TABLE t1(a, b);
          336  +  CREATE TABLE t2(c, d);
          337  +
          338  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          339  +  INSERT INTO t1 SELECT (i-1)/50, (i-1)/20 FROM s;
          340  +
          341  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
          342  +  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
          343  +}
          344  +do_candidates_test 3.1 {
          345  +  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
          346  +} {
          347  +  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
          348  +  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
          349  +  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
          350  +  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
          351  +}
          352  +
          353  +do_candidates_test 3.2 {
          354  +  SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=?
          355  +} {
          356  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 17
          357  +  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
          358  +}
          359  +
          360  +do_execsql_test 3.2 {
          361  +  CREATE INDEX t1_idx_00000061 ON t1(a); -- stat1: 100 50 
          362  +  CREATE INDEX t1_idx_00000062 ON t1(b); -- stat1: 100 20 
          363  +  CREATE INDEX t1_idx_000123a7 ON t1(a, b); -- stat1: 100 50 16
          364  +
          365  +  CREATE INDEX t2_idx_00000063 ON t2(c); -- stat1: 100 20 
          366  +  CREATE INDEX t2_idx_00000064 ON t2(d); -- stat1: 100 5
          367  +  CREATE INDEX t2_idx_0001295b ON t2(c, d); -- stat1: 100 20 5
          368  +
          369  +  ANALYZE;
          370  +  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
          371  +} {
          372  +  t1 t1_idx_00000061 {100 50} 
          373  +  t1 t1_idx_00000062 {100 20}
          374  +  t1 t1_idx_000123a7 {100 50 17}
          375  +  t2 t2_idx_00000063 {100 20} 
          376  +  t2 t2_idx_00000064 {100 5} 
          377  +  t2 t2_idx_0001295b {100 20 5}
          378  +}
          379  +
          380  +
          381  +finish_test
          382  +

Added ext/expert/sqlite3expert.c.

            1  +/*
            2  +** 2017 April 09
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +#include "sqlite3expert.h"
           14  +#include <assert.h>
           15  +#include <string.h>
           16  +#include <stdio.h>
           17  +
           18  +#ifndef SQLITE_OMIT_VIRTUALTABLE 
           19  +
           20  +typedef sqlite3_int64 i64;
           21  +typedef sqlite3_uint64 u64;
           22  +
           23  +typedef struct IdxColumn IdxColumn;
           24  +typedef struct IdxConstraint IdxConstraint;
           25  +typedef struct IdxScan IdxScan;
           26  +typedef struct IdxStatement IdxStatement;
           27  +typedef struct IdxTable IdxTable;
           28  +typedef struct IdxWrite IdxWrite;
           29  +
           30  +#define STRLEN  (int)strlen
           31  +
           32  +/*
           33  +** A temp table name that we assume no user database will actually use.
           34  +** If this assumption proves incorrect triggers on the table with the
           35  +** conflicting name will be ignored.
           36  +*/
           37  +#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776"
           38  +
           39  +/*
           40  +** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
           41  +** any other type of single-ended range constraint on a column).
           42  +**
           43  +** pLink:
           44  +**   Used to temporarily link IdxConstraint objects into lists while
           45  +**   creating candidate indexes.
           46  +*/
           47  +struct IdxConstraint {
           48  +  char *zColl;                    /* Collation sequence */
           49  +  int bRange;                     /* True for range, false for eq */
           50  +  int iCol;                       /* Constrained table column */
           51  +  int bFlag;                      /* Used by idxFindCompatible() */
           52  +  int bDesc;                      /* True if ORDER BY <expr> DESC */
           53  +  IdxConstraint *pNext;           /* Next constraint in pEq or pRange list */
           54  +  IdxConstraint *pLink;           /* See above */
           55  +};
           56  +
           57  +/*
           58  +** A single scan of a single table.
           59  +*/
           60  +struct IdxScan {
           61  +  IdxTable *pTab;                 /* Associated table object */
           62  +  int iDb;                        /* Database containing table zTable */
           63  +  i64 covering;                   /* Mask of columns required for cov. index */
           64  +  IdxConstraint *pOrder;          /* ORDER BY columns */
           65  +  IdxConstraint *pEq;             /* List of == constraints */
           66  +  IdxConstraint *pRange;          /* List of < constraints */
           67  +  IdxScan *pNextScan;             /* Next IdxScan object for same analysis */
           68  +};
           69  +
           70  +/*
           71  +** Information regarding a single database table. Extracted from 
           72  +** "PRAGMA table_info" by function idxGetTableInfo().
           73  +*/
           74  +struct IdxColumn {
           75  +  char *zName;
           76  +  char *zColl;
           77  +  int iPk;
           78  +};
           79  +struct IdxTable {
           80  +  int nCol;
           81  +  char *zName;                    /* Table name */
           82  +  IdxColumn *aCol;
           83  +  IdxTable *pNext;                /* Next table in linked list of all tables */
           84  +};
           85  +
           86  +/*
           87  +** An object of the following type is created for each unique table/write-op
           88  +** seen. The objects are stored in a singly-linked list beginning at
           89  +** sqlite3expert.pWrite.
           90  +*/
           91  +struct IdxWrite {
           92  +  IdxTable *pTab;
           93  +  int eOp;                        /* SQLITE_UPDATE, DELETE or INSERT */
           94  +  IdxWrite *pNext;
           95  +};
           96  +
           97  +/*
           98  +** Each statement being analyzed is represented by an instance of this
           99  +** structure.
          100  +*/
          101  +struct IdxStatement {
          102  +  int iId;                        /* Statement number */
          103  +  char *zSql;                     /* SQL statement */
          104  +  char *zIdx;                     /* Indexes */
          105  +  char *zEQP;                     /* Plan */
          106  +  IdxStatement *pNext;
          107  +};
          108  +
          109  +
          110  +/*
          111  +** A hash table for storing strings. With space for a payload string
          112  +** with each entry. Methods are:
          113  +**
          114  +**   idxHashInit()
          115  +**   idxHashClear()
          116  +**   idxHashAdd()
          117  +**   idxHashSearch()
          118  +*/
          119  +#define IDX_HASH_SIZE 1023
          120  +typedef struct IdxHashEntry IdxHashEntry;
          121  +typedef struct IdxHash IdxHash;
          122  +struct IdxHashEntry {
          123  +  char *zKey;                     /* nul-terminated key */
          124  +  char *zVal;                     /* nul-terminated value string */
          125  +  char *zVal2;                    /* nul-terminated value string 2 */
          126  +  IdxHashEntry *pHashNext;        /* Next entry in same hash bucket */
          127  +  IdxHashEntry *pNext;            /* Next entry in hash */
          128  +};
          129  +struct IdxHash {
          130  +  IdxHashEntry *pFirst;
          131  +  IdxHashEntry *aHash[IDX_HASH_SIZE];
          132  +};
          133  +
          134  +/*
          135  +** sqlite3expert object.
          136  +*/
          137  +struct sqlite3expert {
          138  +  int iSample;                    /* Percentage of tables to sample for stat1 */
          139  +  sqlite3 *db;                    /* User database */
          140  +  sqlite3 *dbm;                   /* In-memory db for this analysis */
          141  +  sqlite3 *dbv;                   /* Vtab schema for this analysis */
          142  +  IdxTable *pTable;               /* List of all IdxTable objects */
          143  +  IdxScan *pScan;                 /* List of scan objects */
          144  +  IdxWrite *pWrite;               /* List of write objects */
          145  +  IdxStatement *pStatement;       /* List of IdxStatement objects */
          146  +  int bRun;                       /* True once analysis has run */
          147  +  char **pzErrmsg;
          148  +  int rc;                         /* Error code from whereinfo hook */
          149  +  IdxHash hIdx;                   /* Hash containing all candidate indexes */
          150  +  char *zCandidates;              /* For EXPERT_REPORT_CANDIDATES */
          151  +};
          152  +
          153  +
          154  +/*
          155  +** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). 
          156  +** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
          157  +*/
          158  +static void *idxMalloc(int *pRc, int nByte){
          159  +  void *pRet;
          160  +  assert( *pRc==SQLITE_OK );
          161  +  assert( nByte>0 );
          162  +  pRet = sqlite3_malloc(nByte);
          163  +  if( pRet ){
          164  +    memset(pRet, 0, nByte);
          165  +  }else{
          166  +    *pRc = SQLITE_NOMEM;
          167  +  }
          168  +  return pRet;
          169  +}
          170  +
          171  +/*
          172  +** Initialize an IdxHash hash table.
          173  +*/
          174  +static void idxHashInit(IdxHash *pHash){
          175  +  memset(pHash, 0, sizeof(IdxHash));
          176  +}
          177  +
          178  +/*
          179  +** Reset an IdxHash hash table.
          180  +*/
          181  +static void idxHashClear(IdxHash *pHash){
          182  +  int i;
          183  +  for(i=0; i<IDX_HASH_SIZE; i++){
          184  +    IdxHashEntry *pEntry;
          185  +    IdxHashEntry *pNext;
          186  +    for(pEntry=pHash->aHash[i]; pEntry; pEntry=pNext){
          187  +      pNext = pEntry->pHashNext;
          188  +      sqlite3_free(pEntry->zVal2);
          189  +      sqlite3_free(pEntry);
          190  +    }
          191  +  }
          192  +  memset(pHash, 0, sizeof(IdxHash));
          193  +}
          194  +
          195  +/*
          196  +** Return the index of the hash bucket that the string specified by the
          197  +** arguments to this function belongs.
          198  +*/
          199  +static int idxHashString(const char *z, int n){
          200  +  unsigned int ret = 0;
          201  +  int i;
          202  +  for(i=0; i<n; i++){
          203  +    ret += (ret<<3) + (unsigned char)(z[i]);
          204  +  }
          205  +  return (int)(ret % IDX_HASH_SIZE);
          206  +}
          207  +
          208  +/*
          209  +** If zKey is already present in the hash table, return non-zero and do
          210  +** nothing. Otherwise, add an entry with key zKey and payload string zVal to
          211  +** the hash table passed as the second argument. 
          212  +*/
          213  +static int idxHashAdd(
          214  +  int *pRc, 
          215  +  IdxHash *pHash, 
          216  +  const char *zKey,
          217  +  const char *zVal
          218  +){
          219  +  int nKey = STRLEN(zKey);
          220  +  int iHash = idxHashString(zKey, nKey);
          221  +  int nVal = (zVal ? STRLEN(zVal) : 0);
          222  +  IdxHashEntry *pEntry;
          223  +  assert( iHash>=0 );
          224  +  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
          225  +    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
          226  +      return 1;
          227  +    }
          228  +  }
          229  +  pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1);
          230  +  if( pEntry ){
          231  +    pEntry->zKey = (char*)&pEntry[1];
          232  +    memcpy(pEntry->zKey, zKey, nKey);
          233  +    if( zVal ){
          234  +      pEntry->zVal = &pEntry->zKey[nKey+1];
          235  +      memcpy(pEntry->zVal, zVal, nVal);
          236  +    }
          237  +    pEntry->pHashNext = pHash->aHash[iHash];
          238  +    pHash->aHash[iHash] = pEntry;
          239  +
          240  +    pEntry->pNext = pHash->pFirst;
          241  +    pHash->pFirst = pEntry;
          242  +  }
          243  +  return 0;
          244  +}
          245  +
          246  +/*
          247  +** If zKey/nKey is present in the hash table, return a pointer to the 
          248  +** hash-entry object.
          249  +*/
          250  +static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
          251  +  int iHash;
          252  +  IdxHashEntry *pEntry;
          253  +  if( nKey<0 ) nKey = STRLEN(zKey);
          254  +  iHash = idxHashString(zKey, nKey);
          255  +  assert( iHash>=0 );
          256  +  for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
          257  +    if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
          258  +      return pEntry;
          259  +    }
          260  +  }
          261  +  return 0;
          262  +}
          263  +
          264  +/*
          265  +** If the hash table contains an entry with a key equal to the string
          266  +** passed as the final two arguments to this function, return a pointer
          267  +** to the payload string. Otherwise, if zKey/nKey is not present in the
          268  +** hash table, return NULL.
          269  +*/
          270  +static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
          271  +  IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey);
          272  +  if( pEntry ) return pEntry->zVal;
          273  +  return 0;
          274  +}
          275  +
          276  +/*
          277  +** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
          278  +** variable to point to a copy of nul-terminated string zColl.
          279  +*/
          280  +static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
          281  +  IdxConstraint *pNew;
          282  +  int nColl = STRLEN(zColl);
          283  +
          284  +  assert( *pRc==SQLITE_OK );
          285  +  pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
          286  +  if( pNew ){
          287  +    pNew->zColl = (char*)&pNew[1];
          288  +    memcpy(pNew->zColl, zColl, nColl+1);
          289  +  }
          290  +  return pNew;
          291  +}
          292  +
          293  +/*
          294  +** An error associated with database handle db has just occurred. Pass
          295  +** the error message to callback function xOut.
          296  +*/
          297  +static void idxDatabaseError(
          298  +  sqlite3 *db,                    /* Database handle */
          299  +  char **pzErrmsg                 /* Write error here */
          300  +){
          301  +  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
          302  +}
          303  +
          304  +/*
          305  +** Prepare an SQL statement.
          306  +*/
          307  +static int idxPrepareStmt(
          308  +  sqlite3 *db,                    /* Database handle to compile against */
          309  +  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
          310  +  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
          311  +  const char *zSql                /* SQL statement to compile */
          312  +){
          313  +  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
          314  +  if( rc!=SQLITE_OK ){
          315  +    *ppStmt = 0;
          316  +    idxDatabaseError(db, pzErrmsg);
          317  +  }
          318  +  return rc;
          319  +}
          320  +
          321  +/*
          322  +** Prepare an SQL statement using the results of a printf() formatting.
          323  +*/
          324  +static int idxPrintfPrepareStmt(
          325  +  sqlite3 *db,                    /* Database handle to compile against */
          326  +  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
          327  +  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
          328  +  const char *zFmt,               /* printf() format of SQL statement */
          329  +  ...                             /* Trailing printf() arguments */
          330  +){
          331  +  va_list ap;
          332  +  int rc;
          333  +  char *zSql;
          334  +  va_start(ap, zFmt);
          335  +  zSql = sqlite3_vmprintf(zFmt, ap);
          336  +  if( zSql==0 ){
          337  +    rc = SQLITE_NOMEM;
          338  +  }else{
          339  +    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
          340  +    sqlite3_free(zSql);
          341  +  }
          342  +  va_end(ap);
          343  +  return rc;
          344  +}
          345  +
          346  +
          347  +/*************************************************************************
          348  +** Beginning of virtual table implementation.
          349  +*/
          350  +typedef struct ExpertVtab ExpertVtab;
          351  +struct ExpertVtab {
          352  +  sqlite3_vtab base;
          353  +  IdxTable *pTab;
          354  +  sqlite3expert *pExpert;
          355  +};
          356  +
          357  +typedef struct ExpertCsr ExpertCsr;
          358  +struct ExpertCsr {
          359  +  sqlite3_vtab_cursor base;
          360  +  sqlite3_stmt *pData;
          361  +};
          362  +
          363  +static char *expertDequote(const char *zIn){
          364  +  int n = STRLEN(zIn);
          365  +  char *zRet = sqlite3_malloc(n);
          366  +
          367  +  assert( zIn[0]=='\'' );
          368  +  assert( zIn[n-1]=='\'' );
          369  +
          370  +  if( zRet ){
          371  +    int iOut = 0;
          372  +    int iIn = 0;
          373  +    for(iIn=1; iIn<(n-1); iIn++){
          374  +      if( zIn[iIn]=='\'' ){
          375  +        assert( zIn[iIn+1]=='\'' );
          376  +        iIn++;
          377  +      }
          378  +      zRet[iOut++] = zIn[iIn];
          379  +    }
          380  +    zRet[iOut] = '\0';
          381  +  }
          382  +
          383  +  return zRet;
          384  +}
          385  +
          386  +/* 
          387  +** This function is the implementation of both the xConnect and xCreate
          388  +** methods of the r-tree virtual table.
          389  +**
          390  +**   argv[0]   -> module name
          391  +**   argv[1]   -> database name
          392  +**   argv[2]   -> table name
          393  +**   argv[...] -> column names...
          394  +*/
          395  +static int expertConnect(
          396  +  sqlite3 *db,
          397  +  void *pAux,
          398  +  int argc, const char *const*argv,
          399  +  sqlite3_vtab **ppVtab,
          400  +  char **pzErr
          401  +){
          402  +  sqlite3expert *pExpert = (sqlite3expert*)pAux;
          403  +  ExpertVtab *p = 0;
          404  +  int rc;
          405  +
          406  +  if( argc!=4 ){
          407  +    *pzErr = sqlite3_mprintf("internal error!");
          408  +    rc = SQLITE_ERROR;
          409  +  }else{
          410  +    char *zCreateTable = expertDequote(argv[3]);
          411  +    if( zCreateTable ){
          412  +      rc = sqlite3_declare_vtab(db, zCreateTable);
          413  +      if( rc==SQLITE_OK ){
          414  +        p = idxMalloc(&rc, sizeof(ExpertVtab));
          415  +      }
          416  +      if( rc==SQLITE_OK ){
          417  +        p->pExpert = pExpert;
          418  +        p->pTab = pExpert->pTable;
          419  +        assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 );
          420  +      }
          421  +      sqlite3_free(zCreateTable);
          422  +    }else{
          423  +      rc = SQLITE_NOMEM;
          424  +    }
          425  +  }
          426  +
          427  +  *ppVtab = (sqlite3_vtab*)p;
          428  +  return rc;
          429  +}
          430  +
          431  +static int expertDisconnect(sqlite3_vtab *pVtab){
          432  +  ExpertVtab *p = (ExpertVtab*)pVtab;
          433  +  sqlite3_free(p);
          434  +  return SQLITE_OK;
          435  +}
          436  +
          437  +static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
          438  +  ExpertVtab *p = (ExpertVtab*)pVtab;
          439  +  int rc = SQLITE_OK;
          440  +  int n = 0;
          441  +  IdxScan *pScan;
          442  +  const int opmask = 
          443  +    SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT |
          444  +    SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE |
          445  +    SQLITE_INDEX_CONSTRAINT_LE;
          446  +
          447  +  pScan = idxMalloc(&rc, sizeof(IdxScan));
          448  +  if( pScan ){
          449  +    int i;
          450  +
          451  +    /* Link the new scan object into the list */
          452  +    pScan->pTab = p->pTab;
          453  +    pScan->pNextScan = p->pExpert->pScan;
          454  +    p->pExpert->pScan = pScan;
          455  +
          456  +    /* Add the constraints to the IdxScan object */
          457  +    for(i=0; i<pIdxInfo->nConstraint; i++){
          458  +      struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
          459  +      if( pCons->usable 
          460  +       && pCons->iColumn>=0 
          461  +       && p->pTab->aCol[pCons->iColumn].iPk==0
          462  +       && (pCons->op & opmask) 
          463  +      ){
          464  +        IdxConstraint *pNew;
          465  +        const char *zColl = sqlite3_vtab_collation(pIdxInfo, i);
          466  +        pNew = idxNewConstraint(&rc, zColl);
          467  +        if( pNew ){
          468  +          pNew->iCol = pCons->iColumn;
          469  +          if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
          470  +            pNew->pNext = pScan->pEq;
          471  +            pScan->pEq = pNew;
          472  +          }else{
          473  +            pNew->bRange = 1;
          474  +            pNew->pNext = pScan->pRange;
          475  +            pScan->pRange = pNew;
          476  +          }
          477  +        }
          478  +        n++;
          479  +        pIdxInfo->aConstraintUsage[i].argvIndex = n;
          480  +      }
          481  +    }
          482  +
          483  +    /* Add the ORDER BY to the IdxScan object */
          484  +    for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
          485  +      int iCol = pIdxInfo->aOrderBy[i].iColumn;
          486  +      if( iCol>=0 ){
          487  +        IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
          488  +        if( pNew ){
          489  +          pNew->iCol = iCol;
          490  +          pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
          491  +          pNew->pNext = pScan->pOrder;
          492  +          pNew->pLink = pScan->pOrder;
          493  +          pScan->pOrder = pNew;
          494  +          n++;
          495  +        }
          496  +      }
          497  +    }
          498  +  }
          499  +
          500  +  pIdxInfo->estimatedCost = 1000000.0 / (n+1);
          501  +  return rc;
          502  +}
          503  +
          504  +static int expertUpdate(
          505  +  sqlite3_vtab *pVtab, 
          506  +  int nData, 
          507  +  sqlite3_value **azData, 
          508  +  sqlite_int64 *pRowid
          509  +){
          510  +  (void)pVtab;
          511  +  (void)nData;
          512  +  (void)azData;
          513  +  (void)pRowid;
          514  +  return SQLITE_OK;
          515  +}
          516  +
          517  +/* 
          518  +** Virtual table module xOpen method.
          519  +*/
          520  +static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
          521  +  int rc = SQLITE_OK;
          522  +  ExpertCsr *pCsr;
          523  +  (void)pVTab;
          524  +  pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
          525  +  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
          526  +  return rc;
          527  +}
          528  +
          529  +/* 
          530  +** Virtual table module xClose method.
          531  +*/
          532  +static int expertClose(sqlite3_vtab_cursor *cur){
          533  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          534  +  sqlite3_finalize(pCsr->pData);
          535  +  sqlite3_free(pCsr);
          536  +  return SQLITE_OK;
          537  +}
          538  +
          539  +/*
          540  +** Virtual table module xEof method.
          541  +**
          542  +** Return non-zero if the cursor does not currently point to a valid 
          543  +** record (i.e if the scan has finished), or zero otherwise.
          544  +*/
          545  +static int expertEof(sqlite3_vtab_cursor *cur){
          546  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          547  +  return pCsr->pData==0;
          548  +}
          549  +
          550  +/* 
          551  +** Virtual table module xNext method.
          552  +*/
          553  +static int expertNext(sqlite3_vtab_cursor *cur){
          554  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          555  +  int rc = SQLITE_OK;
          556  +
          557  +  assert( pCsr->pData );
          558  +  rc = sqlite3_step(pCsr->pData);
          559  +  if( rc!=SQLITE_ROW ){
          560  +    rc = sqlite3_finalize(pCsr->pData);
          561  +    pCsr->pData = 0;
          562  +  }else{
          563  +    rc = SQLITE_OK;
          564  +  }
          565  +
          566  +  return rc;
          567  +}
          568  +
          569  +/* 
          570  +** Virtual table module xRowid method.
          571  +*/
          572  +static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          573  +  (void)cur;
          574  +  *pRowid = 0;
          575  +  return SQLITE_OK;
          576  +}
          577  +
          578  +/* 
          579  +** Virtual table module xColumn method.
          580  +*/
          581  +static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
          582  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          583  +  sqlite3_value *pVal;
          584  +  pVal = sqlite3_column_value(pCsr->pData, i);
          585  +  if( pVal ){
          586  +    sqlite3_result_value(ctx, pVal);
          587  +  }
          588  +  return SQLITE_OK;
          589  +}
          590  +
          591  +/* 
          592  +** Virtual table module xFilter method.
          593  +*/
          594  +static int expertFilter(
          595  +  sqlite3_vtab_cursor *cur, 
          596  +  int idxNum, const char *idxStr,
          597  +  int argc, sqlite3_value **argv
          598  +){
          599  +  ExpertCsr *pCsr = (ExpertCsr*)cur;
          600  +  ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
          601  +  sqlite3expert *pExpert = pVtab->pExpert;
          602  +  int rc;
          603  +
          604  +  (void)idxNum;
          605  +  (void)idxStr;
          606  +  (void)argc;
          607  +  (void)argv;
          608  +  rc = sqlite3_finalize(pCsr->pData);
          609  +  pCsr->pData = 0;
          610  +  if( rc==SQLITE_OK ){
          611  +    rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
          612  +        "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
          613  +    );
          614  +  }
          615  +
          616  +  if( rc==SQLITE_OK ){
          617  +    rc = expertNext(cur);
          618  +  }
          619  +  return rc;
          620  +}
          621  +
          622  +static int idxRegisterVtab(sqlite3expert *p){
          623  +  static sqlite3_module expertModule = {
          624  +    2,                            /* iVersion */
          625  +    expertConnect,                /* xCreate - create a table */
          626  +    expertConnect,                /* xConnect - connect to an existing table */
          627  +    expertBestIndex,              /* xBestIndex - Determine search strategy */
          628  +    expertDisconnect,             /* xDisconnect - Disconnect from a table */
          629  +    expertDisconnect,             /* xDestroy - Drop a table */
          630  +    expertOpen,                   /* xOpen - open a cursor */
          631  +    expertClose,                  /* xClose - close a cursor */
          632  +    expertFilter,                 /* xFilter - configure scan constraints */
          633  +    expertNext,                   /* xNext - advance a cursor */
          634  +    expertEof,                    /* xEof */
          635  +    expertColumn,                 /* xColumn - read data */
          636  +    expertRowid,                  /* xRowid - read data */
          637  +    expertUpdate,                 /* xUpdate - write data */
          638  +    0,                            /* xBegin - begin transaction */
          639  +    0,                            /* xSync - sync transaction */
          640  +    0,                            /* xCommit - commit transaction */
          641  +    0,                            /* xRollback - rollback transaction */
          642  +    0,                            /* xFindFunction - function overloading */
          643  +    0,                            /* xRename - rename the table */
          644  +    0,                            /* xSavepoint */
          645  +    0,                            /* xRelease */
          646  +    0,                            /* xRollbackTo */
          647  +  };
          648  +
          649  +  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
          650  +}
          651  +/*
          652  +** End of virtual table implementation.
          653  +*************************************************************************/
          654  +/*
          655  +** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
          656  +** is called, set it to the return value of sqlite3_finalize() before
          657  +** returning. Otherwise, discard the sqlite3_finalize() return value.
          658  +*/
          659  +static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
          660  +  int rc = sqlite3_finalize(pStmt);
          661  +  if( *pRc==SQLITE_OK ) *pRc = rc;
          662  +}
          663  +
          664  +/*
          665  +** Attempt to allocate an IdxTable structure corresponding to table zTab
          666  +** in the main database of connection db. If successful, set (*ppOut) to
          667  +** point to the new object and return SQLITE_OK. Otherwise, return an
          668  +** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be
          669  +** set to point to an error string.
          670  +**
          671  +** It is the responsibility of the caller to eventually free either the
          672  +** IdxTable object or error message using sqlite3_free().
          673  +*/
          674  +static int idxGetTableInfo(
          675  +  sqlite3 *db,                    /* Database connection to read details from */
          676  +  const char *zTab,               /* Table name */
          677  +  IdxTable **ppOut,               /* OUT: New object (if successful) */
          678  +  char **pzErrmsg                 /* OUT: Error message (if not) */
          679  +){
          680  +  sqlite3_stmt *p1 = 0;
          681  +  int nCol = 0;
          682  +  int nTab = STRLEN(zTab);
          683  +  int nByte = sizeof(IdxTable) + nTab + 1;
          684  +  IdxTable *pNew = 0;
          685  +  int rc, rc2;
          686  +  char *pCsr = 0;
          687  +
          688  +  rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
          689  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
          690  +    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
          691  +    nByte += 1 + STRLEN(zCol);
          692  +    rc = sqlite3_table_column_metadata(
          693  +        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
          694  +    );
          695  +    nByte += 1 + STRLEN(zCol);
          696  +    nCol++;
          697  +  }
          698  +  rc2 = sqlite3_reset(p1);
          699  +  if( rc==SQLITE_OK ) rc = rc2;
          700  +
          701  +  nByte += sizeof(IdxColumn) * nCol;
          702  +  if( rc==SQLITE_OK ){
          703  +    pNew = idxMalloc(&rc, nByte);
          704  +  }
          705  +  if( rc==SQLITE_OK ){
          706  +    pNew->aCol = (IdxColumn*)&pNew[1];
          707  +    pNew->nCol = nCol;
          708  +    pCsr = (char*)&pNew->aCol[nCol];
          709  +  }
          710  +
          711  +  nCol = 0;
          712  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
          713  +    const char *zCol = (const char*)sqlite3_column_text(p1, 1);
          714  +    int nCopy = STRLEN(zCol) + 1;
          715  +    pNew->aCol[nCol].zName = pCsr;
          716  +    pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
          717  +    memcpy(pCsr, zCol, nCopy);
          718  +    pCsr += nCopy;
          719  +
          720  +    rc = sqlite3_table_column_metadata(
          721  +        db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
          722  +    );
          723  +    if( rc==SQLITE_OK ){
          724  +      nCopy = STRLEN(zCol) + 1;
          725  +      pNew->aCol[nCol].zColl = pCsr;
          726  +      memcpy(pCsr, zCol, nCopy);
          727  +      pCsr += nCopy;
          728  +    }
          729  +
          730  +    nCol++;
          731  +  }
          732  +  idxFinalize(&rc, p1);
          733  +
          734  +  if( rc!=SQLITE_OK ){
          735  +    sqlite3_free(pNew);
          736  +    pNew = 0;
          737  +  }else{
          738  +    pNew->zName = pCsr;
          739  +    memcpy(pNew->zName, zTab, nTab+1);
          740  +  }
          741  +
          742  +  *ppOut = pNew;
          743  +  return rc;
          744  +}
          745  +
          746  +/*
          747  +** This function is a no-op if *pRc is set to anything other than 
          748  +** SQLITE_OK when it is called.
          749  +**
          750  +** If *pRc is initially set to SQLITE_OK, then the text specified by
          751  +** the printf() style arguments is appended to zIn and the result returned
          752  +** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on
          753  +** zIn before returning.
          754  +*/
          755  +static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
          756  +  va_list ap;
          757  +  char *zAppend = 0;
          758  +  char *zRet = 0;
          759  +  int nIn = zIn ? STRLEN(zIn) : 0;
          760  +  int nAppend = 0;
          761  +  va_start(ap, zFmt);
          762  +  if( *pRc==SQLITE_OK ){
          763  +    zAppend = sqlite3_vmprintf(zFmt, ap);
          764  +    if( zAppend ){
          765  +      nAppend = STRLEN(zAppend);
          766  +      zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
          767  +    }
          768  +    if( zAppend && zRet ){
          769  +      if( nIn ) memcpy(zRet, zIn, nIn);
          770  +      memcpy(&zRet[nIn], zAppend, nAppend+1);
          771  +    }else{
          772  +      sqlite3_free(zRet);
          773  +      zRet = 0;
          774  +      *pRc = SQLITE_NOMEM;
          775  +    }
          776  +    sqlite3_free(zAppend);
          777  +    sqlite3_free(zIn);
          778  +  }
          779  +  va_end(ap);
          780  +  return zRet;
          781  +}
          782  +
          783  +/*
          784  +** Return true if zId must be quoted in order to use it as an SQL
          785  +** identifier, or false otherwise.
          786  +*/
          787  +static int idxIdentifierRequiresQuotes(const char *zId){
          788  +  int i;
          789  +  for(i=0; zId[i]; i++){
          790  +    if( !(zId[i]=='_')
          791  +     && !(zId[i]>='0' && zId[i]<='9')
          792  +     && !(zId[i]>='a' && zId[i]<='z')
          793  +     && !(zId[i]>='A' && zId[i]<='Z')
          794  +    ){
          795  +      return 1;
          796  +    }
          797  +  }
          798  +  return 0;
          799  +}
          800  +
          801  +/*
          802  +** This function appends an index column definition suitable for constraint
          803  +** pCons to the string passed as zIn and returns the result.
          804  +*/
          805  +static char *idxAppendColDefn(
          806  +  int *pRc,                       /* IN/OUT: Error code */
          807  +  char *zIn,                      /* Column defn accumulated so far */
          808  +  IdxTable *pTab,                 /* Table index will be created on */
          809  +  IdxConstraint *pCons
          810  +){
          811  +  char *zRet = zIn;
          812  +  IdxColumn *p = &pTab->aCol[pCons->iCol];
          813  +  if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
          814  +
          815  +  if( idxIdentifierRequiresQuotes(p->zName) ){
          816  +    zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
          817  +  }else{
          818  +    zRet = idxAppendText(pRc, zRet, "%s", p->zName);
          819  +  }
          820  +
          821  +  if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
          822  +    if( idxIdentifierRequiresQuotes(pCons->zColl) ){
          823  +      zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
          824  +    }else{
          825  +      zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
          826  +    }
          827  +  }
          828  +
          829  +  if( pCons->bDesc ){
          830  +    zRet = idxAppendText(pRc, zRet, " DESC");
          831  +  }
          832  +  return zRet;
          833  +}
          834  +
          835  +/*
          836  +** Search database dbm for an index compatible with the one idxCreateFromCons()
          837  +** would create from arguments pScan, pEq and pTail. If no error occurs and 
          838  +** such an index is found, return non-zero. Or, if no such index is found,
          839  +** return zero.
          840  +**
          841  +** If an error occurs, set *pRc to an SQLite error code and return zero.
          842  +*/
          843  +static int idxFindCompatible(
          844  +  int *pRc,                       /* OUT: Error code */
          845  +  sqlite3* dbm,                   /* Database to search */
          846  +  IdxScan *pScan,                 /* Scan for table to search for index on */
          847  +  IdxConstraint *pEq,             /* List of == constraints */
          848  +  IdxConstraint *pTail            /* List of range constraints */
          849  +){
          850  +  const char *zTbl = pScan->pTab->zName;
          851  +  sqlite3_stmt *pIdxList = 0;
          852  +  IdxConstraint *pIter;
          853  +  int nEq = 0;                    /* Number of elements in pEq */
          854  +  int rc;
          855  +
          856  +  /* Count the elements in list pEq */
          857  +  for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
          858  +
          859  +  rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
          860  +  while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
          861  +    int bMatch = 1;
          862  +    IdxConstraint *pT = pTail;
          863  +    sqlite3_stmt *pInfo = 0;
          864  +    const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
          865  +
          866  +    /* Zero the IdxConstraint.bFlag values in the pEq list */
          867  +    for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0;
          868  +
          869  +    rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
          870  +    while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
          871  +      int iIdx = sqlite3_column_int(pInfo, 0);
          872  +      int iCol = sqlite3_column_int(pInfo, 1);
          873  +      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
          874  +
          875  +      if( iIdx<nEq ){
          876  +        for(pIter=pEq; pIter; pIter=pIter->pLink){
          877  +          if( pIter->bFlag ) continue;
          878  +          if( pIter->iCol!=iCol ) continue;
          879  +          if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
          880  +          pIter->bFlag = 1;
          881  +          break;
          882  +        }
          883  +        if( pIter==0 ){
          884  +          bMatch = 0;
          885  +          break;
          886  +        }
          887  +      }else{
          888  +        if( pT ){
          889  +          if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){
          890  +            bMatch = 0;
          891  +            break;
          892  +          }
          893  +          pT = pT->pLink;
          894  +        }
          895  +      }
          896  +    }
          897  +    idxFinalize(&rc, pInfo);
          898  +
          899  +    if( rc==SQLITE_OK && bMatch ){
          900  +      sqlite3_finalize(pIdxList);
          901  +      return 1;
          902  +    }
          903  +  }
          904  +  idxFinalize(&rc, pIdxList);
          905  +
          906  +  *pRc = rc;
          907  +  return 0;
          908  +}
          909  +
          910  +static int idxCreateFromCons(
          911  +  sqlite3expert *p,
          912  +  IdxScan *pScan,
          913  +  IdxConstraint *pEq, 
          914  +  IdxConstraint *pTail
          915  +){
          916  +  sqlite3 *dbm = p->dbm;
          917  +  int rc = SQLITE_OK;
          918  +  if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
          919  +    IdxTable *pTab = pScan->pTab;
          920  +    char *zCols = 0;
          921  +    char *zIdx = 0;
          922  +    IdxConstraint *pCons;
          923  +    unsigned int h = 0;
          924  +    const char *zFmt;
          925  +
          926  +    for(pCons=pEq; pCons; pCons=pCons->pLink){
          927  +      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
          928  +    }
          929  +    for(pCons=pTail; pCons; pCons=pCons->pLink){
          930  +      zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
          931  +    }
          932  +
          933  +    if( rc==SQLITE_OK ){
          934  +      /* Hash the list of columns to come up with a name for the index */
          935  +      const char *zTable = pScan->pTab->zName;
          936  +      char *zName;                /* Index name */
          937  +      int i;
          938  +      for(i=0; zCols[i]; i++){
          939  +        h += ((h<<3) + zCols[i]);
          940  +      }
          941  +      zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
          942  +      if( zName==0 ){ 
          943  +        rc = SQLITE_NOMEM;
          944  +      }else{
          945  +        if( idxIdentifierRequiresQuotes(zTable) ){
          946  +          zFmt = "CREATE INDEX '%q' ON %Q(%s)";
          947  +        }else{
          948  +          zFmt = "CREATE INDEX %s ON %s(%s)";
          949  +        }
          950  +        zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols);
          951  +        if( !zIdx ){
          952  +          rc = SQLITE_NOMEM;
          953  +        }else{
          954  +          rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
          955  +          idxHashAdd(&rc, &p->hIdx, zName, zIdx);
          956  +        }
          957  +        sqlite3_free(zName);
          958  +        sqlite3_free(zIdx);
          959  +      }
          960  +    }
          961  +
          962  +    sqlite3_free(zCols);
          963  +  }
          964  +  return rc;
          965  +}
          966  +
          967  +/*
          968  +** Return true if list pList (linked by IdxConstraint.pLink) contains
          969  +** a constraint compatible with *p. Otherwise return false.
          970  +*/
          971  +static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
          972  +  IdxConstraint *pCmp;
          973  +  for(pCmp=pList; pCmp; pCmp=pCmp->pLink){
          974  +    if( p->iCol==pCmp->iCol ) return 1;
          975  +  }
          976  +  return 0;
          977  +}
          978  +
          979  +static int idxCreateFromWhere(
          980  +  sqlite3expert *p, 
          981  +  IdxScan *pScan,                 /* Create indexes for this scan */
          982  +  IdxConstraint *pTail            /* range/ORDER BY constraints for inclusion */
          983  +){
          984  +  IdxConstraint *p1 = 0;
          985  +  IdxConstraint *pCon;
          986  +  int rc;
          987  +
          988  +  /* Gather up all the == constraints. */
          989  +  for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
          990  +    if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
          991  +      pCon->pLink = p1;
          992  +      p1 = pCon;
          993  +    }
          994  +  }
          995  +
          996  +  /* Create an index using the == constraints collected above. And the
          997  +  ** range constraint/ORDER BY terms passed in by the caller, if any. */
          998  +  rc = idxCreateFromCons(p, pScan, p1, pTail);
          999  +
         1000  +  /* If no range/ORDER BY passed by the caller, create a version of the
         1001  +  ** index for each range constraint.  */
         1002  +  if( pTail==0 ){
         1003  +    for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
         1004  +      assert( pCon->pLink==0 );
         1005  +      if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
         1006  +        rc = idxCreateFromCons(p, pScan, p1, pCon);
         1007  +      }
         1008  +    }
         1009  +  }
         1010  +
         1011  +  return rc;
         1012  +}
         1013  +
         1014  +/*
         1015  +** Create candidate indexes in database [dbm] based on the data in 
         1016  +** linked-list pScan.
         1017  +*/
         1018  +static int idxCreateCandidates(sqlite3expert *p){
         1019  +  int rc = SQLITE_OK;
         1020  +  IdxScan *pIter;
         1021  +
         1022  +  for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
         1023  +    rc = idxCreateFromWhere(p, pIter, 0);
         1024  +    if( rc==SQLITE_OK && pIter->pOrder ){
         1025  +      rc = idxCreateFromWhere(p, pIter, pIter->pOrder);
         1026  +    }
         1027  +  }
         1028  +
         1029  +  return rc;
         1030  +}
         1031  +
         1032  +/*
         1033  +** Free all elements of the linked list starting at pConstraint.
         1034  +*/
         1035  +static void idxConstraintFree(IdxConstraint *pConstraint){
         1036  +  IdxConstraint *pNext;
         1037  +  IdxConstraint *p;
         1038  +
         1039  +  for(p=pConstraint; p; p=pNext){
         1040  +    pNext = p->pNext;
         1041  +    sqlite3_free(p);
         1042  +  }
         1043  +}
         1044  +
         1045  +/*
         1046  +** Free all elements of the linked list starting from pScan up until pLast
         1047  +** (pLast is not freed).
         1048  +*/
         1049  +static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
         1050  +  IdxScan *p;
         1051  +  IdxScan *pNext;
         1052  +  for(p=pScan; p!=pLast; p=pNext){
         1053  +    pNext = p->pNextScan;
         1054  +    idxConstraintFree(p->pOrder);
         1055  +    idxConstraintFree(p->pEq);
         1056  +    idxConstraintFree(p->pRange);
         1057  +    sqlite3_free(p);
         1058  +  }
         1059  +}
         1060  +
         1061  +/*
         1062  +** Free all elements of the linked list starting from pStatement up 
         1063  +** until pLast (pLast is not freed).
         1064  +*/
         1065  +static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
         1066  +  IdxStatement *p;
         1067  +  IdxStatement *pNext;
         1068  +  for(p=pStatement; p!=pLast; p=pNext){
         1069  +    pNext = p->pNext;
         1070  +    sqlite3_free(p->zEQP);
         1071  +    sqlite3_free(p->zIdx);
         1072  +    sqlite3_free(p);
         1073  +  }
         1074  +}
         1075  +
         1076  +/*
         1077  +** Free the linked list of IdxTable objects starting at pTab.
         1078  +*/
         1079  +static void idxTableFree(IdxTable *pTab){
         1080  +  IdxTable *pIter;
         1081  +  IdxTable *pNext;
         1082  +  for(pIter=pTab; pIter; pIter=pNext){
         1083  +    pNext = pIter->pNext;
         1084  +    sqlite3_free(pIter);
         1085  +  }
         1086  +}
         1087  +
         1088  +/*
         1089  +** Free the linked list of IdxWrite objects starting at pTab.
         1090  +*/
         1091  +static void idxWriteFree(IdxWrite *pTab){
         1092  +  IdxWrite *pIter;
         1093  +  IdxWrite *pNext;
         1094  +  for(pIter=pTab; pIter; pIter=pNext){
         1095  +    pNext = pIter->pNext;
         1096  +    sqlite3_free(pIter);
         1097  +  }
         1098  +}
         1099  +
         1100  +
         1101  +
         1102  +/*
         1103  +** This function is called after candidate indexes have been created. It
         1104  +** runs all the queries to see which indexes they prefer, and populates
         1105  +** IdxStatement.zIdx and IdxStatement.zEQP with the results.
         1106  +*/
         1107  +int idxFindIndexes(
         1108  +  sqlite3expert *p,
         1109  +  char **pzErr                         /* OUT: Error message (sqlite3_malloc) */
         1110  +){
         1111  +  IdxStatement *pStmt;
         1112  +  sqlite3 *dbm = p->dbm;
         1113  +  int rc = SQLITE_OK;
         1114  +
         1115  +  IdxHash hIdx;
         1116  +  idxHashInit(&hIdx);
         1117  +
         1118  +  for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
         1119  +    IdxHashEntry *pEntry;
         1120  +    sqlite3_stmt *pExplain = 0;
         1121  +    idxHashClear(&hIdx);
         1122  +    rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
         1123  +        "EXPLAIN QUERY PLAN %s", pStmt->zSql
         1124  +    );
         1125  +    while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
         1126  +      int iSelectid = sqlite3_column_int(pExplain, 0);
         1127  +      int iOrder = sqlite3_column_int(pExplain, 1);
         1128  +      int iFrom = sqlite3_column_int(pExplain, 2);
         1129  +      const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
         1130  +      int nDetail = STRLEN(zDetail);
         1131  +      int i;
         1132  +
         1133  +      for(i=0; i<nDetail; i++){
         1134  +        const char *zIdx = 0;
         1135  +        if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
         1136  +          zIdx = &zDetail[i+13];
         1137  +        }else if( memcmp(&zDetail[i], " USING COVERING INDEX ", 22)==0 ){
         1138  +          zIdx = &zDetail[i+22];
         1139  +        }
         1140  +        if( zIdx ){
         1141  +          const char *zSql;
         1142  +          int nIdx = 0;
         1143  +          while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){
         1144  +            nIdx++;
         1145  +          }
         1146  +          zSql = idxHashSearch(&p->hIdx, zIdx, nIdx);
         1147  +          if( zSql ){
         1148  +            idxHashAdd(&rc, &hIdx, zSql, 0);
         1149  +            if( rc ) goto find_indexes_out;
         1150  +          }
         1151  +          break;
         1152  +        }
         1153  +      }
         1154  +
         1155  +      pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
         1156  +          iSelectid, iOrder, iFrom, zDetail
         1157  +      );
         1158  +    }
         1159  +
         1160  +    for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
         1161  +      pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
         1162  +    }
         1163  +
         1164  +    idxFinalize(&rc, pExplain);
         1165  +  }
         1166  +
         1167  + find_indexes_out:
         1168  +  idxHashClear(&hIdx);
         1169  +  return rc;
         1170  +}
         1171  +
         1172  +static int idxAuthCallback(
         1173  +  void *pCtx,
         1174  +  int eOp,
         1175  +  const char *z3,
         1176  +  const char *z4,
         1177  +  const char *zDb,
         1178  +  const char *zTrigger
         1179  +){
         1180  +  int rc = SQLITE_OK;
         1181  +  (void)z4;
         1182  +  (void)zTrigger;
         1183  +  if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){
         1184  +    if( sqlite3_stricmp(zDb, "main")==0 ){
         1185  +      sqlite3expert *p = (sqlite3expert*)pCtx;
         1186  +      IdxTable *pTab;
         1187  +      for(pTab=p->pTable; pTab; pTab=pTab->pNext){
         1188  +        if( 0==sqlite3_stricmp(z3, pTab->zName) ) break;
         1189  +      }
         1190  +      if( pTab ){
         1191  +        IdxWrite *pWrite;
         1192  +        for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){
         1193  +          if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break;
         1194  +        }
         1195  +        if( pWrite==0 ){
         1196  +          pWrite = idxMalloc(&rc, sizeof(IdxWrite));
         1197  +          if( rc==SQLITE_OK ){
         1198  +            pWrite->pTab = pTab;
         1199  +            pWrite->eOp = eOp;
         1200  +            pWrite->pNext = p->pWrite;
         1201  +            p->pWrite = pWrite;
         1202  +          }
         1203  +        }
         1204  +      }
         1205  +    }
         1206  +  }
         1207  +  return rc;
         1208  +}
         1209  +
         1210  +static int idxProcessOneTrigger(
         1211  +  sqlite3expert *p, 
         1212  +  IdxWrite *pWrite, 
         1213  +  char **pzErr
         1214  +){
         1215  +  static const char *zInt = UNIQUE_TABLE_NAME;
         1216  +  static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME;
         1217  +  IdxTable *pTab = pWrite->pTab;
         1218  +  const char *zTab = pTab->zName;
         1219  +  const char *zSql = 
         1220  +    "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_master "
         1221  +    "WHERE tbl_name = %Q AND type IN ('table', 'trigger') "
         1222  +    "ORDER BY type;";
         1223  +  sqlite3_stmt *pSelect = 0;
         1224  +  int rc = SQLITE_OK;
         1225  +  char *zWrite = 0;
         1226  +
         1227  +  /* Create the table and its triggers in the temp schema */
         1228  +  rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab);
         1229  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){
         1230  +    const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0);
         1231  +    rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr);
         1232  +  }
         1233  +  idxFinalize(&rc, pSelect);
         1234  +
         1235  +  /* Rename the table in the temp schema to zInt */
         1236  +  if( rc==SQLITE_OK ){
         1237  +    char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt);
         1238  +    if( z==0 ){
         1239  +      rc = SQLITE_NOMEM;
         1240  +    }else{
         1241  +      rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr);
         1242  +      sqlite3_free(z);
         1243  +    }
         1244  +  }
         1245  +
         1246  +  switch( pWrite->eOp ){
         1247  +    case SQLITE_INSERT: {
         1248  +      int i;
         1249  +      zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt);
         1250  +      for(i=0; i<pTab->nCol; i++){
         1251  +        zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", ");
         1252  +      }
         1253  +      zWrite = idxAppendText(&rc, zWrite, ")");
         1254  +      break;
         1255  +    }
         1256  +    case SQLITE_UPDATE: {
         1257  +      int i;
         1258  +      zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt);
         1259  +      for(i=0; i<pTab->nCol; i++){
         1260  +        zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", 
         1261  +            pTab->aCol[i].zName
         1262  +        );
         1263  +      }
         1264  +      break;
         1265  +    }
         1266  +    default: {
         1267  +      assert( pWrite->eOp==SQLITE_DELETE );
         1268  +      if( rc==SQLITE_OK ){
         1269  +        zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt);
         1270  +        if( zWrite==0 ) rc = SQLITE_NOMEM;
         1271  +      }
         1272  +    }
         1273  +  }
         1274  +
         1275  +  if( rc==SQLITE_OK ){
         1276  +    sqlite3_stmt *pX = 0;
         1277  +    rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
         1278  +    idxFinalize(&rc, pX);
         1279  +    if( rc!=SQLITE_OK ){
         1280  +      idxDatabaseError(p->dbv, pzErr);
         1281  +    }
         1282  +  }
         1283  +  sqlite3_free(zWrite);
         1284  +
         1285  +  if( rc==SQLITE_OK ){
         1286  +    rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr);
         1287  +  }
         1288  +
         1289  +  return rc;
         1290  +}
         1291  +
         1292  +static int idxProcessTriggers(sqlite3expert *p, char **pzErr){
         1293  +  int rc = SQLITE_OK;
         1294  +  IdxWrite *pEnd = 0;
         1295  +  IdxWrite *pFirst = p->pWrite;
         1296  +
         1297  +  while( rc==SQLITE_OK && pFirst!=pEnd ){
         1298  +    IdxWrite *pIter;
         1299  +    for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){
         1300  +      rc = idxProcessOneTrigger(p, pIter, pzErr);
         1301  +    }
         1302  +    pEnd = pFirst;
         1303  +    pFirst = p->pWrite;
         1304  +  }
         1305  +
         1306  +  return rc;
         1307  +}
         1308  +
         1309  +
         1310  +static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
         1311  +  int rc = idxRegisterVtab(p);
         1312  +  sqlite3_stmt *pSchema = 0;
         1313  +
         1314  +  /* For each table in the main db schema:
         1315  +  **
         1316  +  **   1) Add an entry to the p->pTable list, and
         1317  +  **   2) Create the equivalent virtual table in dbv.
         1318  +  */
         1319  +  rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
         1320  +      "SELECT type, name, sql, 1 FROM sqlite_master "
         1321  +      "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' "
         1322  +      " UNION ALL "
         1323  +      "SELECT type, name, sql, 2 FROM sqlite_master "
         1324  +      "WHERE type = 'trigger'"
         1325  +      "  AND tbl_name IN(SELECT name FROM sqlite_master WHERE type = 'view') "
         1326  +      "ORDER BY 4, 1"
         1327  +  );
         1328  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
         1329  +    const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
         1330  +    const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
         1331  +    const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
         1332  +
         1333  +    if( zType[0]=='v' || zType[1]=='r' ){
         1334  +      rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
         1335  +    }else{
         1336  +      IdxTable *pTab;
         1337  +      rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
         1338  +      if( rc==SQLITE_OK ){
         1339  +        int i;
         1340  +        char *zInner = 0;
         1341  +        char *zOuter = 0;
         1342  +        pTab->pNext = p->pTable;
         1343  +        p->pTable = pTab;
         1344  +
         1345  +        /* The statement the vtab will pass to sqlite3_declare_vtab() */
         1346  +        zInner = idxAppendText(&rc, 0, "CREATE TABLE x(");
         1347  +        for(i=0; i<pTab->nCol; i++){
         1348  +          zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", 
         1349  +              (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl
         1350  +          );
         1351  +        }
         1352  +        zInner = idxAppendText(&rc, zInner, ")");
         1353  +
         1354  +        /* The CVT statement to create the vtab */
         1355  +        zOuter = idxAppendText(&rc, 0, 
         1356  +            "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner
         1357  +        );
         1358  +        if( rc==SQLITE_OK ){
         1359  +          rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg);
         1360  +        }
         1361  +        sqlite3_free(zInner);
         1362  +        sqlite3_free(zOuter);
         1363  +      }
         1364  +    }
         1365  +  }
         1366  +  idxFinalize(&rc, pSchema);
         1367  +  return rc;
         1368  +}
         1369  +
         1370  +struct IdxSampleCtx {
         1371  +  int iTarget;
         1372  +  double target;                  /* Target nRet/nRow value */
         1373  +  double nRow;                    /* Number of rows seen */
         1374  +  double nRet;                    /* Number of rows returned */
         1375  +};
         1376  +
         1377  +static void idxSampleFunc(
         1378  +  sqlite3_context *pCtx,
         1379  +  int argc,
         1380  +  sqlite3_value **argv
         1381  +){
         1382  +  struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
         1383  +  int bRet;
         1384  +
         1385  +  (void)argv;
         1386  +  assert( argc==0 );
         1387  +  if( p->nRow==0.0 ){
         1388  +    bRet = 1;
         1389  +  }else{
         1390  +    bRet = (p->nRet / p->nRow) <= p->target;
         1391  +    if( bRet==0 ){
         1392  +      unsigned short rnd;
         1393  +      sqlite3_randomness(2, (void*)&rnd);
         1394  +      bRet = ((int)rnd % 100) <= p->iTarget;
         1395  +    }
         1396  +  }
         1397  +
         1398  +  sqlite3_result_int(pCtx, bRet);
         1399  +  p->nRow += 1.0;
         1400  +  p->nRet += (double)bRet;
         1401  +}
         1402  +
         1403  +struct IdxRemCtx {
         1404  +  int nSlot;
         1405  +  struct IdxRemSlot {
         1406  +    int eType;                    /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */
         1407  +    i64 iVal;                     /* SQLITE_INTEGER value */
         1408  +    double rVal;                  /* SQLITE_FLOAT value */
         1409  +    int nByte;                    /* Bytes of space allocated at z */
         1410  +    int n;                        /* Size of buffer z */
         1411  +    char *z;                      /* SQLITE_TEXT/BLOB value */
         1412  +  } aSlot[1];
         1413  +};
         1414  +
         1415  +/*
         1416  +** Implementation of scalar function rem().
         1417  +*/
         1418  +static void idxRemFunc(
         1419  +  sqlite3_context *pCtx,
         1420  +  int argc,
         1421  +  sqlite3_value **argv
         1422  +){
         1423  +  struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx);
         1424  +  struct IdxRemSlot *pSlot;
         1425  +  int iSlot;
         1426  +  assert( argc==2 );
         1427  +
         1428  +  iSlot = sqlite3_value_int(argv[0]);
         1429  +  assert( iSlot<=p->nSlot );
         1430  +  pSlot = &p->aSlot[iSlot];
         1431  +
         1432  +  switch( pSlot->eType ){
         1433  +    case SQLITE_NULL:
         1434  +      /* no-op */
         1435  +      break;
         1436  +
         1437  +    case SQLITE_INTEGER:
         1438  +      sqlite3_result_int64(pCtx, pSlot->iVal);
         1439  +      break;
         1440  +
         1441  +    case SQLITE_FLOAT:
         1442  +      sqlite3_result_double(pCtx, pSlot->rVal);
         1443  +      break;
         1444  +
         1445  +    case SQLITE_BLOB:
         1446  +      sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
         1447  +      break;
         1448  +
         1449  +    case SQLITE_TEXT:
         1450  +      sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
         1451  +      break;
         1452  +  }
         1453  +
         1454  +  pSlot->eType = sqlite3_value_type(argv[1]);
         1455  +  switch( pSlot->eType ){
         1456  +    case SQLITE_NULL:
         1457  +      /* no-op */
         1458  +      break;
         1459  +
         1460  +    case SQLITE_INTEGER:
         1461  +      pSlot->iVal = sqlite3_value_int64(argv[1]);
         1462  +      break;
         1463  +
         1464  +    case SQLITE_FLOAT:
         1465  +      pSlot->rVal = sqlite3_value_double(argv[1]);
         1466  +      break;
         1467  +
         1468  +    case SQLITE_BLOB:
         1469  +    case SQLITE_TEXT: {
         1470  +      int nByte = sqlite3_value_bytes(argv[1]);
         1471  +      if( nByte>pSlot->nByte ){
         1472  +        char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2);
         1473  +        if( zNew==0 ){
         1474  +          sqlite3_result_error_nomem(pCtx);
         1475  +          return;
         1476  +        }
         1477  +        pSlot->nByte = nByte*2;
         1478  +        pSlot->z = zNew;
         1479  +      }
         1480  +      pSlot->n = nByte;
         1481  +      if( pSlot->eType==SQLITE_BLOB ){
         1482  +        memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte);
         1483  +      }else{
         1484  +        memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte);
         1485  +      }
         1486  +      break;
         1487  +    }
         1488  +  }
         1489  +}
         1490  +
         1491  +static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){
         1492  +  int rc = SQLITE_OK;
         1493  +  const char *zMax = 
         1494  +    "SELECT max(i.seqno) FROM "
         1495  +    "  sqlite_master AS s, "
         1496  +    "  pragma_index_list(s.name) AS l, "
         1497  +    "  pragma_index_info(l.name) AS i "
         1498  +    "WHERE s.type = 'table'";
         1499  +  sqlite3_stmt *pMax = 0;
         1500  +
         1501  +  *pnMax = 0;
         1502  +  rc = idxPrepareStmt(db, &pMax, pzErr, zMax);
         1503  +  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
         1504  +    *pnMax = sqlite3_column_int(pMax, 0) + 1;
         1505  +  }
         1506  +  idxFinalize(&rc, pMax);
         1507  +
         1508  +  return rc;
         1509  +}
         1510  +
         1511  +static int idxPopulateOneStat1(
         1512  +  sqlite3expert *p,
         1513  +  sqlite3_stmt *pIndexXInfo,
         1514  +  sqlite3_stmt *pWriteStat,
         1515  +  const char *zTab,
         1516  +  const char *zIdx,
         1517  +  char **pzErr
         1518  +){
         1519  +  char *zCols = 0;
         1520  +  char *zOrder = 0;
         1521  +  char *zQuery = 0;
         1522  +  int nCol = 0;
         1523  +  int i;
         1524  +  sqlite3_stmt *pQuery = 0;
         1525  +  int *aStat = 0;
         1526  +  int rc = SQLITE_OK;
         1527  +
         1528  +  assert( p->iSample>0 );
         1529  +
         1530  +  /* Formulate the query text */
         1531  +  sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
         1532  +  while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
         1533  +    const char *zComma = zCols==0 ? "" : ", ";
         1534  +    const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
         1535  +    const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
         1536  +    zCols = idxAppendText(&rc, zCols, 
         1537  +        "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
         1538  +    );
         1539  +    zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
         1540  +  }
         1541  +  sqlite3_reset(pIndexXInfo);
         1542  +  if( rc==SQLITE_OK ){
         1543  +    if( p->iSample==100 ){
         1544  +      zQuery = sqlite3_mprintf(
         1545  +          "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
         1546  +      );
         1547  +    }else{
         1548  +      zQuery = sqlite3_mprintf(
         1549  +          "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder
         1550  +      );
         1551  +    }
         1552  +  }
         1553  +  sqlite3_free(zCols);
         1554  +  sqlite3_free(zOrder);
         1555  +
         1556  +  /* Formulate the query text */
         1557  +  if( rc==SQLITE_OK ){
         1558  +    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
         1559  +    rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
         1560  +  }
         1561  +  sqlite3_free(zQuery);
         1562  +
         1563  +  if( rc==SQLITE_OK ){
         1564  +    aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1));
         1565  +  }
         1566  +  if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
         1567  +    IdxHashEntry *pEntry;
         1568  +    char *zStat = 0;
         1569  +    for(i=0; i<=nCol; i++) aStat[i] = 1;
         1570  +    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
         1571  +      aStat[0]++;
         1572  +      for(i=0; i<nCol; i++){
         1573  +        if( sqlite3_column_int(pQuery, i)==0 ) break;
         1574  +      }
         1575  +      for(/*no-op*/; i<nCol; i++){
         1576  +        aStat[i+1]++;
         1577  +      }
         1578  +    }
         1579  +
         1580  +    if( rc==SQLITE_OK ){
         1581  +      int s0 = aStat[0];
         1582  +      zStat = sqlite3_mprintf("%d", s0);
         1583  +      if( zStat==0 ) rc = SQLITE_NOMEM;
         1584  +      for(i=1; rc==SQLITE_OK && i<=nCol; i++){
         1585  +        zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]);
         1586  +      }
         1587  +    }
         1588  +
         1589  +    if( rc==SQLITE_OK ){
         1590  +      sqlite3_bind_text(pWriteStat, 1, zTab, -1, SQLITE_STATIC);
         1591  +      sqlite3_bind_text(pWriteStat, 2, zIdx, -1, SQLITE_STATIC);
         1592  +      sqlite3_bind_text(pWriteStat, 3, zStat, -1, SQLITE_STATIC);
         1593  +      sqlite3_step(pWriteStat);
         1594  +      rc = sqlite3_reset(pWriteStat);
         1595  +    }
         1596  +
         1597  +    pEntry = idxHashFind(&p->hIdx, zIdx, STRLEN(zIdx));
         1598  +    if( pEntry ){
         1599  +      assert( pEntry->zVal2==0 );
         1600  +      pEntry->zVal2 = zStat;
         1601  +    }else{
         1602  +      sqlite3_free(zStat);
         1603  +    }
         1604  +  }
         1605  +  sqlite3_free(aStat);
         1606  +  idxFinalize(&rc, pQuery);
         1607  +
         1608  +  return rc;
         1609  +}
         1610  +
         1611  +static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
         1612  +  int rc;
         1613  +  char *zSql;
         1614  +
         1615  +  rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
         1616  +  if( rc!=SQLITE_OK ) return rc;
         1617  +
         1618  +  zSql = sqlite3_mprintf(
         1619  +      "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
         1620  +  );
         1621  +  if( zSql==0 ) return SQLITE_NOMEM;
         1622  +  rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
         1623  +  sqlite3_free(zSql);
         1624  +
         1625  +  return rc;
         1626  +}
         1627  +
         1628  +/*
         1629  +** This function is called as part of sqlite3_expert_analyze(). Candidate
         1630  +** indexes have already been created in database sqlite3expert.dbm, this
         1631  +** function populates sqlite_stat1 table in the same database.
         1632  +**
         1633  +** The stat1 data is generated by querying the 
         1634  +*/
         1635  +static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
         1636  +  int rc = SQLITE_OK;
         1637  +  int nMax =0;
         1638  +  struct IdxRemCtx *pCtx = 0;
         1639  +  struct IdxSampleCtx samplectx; 
         1640  +  int i;
         1641  +  i64 iPrev = -100000;
         1642  +  sqlite3_stmt *pAllIndex = 0;
         1643  +  sqlite3_stmt *pIndexXInfo = 0;
         1644  +  sqlite3_stmt *pWrite = 0;
         1645  +
         1646  +  const char *zAllIndex =
         1647  +    "SELECT s.rowid, s.name, l.name FROM "
         1648  +    "  sqlite_master AS s, "
         1649  +    "  pragma_index_list(s.name) AS l "
         1650  +    "WHERE s.type = 'table'";
         1651  +  const char *zIndexXInfo = 
         1652  +    "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
         1653  +  const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
         1654  +
         1655  +  /* If iSample==0, no sqlite_stat1 data is required. */
         1656  +  if( p->iSample==0 ) return SQLITE_OK;
         1657  +
         1658  +  rc = idxLargestIndex(p->dbm, &nMax, pzErr);
         1659  +  if( nMax<=0 || rc!=SQLITE_OK ) return rc;
         1660  +
         1661  +  rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0);
         1662  +
         1663  +  if( rc==SQLITE_OK ){
         1664  +    int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax);
         1665  +    pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte);
         1666  +  }
         1667  +
         1668  +  if( rc==SQLITE_OK ){
         1669  +    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
         1670  +    rc = sqlite3_create_function(
         1671  +        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
         1672  +    );
         1673  +  }
         1674  +  if( rc==SQLITE_OK ){
         1675  +    rc = sqlite3_create_function(
         1676  +        p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
         1677  +    );
         1678  +  }
         1679  +
         1680  +  if( rc==SQLITE_OK ){
         1681  +    pCtx->nSlot = nMax+1;
         1682  +    rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex);
         1683  +  }
         1684  +  if( rc==SQLITE_OK ){
         1685  +    rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo);
         1686  +  }
         1687  +  if( rc==SQLITE_OK ){
         1688  +    rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite);
         1689  +  }
         1690  +
         1691  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
         1692  +    i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
         1693  +    const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
         1694  +    const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
         1695  +    if( p->iSample<100 && iPrev!=iRowid ){
         1696  +      samplectx.target = (double)p->iSample / 100.0;
         1697  +      samplectx.iTarget = p->iSample;
         1698  +      samplectx.nRow = 0.0;
         1699  +      samplectx.nRet = 0.0;
         1700  +      rc = idxBuildSampleTable(p, zTab);
         1701  +      if( rc!=SQLITE_OK ) break;
         1702  +    }
         1703  +    rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
         1704  +    iPrev = iRowid;
         1705  +  }
         1706  +  if( rc==SQLITE_OK && p->iSample<100 ){
         1707  +    rc = sqlite3_exec(p->dbv, 
         1708  +        "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0
         1709  +    );
         1710  +  }
         1711  +
         1712  +  idxFinalize(&rc, pAllIndex);
         1713  +  idxFinalize(&rc, pIndexXInfo);
         1714  +  idxFinalize(&rc, pWrite);
         1715  +
         1716  +  for(i=0; i<pCtx->nSlot; i++){
         1717  +    sqlite3_free(pCtx->aSlot[i].z);
         1718  +  }
         1719  +  sqlite3_free(pCtx);
         1720  +
         1721  +  if( rc==SQLITE_OK ){
         1722  +    rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
         1723  +  }
         1724  +
         1725  +  sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
         1726  +  return rc;
         1727  +}
         1728  +
         1729  +/*
         1730  +** Allocate a new sqlite3expert object.
         1731  +*/
         1732  +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
         1733  +  int rc = SQLITE_OK;
         1734  +  sqlite3expert *pNew;
         1735  +
         1736  +  pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
         1737  +
         1738  +  /* Open two in-memory databases to work with. The "vtab database" (dbv)
         1739  +  ** will contain a virtual table corresponding to each real table in
         1740  +  ** the user database schema, and a copy of each view. It is used to
         1741  +  ** collect information regarding the WHERE, ORDER BY and other clauses
         1742  +  ** of the user's query.
         1743  +  */
         1744  +  if( rc==SQLITE_OK ){
         1745  +    pNew->db = db;
         1746  +    pNew->iSample = 100;
         1747  +    rc = sqlite3_open(":memory:", &pNew->dbv);
         1748  +  }
         1749  +  if( rc==SQLITE_OK ){
         1750  +    rc = sqlite3_open(":memory:", &pNew->dbm);
         1751  +    if( rc==SQLITE_OK ){
         1752  +      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
         1753  +    }
         1754  +  }
         1755  +  
         1756  +
         1757  +  /* Copy the entire schema of database [db] into [dbm]. */
         1758  +  if( rc==SQLITE_OK ){
         1759  +    sqlite3_stmt *pSql;
         1760  +    rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         1761  +        "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
         1762  +        " AND sql NOT LIKE 'CREATE VIRTUAL %%'"
         1763  +    );
         1764  +    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
         1765  +      const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
         1766  +      rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
         1767  +    }
         1768  +    idxFinalize(&rc, pSql);
         1769  +  }
         1770  +
         1771  +  /* Create the vtab schema */
         1772  +  if( rc==SQLITE_OK ){
         1773  +    rc = idxCreateVtabSchema(pNew, pzErrmsg);
         1774  +  }
         1775  +
         1776  +  /* Register the auth callback with dbv */
         1777  +  if( rc==SQLITE_OK ){
         1778  +    sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew);
         1779  +  }
         1780  +
         1781  +  /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
         1782  +  ** return the new sqlite3expert handle.  */
         1783  +  if( rc!=SQLITE_OK ){
         1784  +    sqlite3_expert_destroy(pNew);
         1785  +    pNew = 0;
         1786  +  }
         1787  +  return pNew;
         1788  +}
         1789  +
         1790  +/*
         1791  +** Configure an sqlite3expert object.
         1792  +*/
         1793  +int sqlite3_expert_config(sqlite3expert *p, int op, ...){
         1794  +  int rc = SQLITE_OK;
         1795  +  va_list ap;
         1796  +  va_start(ap, op);
         1797  +  switch( op ){
         1798  +    case EXPERT_CONFIG_SAMPLE: {
         1799  +      int iVal = va_arg(ap, int);
         1800  +      if( iVal<0 ) iVal = 0;
         1801  +      if( iVal>100 ) iVal = 100;
         1802  +      p->iSample = iVal;
         1803  +      break;
         1804  +    }
         1805  +    default:
         1806  +      rc = SQLITE_NOTFOUND;
         1807  +      break;
         1808  +  }
         1809  +
         1810  +  va_end(ap);
         1811  +  return rc;
         1812  +}
         1813  +
         1814  +/*
         1815  +** Add an SQL statement to the analysis.
         1816  +*/
         1817  +int sqlite3_expert_sql(
         1818  +  sqlite3expert *p,               /* From sqlite3_expert_new() */
         1819  +  const char *zSql,               /* SQL statement to add */
         1820  +  char **pzErr                    /* OUT: Error message (if any) */
         1821  +){
         1822  +  IdxScan *pScanOrig = p->pScan;
         1823  +  IdxStatement *pStmtOrig = p->pStatement;
         1824  +  int rc = SQLITE_OK;
         1825  +  const char *zStmt = zSql;
         1826  +
         1827  +  if( p->bRun ) return SQLITE_MISUSE;
         1828  +
         1829  +  while( rc==SQLITE_OK && zStmt && zStmt[0] ){
         1830  +    sqlite3_stmt *pStmt = 0;
         1831  +    rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
         1832  +    if( rc==SQLITE_OK ){
         1833  +      if( pStmt ){
         1834  +        IdxStatement *pNew;
         1835  +        const char *z = sqlite3_sql(pStmt);
         1836  +        int n = STRLEN(z);
         1837  +        pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
         1838  +        if( rc==SQLITE_OK ){
         1839  +          pNew->zSql = (char*)&pNew[1];
         1840  +          memcpy(pNew->zSql, z, n+1);
         1841  +          pNew->pNext = p->pStatement;
         1842  +          if( p->pStatement ) pNew->iId = p->pStatement->iId+1;
         1843  +          p->pStatement = pNew;
         1844  +        }
         1845  +        sqlite3_finalize(pStmt);
         1846  +      }
         1847  +    }else{
         1848  +      idxDatabaseError(p->dbv, pzErr);
         1849  +    }
         1850  +  }
         1851  +
         1852  +  if( rc!=SQLITE_OK ){
         1853  +    idxScanFree(p->pScan, pScanOrig);
         1854  +    idxStatementFree(p->pStatement, pStmtOrig);
         1855  +    p->pScan = pScanOrig;
         1856  +    p->pStatement = pStmtOrig;
         1857  +  }
         1858  +
         1859  +  return rc;
         1860  +}
         1861  +
         1862  +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
         1863  +  int rc;
         1864  +  IdxHashEntry *pEntry;
         1865  +
         1866  +  /* Do trigger processing to collect any extra IdxScan structures */
         1867  +  rc = idxProcessTriggers(p, pzErr);
         1868  +
         1869  +  /* Create candidate indexes within the in-memory database file */
         1870  +  if( rc==SQLITE_OK ){
         1871  +    rc = idxCreateCandidates(p);
         1872  +  }
         1873  +
         1874  +  /* Generate the stat1 data */
         1875  +  if( rc==SQLITE_OK ){
         1876  +    rc = idxPopulateStat1(p, pzErr);
         1877  +  }
         1878  +
         1879  +  /* Formulate the EXPERT_REPORT_CANDIDATES text */
         1880  +  for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
         1881  +    p->zCandidates = idxAppendText(&rc, p->zCandidates, 
         1882  +        "%s;%s%s\n", pEntry->zVal, 
         1883  +        pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2
         1884  +    );
         1885  +  }
         1886  +
         1887  +  /* Figure out which of the candidate indexes are preferred by the query
         1888  +  ** planner and report the results to the user.  */
         1889  +  if( rc==SQLITE_OK ){
         1890  +    rc = idxFindIndexes(p, pzErr);
         1891  +  }
         1892  +
         1893  +  if( rc==SQLITE_OK ){
         1894  +    p->bRun = 1;
         1895  +  }
         1896  +  return rc;
         1897  +}
         1898  +
         1899  +/*
         1900  +** Return the total number of statements that have been added to this
         1901  +** sqlite3expert using sqlite3_expert_sql().
         1902  +*/
         1903  +int sqlite3_expert_count(sqlite3expert *p){
         1904  +  int nRet = 0;
         1905  +  if( p->pStatement ) nRet = p->pStatement->iId+1;
         1906  +  return nRet;
         1907  +}
         1908  +
         1909  +/*
         1910  +** Return a component of the report.
         1911  +*/
         1912  +const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
         1913  +  const char *zRet = 0;
         1914  +  IdxStatement *pStmt;
         1915  +
         1916  +  if( p->bRun==0 ) return 0;
         1917  +  for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
         1918  +  switch( eReport ){
         1919  +    case EXPERT_REPORT_SQL:
         1920  +      if( pStmt ) zRet = pStmt->zSql;
         1921  +      break;
         1922  +    case EXPERT_REPORT_INDEXES:
         1923  +      if( pStmt ) zRet = pStmt->zIdx;
         1924  +      break;
         1925  +    case EXPERT_REPORT_PLAN:
         1926  +      if( pStmt ) zRet = pStmt->zEQP;
         1927  +      break;
         1928  +    case EXPERT_REPORT_CANDIDATES:
         1929  +      zRet = p->zCandidates;
         1930  +      break;
         1931  +  }
         1932  +  return zRet;
         1933  +}
         1934  +
         1935  +/*
         1936  +** Free an sqlite3expert object.
         1937  +*/
         1938  +void sqlite3_expert_destroy(sqlite3expert *p){
         1939  +  if( p ){
         1940  +    sqlite3_close(p->dbm);
         1941  +    sqlite3_close(p->dbv);
         1942  +    idxScanFree(p->pScan, 0);
         1943  +    idxStatementFree(p->pStatement, 0);
         1944  +    idxTableFree(p->pTable);
         1945  +    idxWriteFree(p->pWrite);
         1946  +    idxHashClear(&p->hIdx);
         1947  +    sqlite3_free(p->zCandidates);
         1948  +    sqlite3_free(p);
         1949  +  }
         1950  +}
         1951  +
         1952  +#endif /* ifndef SQLITE_OMIT_VIRTUAL_TABLE */

Added ext/expert/sqlite3expert.h.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +
           15  +#include "sqlite3.h"
           16  +
           17  +typedef struct sqlite3expert sqlite3expert;
           18  +
           19  +/*
           20  +** Create a new sqlite3expert object.
           21  +**
           22  +** If successful, a pointer to the new object is returned and (*pzErr) set
           23  +** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
           24  +** an English-language error message. In this case it is the responsibility
           25  +** of the caller to eventually free the error message buffer using
           26  +** sqlite3_free().
           27  +*/
           28  +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
           29  +
           30  +/*
           31  +** Configure an sqlite3expert object.
           32  +**
           33  +** EXPERT_CONFIG_SAMPLE:
           34  +**   By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
           35  +**   each candidate index. This involves scanning and sorting the entire
           36  +**   contents of each user database table once for each candidate index
           37  +**   associated with the table. For large databases, this can be 
           38  +**   prohibitively slow. This option allows the sqlite3expert object to
           39  +**   be configured so that sqlite_stat1 data is instead generated based on a
           40  +**   subset of each table, or so that no sqlite_stat1 data is used at all.
           41  +**
           42  +**   A single integer argument is passed to this option. If the value is less
           43  +**   than or equal to zero, then no sqlite_stat1 data is generated or used by
           44  +**   the analysis - indexes are recommended based on the database schema only.
           45  +**   Or, if the value is 100 or greater, complete sqlite_stat1 data is
           46  +**   generated for each candidate index (this is the default). Finally, if the
           47  +**   value falls between 0 and 100, then it represents the percentage of user
           48  +**   table rows that should be considered when generating sqlite_stat1 data.
           49  +**
           50  +**   Examples:
           51  +**
           52  +**     // Do not generate any sqlite_stat1 data
           53  +**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
           54  +**
           55  +**     // Generate sqlite_stat1 data based on 10% of the rows in each table.
           56  +**     sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
           57  +*/
           58  +int sqlite3_expert_config(sqlite3expert *p, int op, ...);
           59  +
           60  +#define EXPERT_CONFIG_SAMPLE 1    /* int */
           61  +
           62  +/*
           63  +** Specify zero or more SQL statements to be included in the analysis.
           64  +**
           65  +** Buffer zSql must contain zero or more complete SQL statements. This
           66  +** function parses all statements contained in the buffer and adds them
           67  +** to the internal list of statements to analyze. If successful, SQLITE_OK
           68  +** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
           69  +** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
           70  +** may be set to point to an English language error message. In this case
           71  +** the caller is responsible for eventually freeing the error message buffer
           72  +** using sqlite3_free().
           73  +**
           74  +** If an error does occur while processing one of the statements in the
           75  +** buffer passed as the second argument, none of the statements in the
           76  +** buffer are added to the analysis.
           77  +**
           78  +** This function must be called before sqlite3_expert_analyze(). If a call
           79  +** to this function is made on an sqlite3expert object that has already
           80  +** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
           81  +** immediately and no statements are added to the analysis.
           82  +*/
           83  +int sqlite3_expert_sql(
           84  +  sqlite3expert *p,               /* From a successful sqlite3_expert_new() */
           85  +  const char *zSql,               /* SQL statement(s) to add */
           86  +  char **pzErr                    /* OUT: Error message (if any) */
           87  +);
           88  +
           89  +
           90  +/*
           91  +** This function is called after the sqlite3expert object has been configured
           92  +** with all SQL statements using sqlite3_expert_sql() to actually perform
           93  +** the analysis. Once this function has been called, it is not possible to
           94  +** add further SQL statements to the analysis.
           95  +**
           96  +** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
           97  +** an error occurs, an SQLite error code is returned and (*pzErr) set to 
           98  +** point to a buffer containing an English language error message. In this
           99  +** case it is the responsibility of the caller to eventually free the buffer
          100  +** using sqlite3_free().
          101  +**
          102  +** If an error does occur within this function, the sqlite3expert object
          103  +** is no longer useful for any purpose. At that point it is no longer
          104  +** possible to add further SQL statements to the object or to re-attempt
          105  +** the analysis. The sqlite3expert object must still be freed using a call
          106  +** sqlite3_expert_destroy().
          107  +*/
          108  +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
          109  +
          110  +/*
          111  +** Return the total number of statements loaded using sqlite3_expert_sql().
          112  +** The total number of SQL statements may be different from the total number
          113  +** to calls to sqlite3_expert_sql().
          114  +*/
          115  +int sqlite3_expert_count(sqlite3expert*);
          116  +
          117  +/*
          118  +** Return a component of the report.
          119  +**
          120  +** This function is called after sqlite3_expert_analyze() to extract the
          121  +** results of the analysis. Each call to this function returns either a
          122  +** NULL pointer or a pointer to a buffer containing a nul-terminated string.
          123  +** The value passed as the third argument must be one of the EXPERT_REPORT_*
          124  +** #define constants defined below.
          125  +**
          126  +** For some EXPERT_REPORT_* parameters, the buffer returned contains 
          127  +** information relating to a specific SQL statement. In these cases that
          128  +** SQL statement is identified by the value passed as the second argument.
          129  +** SQL statements are numbered from 0 in the order in which they are parsed.
          130  +** If an out-of-range value (less than zero or equal to or greater than the
          131  +** value returned by sqlite3_expert_count()) is passed as the second argument
          132  +** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
          133  +**
          134  +** EXPERT_REPORT_SQL:
          135  +**   Return the text of SQL statement iStmt.
          136  +**
          137  +** EXPERT_REPORT_INDEXES:
          138  +**   Return a buffer containing the CREATE INDEX statements for all recommended
          139  +**   indexes for statement iStmt. If there are no new recommeded indexes, NULL 
          140  +**   is returned.
          141  +**
          142  +** EXPERT_REPORT_PLAN:
          143  +**   Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
          144  +**   iStmt after the proposed indexes have been added to the database schema.
          145  +**
          146  +** EXPERT_REPORT_CANDIDATES:
          147  +**   Return a pointer to a buffer containing the CREATE INDEX statements 
          148  +**   for all indexes that were tested (for all SQL statements). The iStmt
          149  +**   parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
          150  +*/
          151  +const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
          152  +
          153  +/*
          154  +** Values for the third argument passed to sqlite3_expert_report().
          155  +*/
          156  +#define EXPERT_REPORT_SQL        1
          157  +#define EXPERT_REPORT_INDEXES    2
          158  +#define EXPERT_REPORT_PLAN       3
          159  +#define EXPERT_REPORT_CANDIDATES 4
          160  +
          161  +/*
          162  +** Free an (sqlite3expert*) handle and all associated resources. There 
          163  +** should be one call to this function for each successful call to 
          164  +** sqlite3-expert_new().
          165  +*/
          166  +void sqlite3_expert_destroy(sqlite3expert*);
          167  +
          168  +

Added ext/expert/test_expert.c.

            1  +/*
            2  +** 2017 April 07
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +*/
           13  +
           14  +#if defined(SQLITE_TEST)
           15  +
           16  +#include "sqlite3expert.h"
           17  +#include <assert.h>
           18  +#include <string.h>
           19  +
           20  +#if defined(INCLUDE_SQLITE_TCL_H)
           21  +#  include "sqlite_tcl.h"
           22  +#else
           23  +#  include "tcl.h"
           24  +#  ifndef SQLITE_TCLAPI
           25  +#    define SQLITE_TCLAPI
           26  +#  endif
           27  +#endif
           28  +
           29  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           30  +
           31  +/*
           32  +** Extract an sqlite3* db handle from the object passed as the second
           33  +** argument. If successful, set *pDb to point to the db handle and return
           34  +** TCL_OK. Otherwise, return TCL_ERROR.
           35  +*/
           36  +static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
           37  +  Tcl_CmdInfo info;
           38  +  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
           39  +    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
           40  +    return TCL_ERROR;
           41  +  }
           42  +
           43  +  *pDb = *(sqlite3 **)info.objClientData;
           44  +  return TCL_OK;
           45  +}
           46  +
           47  +
           48  +/*
           49  +** Tclcmd:  $expert sql SQL
           50  +**          $expert analyze
           51  +**          $expert count
           52  +**          $expert report STMT EREPORT
           53  +**          $expert destroy
           54  +*/
           55  +static int SQLITE_TCLAPI testExpertCmd(
           56  +  void *clientData,
           57  +  Tcl_Interp *interp,
           58  +  int objc,
           59  +  Tcl_Obj *CONST objv[]
           60  +){
           61  +  sqlite3expert *pExpert = (sqlite3expert*)clientData;
           62  +  struct Subcmd {
           63  +    const char *zSub;
           64  +    int nArg;
           65  +    const char *zMsg;
           66  +  } aSub[] = {
           67  +    { "sql",       1, "TABLE",        }, /* 0 */
           68  +    { "analyze",   0, "",             }, /* 1 */
           69  +    { "count",     0, "",             }, /* 2 */
           70  +    { "report",    2, "STMT EREPORT", }, /* 3 */
           71  +    { "destroy",   0, "",             }, /* 4 */
           72  +    { 0 }
           73  +  };
           74  +  int iSub;
           75  +  int rc = TCL_OK;
           76  +  char *zErr = 0;
           77  +
           78  +  if( objc<2 ){
           79  +    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
           80  +    return TCL_ERROR;
           81  +  }
           82  +  rc = Tcl_GetIndexFromObjStruct(interp, 
           83  +      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
           84  +  );
           85  +  if( rc!=TCL_OK ) return rc;
           86  +  if( objc!=2+aSub[iSub].nArg ){
           87  +    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
           88  +    return TCL_ERROR;
           89  +  }
           90  +
           91  +  switch( iSub ){
           92  +    case 0: {      /* sql */
           93  +      char *zArg = Tcl_GetString(objv[2]);
           94  +      rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
           95  +      break;
           96  +    }
           97  +
           98  +    case 1: {      /* analyze */
           99  +      rc = sqlite3_expert_analyze(pExpert, &zErr);
          100  +      break;
          101  +    }
          102  +
          103  +    case 2: {      /* count */
          104  +      int n = sqlite3_expert_count(pExpert);
          105  +      Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
          106  +      break;
          107  +    }
          108  +
          109  +    case 3: {      /* report */
          110  +      const char *aEnum[] = {
          111  +        "sql", "indexes", "plan", "candidates", 0
          112  +      };
          113  +      int iEnum;
          114  +      int iStmt;
          115  +      const char *zReport;
          116  +
          117  +      if( Tcl_GetIntFromObj(interp, objv[2], &iStmt) 
          118  +       || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
          119  +      ){
          120  +        return TCL_ERROR;
          121  +      }
          122  +
          123  +      assert( EXPERT_REPORT_SQL==1 );
          124  +      assert( EXPERT_REPORT_INDEXES==2 );
          125  +      assert( EXPERT_REPORT_PLAN==3 );
          126  +      assert( EXPERT_REPORT_CANDIDATES==4 );
          127  +      zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
          128  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
          129  +      break;
          130  +    }
          131  +
          132  +    default:       /* destroy */
          133  +      assert( iSub==4 );     
          134  +      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
          135  +      break;
          136  +  }
          137  +
          138  +  if( rc!=TCL_OK ){
          139  +    if( zErr ){
          140  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
          141  +    }else{
          142  +      extern const char *sqlite3ErrName(int);
          143  +      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
          144  +    }
          145  +  }
          146  +  sqlite3_free(zErr);
          147  +  return rc;
          148  +}
          149  +
          150  +static void SQLITE_TCLAPI testExpertDel(void *clientData){
          151  +  sqlite3expert *pExpert = (sqlite3expert*)clientData;
          152  +  sqlite3_expert_destroy(pExpert);
          153  +}
          154  +
          155  +/*
          156  +** sqlite3_expert_new DB
          157  +*/
          158  +static int SQLITE_TCLAPI test_sqlite3_expert_new(
          159  +  void * clientData,
          160  +  Tcl_Interp *interp,
          161  +  int objc,
          162  +  Tcl_Obj *CONST objv[]
          163  +){
          164  +  static int iCmd = 0;
          165  +  sqlite3 *db;
          166  +  char *zCmd = 0;
          167  +  char *zErr = 0;
          168  +  sqlite3expert *pExpert;
          169  +  int rc = TCL_OK;
          170  +
          171  +  if( objc!=2 ){
          172  +    Tcl_WrongNumArgs(interp, 1, objv, "DB");
          173  +    return TCL_ERROR;
          174  +  }
          175  +  if( dbHandleFromObj(interp, objv[1], &db) ){
          176  +    return TCL_ERROR;
          177  +  }
          178  +
          179  +  zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
          180  +  if( zCmd==0 ){
          181  +    Tcl_AppendResult(interp, "out of memory", (char*)0);
          182  +    return TCL_ERROR;
          183  +  }
          184  +
          185  +  pExpert = sqlite3_expert_new(db, &zErr);
          186  +  if( pExpert==0 ){
          187  +    Tcl_AppendResult(interp, zErr, (char*)0);
          188  +    rc = TCL_ERROR;
          189  +  }else{
          190  +    void *p = (void*)pExpert;
          191  +    Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
          192  +    Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
          193  +  }
          194  +
          195  +  sqlite3_free(zCmd);
          196  +  sqlite3_free(zErr);
          197  +  return rc;
          198  +}
          199  +
          200  +#endif  /* ifndef SQLITE_OMIT_VIRTUALTABLE */
          201  +
          202  +int TestExpert_Init(Tcl_Interp *interp){
          203  +#ifndef SQLITE_OMIT_VIRTUALTABLE
          204  +  struct Cmd {
          205  +    const char *zCmd;
          206  +    Tcl_ObjCmdProc *xProc;
          207  +  } aCmd[] = {
          208  +    { "sqlite3_expert_new", test_sqlite3_expert_new },
          209  +  };
          210  +  int i;
          211  +
          212  +  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
          213  +    struct Cmd *p = &aCmd[i];
          214  +    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
          215  +  }
          216  +#endif
          217  +  return TCL_OK;
          218  +}
          219  +
          220  +#endif

Changes to ext/fts3/fts3_write.c.

  1904   1904     sqlite3_stmt *pStmt;
  1905   1905     int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
  1906   1906     if( rc==SQLITE_OK ){
  1907   1907       sqlite3_bind_int64(pStmt, 1, iBlock);
  1908   1908       sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
  1909   1909       sqlite3_step(pStmt);
  1910   1910       rc = sqlite3_reset(pStmt);
         1911  +    sqlite3_bind_null(pStmt, 2);
  1911   1912     }
  1912   1913     return rc;
  1913   1914   }
  1914   1915   
  1915   1916   /*
  1916   1917   ** Find the largest relative level number in the table. If successful, set
  1917   1918   ** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs,
................................................................................
  1960   1961         char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData);
  1961   1962         if( !zEnd ) return SQLITE_NOMEM;
  1962   1963         sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free);
  1963   1964       }
  1964   1965       sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
  1965   1966       sqlite3_step(pStmt);
  1966   1967       rc = sqlite3_reset(pStmt);
         1968  +    sqlite3_bind_null(pStmt, 6);
  1967   1969     }
  1968   1970     return rc;
  1969   1971   }
  1970   1972   
  1971   1973   /*
  1972   1974   ** Return the size of the common prefix (if any) shared by zPrev and
  1973   1975   ** zNext, in bytes. For example, 
................................................................................
  3439   3441       *pRC = rc;
  3440   3442       return;
  3441   3443     }
  3442   3444     sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
  3443   3445     sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
  3444   3446     sqlite3_step(pStmt);
  3445   3447     *pRC = sqlite3_reset(pStmt);
         3448  +  sqlite3_bind_null(pStmt, 2);
  3446   3449     sqlite3_free(a);
  3447   3450   }
  3448   3451   
  3449   3452   /*
  3450   3453   ** Merge the entire database so that there is one segment for each 
  3451   3454   ** iIndex/iLangid combination.
  3452   3455   */
................................................................................
  4627   4630       if( rc==SQLITE_OK ){
  4628   4631         sqlite3_bind_int64(pChomp, 1, iNewStart);
  4629   4632         sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC);
  4630   4633         sqlite3_bind_int64(pChomp, 3, iAbsLevel);
  4631   4634         sqlite3_bind_int(pChomp, 4, iIdx);
  4632   4635         sqlite3_step(pChomp);
  4633   4636         rc = sqlite3_reset(pChomp);
         4637  +      sqlite3_bind_null(pChomp, 2);
  4634   4638       }
  4635   4639     }
  4636   4640   
  4637   4641     sqlite3_free(root.a);
  4638   4642     sqlite3_free(block.a);
  4639   4643     return rc;
  4640   4644   }
................................................................................
  4706   4710   
  4707   4711     rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0);
  4708   4712     if( rc==SQLITE_OK ){
  4709   4713       sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
  4710   4714       sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
  4711   4715       sqlite3_step(pReplace);
  4712   4716       rc = sqlite3_reset(pReplace);
         4717  +    sqlite3_bind_null(pReplace, 2);
  4713   4718     }
  4714   4719   
  4715   4720     return rc;
  4716   4721   }
  4717   4722   
  4718   4723   /*
  4719   4724   ** Load an incr-merge hint from the database. The incr-merge hint, if one 
................................................................................
  5520   5525     sqlite3_vtab *pVtab,            /* FTS3 vtab object */
  5521   5526     int nArg,                       /* Size of argument array */
  5522   5527     sqlite3_value **apVal,          /* Array of arguments */
  5523   5528     sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
  5524   5529   ){
  5525   5530     Fts3Table *p = (Fts3Table *)pVtab;
  5526   5531     int rc = SQLITE_OK;             /* Return Code */
  5527         -  int isRemove = 0;               /* True for an UPDATE or DELETE */
  5528   5532     u32 *aSzIns = 0;                /* Sizes of inserted documents */
  5529   5533     u32 *aSzDel = 0;                /* Sizes of deleted documents */
  5530   5534     int nChng = 0;                  /* Net change in number of documents */
  5531   5535     int bInsertDone = 0;
  5532   5536   
  5533   5537     /* At this point it must be known if the %_stat table exists or not.
  5534   5538     ** So bHasStat may not be 2.  */
................................................................................
  5618   5622       goto update_out;
  5619   5623     }
  5620   5624   
  5621   5625     /* If this is a DELETE or UPDATE operation, remove the old record. */
  5622   5626     if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  5623   5627       assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
  5624   5628       rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
  5625         -    isRemove = 1;
  5626   5629     }
  5627   5630     
  5628   5631     /* If this is an INSERT or UPDATE operation, insert the new record. */
  5629   5632     if( nArg>1 && rc==SQLITE_OK ){
  5630   5633       int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
  5631   5634       if( bInsertDone==0 ){
  5632   5635         rc = fts3InsertData(p, apVal, pRowid);
  5633   5636         if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
  5634   5637           rc = FTS_CORRUPT_VTAB;
  5635   5638         }
  5636   5639       }
  5637         -    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
         5640  +    if( rc==SQLITE_OK ){
  5638   5641         rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
  5639   5642       }
  5640   5643       if( rc==SQLITE_OK ){
  5641   5644         assert( p->iPrevDocid==*pRowid );
  5642   5645         rc = fts3InsertTerms(p, iLangid, apVal, aSzIns);
  5643   5646       }
  5644   5647       if( p->bHasDocsize ){

Changes to ext/fts5/fts5Int.h.

   717    717   
   718    718   Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   719    719     Fts5Parse *pParse, 
   720    720     Fts5ExprPhrase *pPhrase, 
   721    721     Fts5Token *pToken,
   722    722     int bPrefix
   723    723   );
          724  +
          725  +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
   724    726   
   725    727   Fts5ExprNearset *sqlite3Fts5ParseNearset(
   726    728     Fts5Parse*, 
   727    729     Fts5ExprNearset*,
   728    730     Fts5ExprPhrase* 
   729    731   );
   730    732

Changes to ext/fts5/fts5_aux.c.

   353    353       if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
   354    354       if( iAdj<0 ) iAdj = 0;
   355    355       *piPos = iAdj;
   356    356     }
   357    357   
   358    358     return rc;
   359    359   }
          360  +
          361  +/*
          362  +** Return the value in pVal interpreted as utf-8 text. Except, if pVal 
          363  +** contains a NULL value, return a pointer to a static string zero
          364  +** bytes in length instead of a NULL pointer.
          365  +*/
          366  +static const char *fts5ValueToText(sqlite3_value *pVal){
          367  +  const char *zRet = (const char*)sqlite3_value_text(pVal);
          368  +  return zRet ? zRet : "";
          369  +}
   360    370   
   361    371   /*
   362    372   ** Implementation of snippet() function.
   363    373   */
   364    374   static void fts5SnippetFunction(
   365    375     const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   366    376     Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................
   389    399       sqlite3_result_error(pCtx, zErr, -1);
   390    400       return;
   391    401     }
   392    402   
   393    403     nCol = pApi->xColumnCount(pFts);
   394    404     memset(&ctx, 0, sizeof(HighlightContext));
   395    405     iCol = sqlite3_value_int(apVal[0]);
   396         -  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   397         -  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   398         -  zEllips = (const char*)sqlite3_value_text(apVal[3]);
          406  +  ctx.zOpen = fts5ValueToText(apVal[1]);
          407  +  ctx.zClose = fts5ValueToText(apVal[2]);
          408  +  zEllips = fts5ValueToText(apVal[3]);
   399    409     nToken = sqlite3_value_int(apVal[4]);
   400    410   
   401    411     iBestCol = (iCol>=0 ? iCol : 0);
   402    412     nPhrase = pApi->xPhraseCount(pFts);
   403    413     aSeen = sqlite3_malloc(nPhrase);
   404    414     if( aSeen==0 ){
   405    415       rc = SQLITE_NOMEM;

Changes to ext/fts5/fts5_expr.c.

    83     83   #define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d))
    84     84   
    85     85   /*
    86     86   ** An instance of the following structure represents a single search term
    87     87   ** or term prefix.
    88     88   */
    89     89   struct Fts5ExprTerm {
    90         -  int bPrefix;                    /* True for a prefix term */
           90  +  u8 bPrefix;                     /* True for a prefix term */
           91  +  u8 bFirst;                      /* True if token must be first in column */
    91     92     char *zTerm;                    /* nul-terminated term */
    92     93     Fts5IndexIter *pIter;           /* Iterator for this term */
    93     94     Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
    94     95   };
    95     96   
    96     97   /*
    97     98   ** A phrase. One or more terms that must appear in a contiguous sequence
................................................................................
   164    165       case '{':  tok = FTS5_LCP;   break;
   165    166       case '}':  tok = FTS5_RCP;   break;
   166    167       case ':':  tok = FTS5_COLON; break;
   167    168       case ',':  tok = FTS5_COMMA; break;
   168    169       case '+':  tok = FTS5_PLUS;  break;
   169    170       case '*':  tok = FTS5_STAR;  break;
   170    171       case '-':  tok = FTS5_MINUS; break;
          172  +    case '^':  tok = FTS5_CARET; break;
   171    173       case '\0': tok = FTS5_EOF;   break;
   172    174   
   173    175       case '"': {
   174    176         const char *z2;
   175    177         tok = FTS5_STRING;
   176    178   
   177    179         for(z2=&z[1]; 1; z2++){
................................................................................
   423    425     int *pbMatch                    /* OUT: Set to true if really a match */
   424    426   ){
   425    427     Fts5PoslistWriter writer = {0};
   426    428     Fts5PoslistReader aStatic[4];
   427    429     Fts5PoslistReader *aIter = aStatic;
   428    430     int i;
   429    431     int rc = SQLITE_OK;
          432  +  int bFirst = pPhrase->aTerm[0].bFirst;
   430    433     
   431    434     fts5BufferZero(&pPhrase->poslist);
   432    435   
   433    436     /* If the aStatic[] array is not large enough, allocate a large array
   434    437     ** using sqlite3_malloc(). This approach could be improved upon. */
   435    438     if( pPhrase->nTerm>ArraySize(aStatic) ){
   436    439       int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
................................................................................
   477    480             }
   478    481             if( pPos->iPos>iAdj ) iPos = pPos->iPos-i;
   479    482           }
   480    483         }
   481    484       }while( bMatch==0 );
   482    485   
   483    486       /* Append position iPos to the output */
   484         -    rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
   485         -    if( rc!=SQLITE_OK ) goto ismatch_out;
          487  +    if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){
          488  +      rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
          489  +      if( rc!=SQLITE_OK ) goto ismatch_out;
          490  +    }
   486    491   
   487    492       for(i=0; i<pPhrase->nTerm; i++){
   488    493         if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
   489    494       }
   490    495     }
   491    496   
   492    497    ismatch_out:
................................................................................
   732    737       int i;
   733    738   
   734    739       /* Check that each phrase in the nearset matches the current row.
   735    740       ** Populate the pPhrase->poslist buffers at the same time. If any
   736    741       ** phrase is not a match, break out of the loop early.  */
   737    742       for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   738    743         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   739         -      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
          744  +      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym 
          745  +       || pNear->pColset || pPhrase->aTerm[0].bFirst
          746  +      ){
   740    747           int bMatch = 0;
   741    748           rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
   742    749           if( bMatch==0 ) break;
   743    750         }else{
   744    751           Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
   745    752           fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData);
   746    753         }
................................................................................
   913    920     int bMatch;                     /* True if all terms are at the same rowid */
   914    921     const int bDesc = pExpr->bDesc;
   915    922   
   916    923     /* Check that this node should not be FTS5_TERM */
   917    924     assert( pNear->nPhrase>1 
   918    925          || pNear->apPhrase[0]->nTerm>1 
   919    926          || pNear->apPhrase[0]->aTerm[0].pSynonym
          927  +       || pNear->apPhrase[0]->aTerm[0].bFirst
   920    928     );
   921    929   
   922    930     /* Initialize iLast, the "lastest" rowid any iterator points to. If the
   923    931     ** iterator skips through rowids in the default ascending order, this means
   924    932     ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
   925    933     ** means the minimum rowid.  */
   926    934     if( pLeft->aTerm[0].pSynonym ){
................................................................................
  1436   1444           sqlite3_free(pSyn);
  1437   1445         }
  1438   1446       }
  1439   1447       if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
  1440   1448       sqlite3_free(pPhrase);
  1441   1449     }
  1442   1450   }
         1451  +
         1452  +/*
         1453  +** Set the "bFirst" flag on the first token of the phrase passed as the
         1454  +** only argument.
         1455  +*/
         1456  +void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
         1457  +  if( pPhrase && pPhrase->nTerm ){
         1458  +    pPhrase->aTerm[0].bFirst = 1;
         1459  +  }
         1460  +}
  1443   1461   
  1444   1462   /*
  1445   1463   ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
  1446   1464   ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
  1447   1465   ** appended to it and the results returned.
  1448   1466   **
  1449   1467   ** If an OOM error occurs, both the pNear and pPhrase objects are freed and
................................................................................
  1715   1733           const char *zTerm = p->zTerm;
  1716   1734           rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
  1717   1735               0, 0);
  1718   1736           tflags = FTS5_TOKEN_COLOCATED;
  1719   1737         }
  1720   1738         if( rc==SQLITE_OK ){
  1721   1739           sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
         1740  +        sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
  1722   1741         }
  1723   1742       }
  1724   1743     }else{
  1725   1744       /* This happens when parsing a token or quoted phrase that contains
  1726   1745       ** no token characters at all. (e.g ... MATCH '""'). */
  1727   1746       sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
  1728   1747     }
................................................................................
  1733   1752       pNew->pConfig = pExpr->pConfig;
  1734   1753       pNew->nPhrase = 1;
  1735   1754       pNew->apExprPhrase[0] = sCtx.pPhrase;
  1736   1755       pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
  1737   1756       pNew->pRoot->pNear->nPhrase = 1;
  1738   1757       sCtx.pPhrase->pNode = pNew->pRoot;
  1739   1758   
  1740         -    if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
         1759  +    if( pOrig->nTerm==1 
         1760  +     && pOrig->aTerm[0].pSynonym==0 
         1761  +     && pOrig->aTerm[0].bFirst==0 
         1762  +    ){
  1741   1763         pNew->pRoot->eType = FTS5_TERM;
  1742   1764         pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
  1743   1765       }else{
  1744   1766         pNew->pRoot->eType = FTS5_STRING;
  1745   1767         pNew->pRoot->xNext = fts5ExprNodeNext_STRING;
  1746   1768       }
  1747   1769     }else{
................................................................................
  2007   2029   
  2008   2030   static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  2009   2031     switch( pNode->eType ){
  2010   2032       case FTS5_STRING: {
  2011   2033         Fts5ExprNearset *pNear = pNode->pNear;
  2012   2034         if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
  2013   2035          && pNear->apPhrase[0]->aTerm[0].pSynonym==0
         2036  +       && pNear->apPhrase[0]->aTerm[0].bFirst==0
  2014   2037         ){
  2015   2038           pNode->eType = FTS5_TERM;
  2016   2039           pNode->xNext = fts5ExprNodeNext_TERM;
  2017   2040         }else{
  2018   2041           pNode->xNext = fts5ExprNodeNext_STRING;
  2019   2042         }
  2020   2043         break;
................................................................................
  2093   2116             pNear->apPhrase[iPhrase]->pNode = pRet;
  2094   2117             if( pNear->apPhrase[iPhrase]->nTerm==0 ){
  2095   2118               pRet->xNext = 0;
  2096   2119               pRet->eType = FTS5_EOF;
  2097   2120             }
  2098   2121           }
  2099   2122   
  2100         -        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
  2101         -         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
  2102         -        ){
  2103         -          assert( pParse->rc==SQLITE_OK );
  2104         -          pParse->rc = SQLITE_ERROR;
  2105         -          assert( pParse->zErr==0 );
  2106         -          pParse->zErr = sqlite3_mprintf(
  2107         -              "fts5: %s queries are not supported (detail!=full)", 
  2108         -              pNear->nPhrase==1 ? "phrase": "NEAR"
  2109         -          );
  2110         -          sqlite3_free(pRet);
  2111         -          pRet = 0;
         2123  +        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
         2124  +          Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
         2125  +          if( pNear->nPhrase!=1 
         2126  +           || pPhrase->nTerm>1
         2127  +           || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
         2128  +          ){
         2129  +            assert( pParse->rc==SQLITE_OK );
         2130  +            pParse->rc = SQLITE_ERROR;
         2131  +            assert( pParse->zErr==0 );
         2132  +            pParse->zErr = sqlite3_mprintf(
         2133  +                "fts5: %s queries are not supported (detail!=full)", 
         2134  +                pNear->nPhrase==1 ? "phrase": "NEAR"
         2135  +                );
         2136  +            sqlite3_free(pRet);
         2137  +            pRet = 0;
         2138  +          }
  2112   2139           }
  2113         -
  2114   2140         }else{
  2115   2141           fts5ExprAddChildren(pRet, pLeft);
  2116   2142           fts5ExprAddChildren(pRet, pRight);
  2117   2143         }
  2118   2144       }
  2119   2145     }
  2120   2146

Changes to ext/fts5/fts5_index.c.

   754    754       if( p->rc ) return;
   755    755     }
   756    756   
   757    757     sqlite3_bind_int64(p->pWriter, 1, iRowid);
   758    758     sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC);
   759    759     sqlite3_step(p->pWriter);
   760    760     p->rc = sqlite3_reset(p->pWriter);
          761  +  sqlite3_bind_null(p->pWriter, 2);
   761    762   }
   762    763   
   763    764   /*
   764    765   ** Execute the following SQL:
   765    766   **
   766    767   **     DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast
   767    768   */
................................................................................
  2382   2383     sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
  2383   2384     if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){
  2384   2385       i64 val = sqlite3_column_int(pIdxSelect, 0);
  2385   2386       iPg = (int)(val>>1);
  2386   2387       bDlidx = (val & 0x0001);
  2387   2388     }
  2388   2389     p->rc = sqlite3_reset(pIdxSelect);
         2390  +  sqlite3_bind_null(pIdxSelect, 2);
  2389   2391   
  2390   2392     if( iPg<pSeg->pgnoFirst ){
  2391   2393       iPg = pSeg->pgnoFirst;
  2392   2394       bDlidx = 0;
  2393   2395     }
  2394   2396   
  2395   2397     pIter->iLeafPgno = iPg - 1;
................................................................................
  3594   3596           sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
  3595   3597           if( p->rc==SQLITE_OK ){
  3596   3598             u8 aBlob[2] = {0xff, 0xff};
  3597   3599             sqlite3_bind_int(pIdxSelect, 1, iSegid);
  3598   3600             sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
  3599   3601             assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
  3600   3602             p->rc = sqlite3_reset(pIdxSelect);
         3603  +          sqlite3_bind_null(pIdxSelect, 2);
  3601   3604           }
  3602   3605         }
  3603   3606   #endif
  3604   3607       }
  3605   3608     }
  3606   3609   
  3607   3610     return iSegid;
................................................................................
  3720   3723       const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:"");
  3721   3724       /* The following was already done in fts5WriteInit(): */
  3722   3725       /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
  3723   3726       sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC);
  3724   3727       sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
  3725   3728       sqlite3_step(p->pIdxWriter);
  3726   3729       p->rc = sqlite3_reset(p->pIdxWriter);
         3730  +    sqlite3_bind_null(p->pIdxWriter, 2);
  3727   3731     }
  3728   3732     pWriter->iBtPage = 0;
  3729   3733   }
  3730   3734   
  3731   3735   /*
  3732   3736   ** This is called once for each leaf page except the first that contains
  3733   3737   ** at least one term. Argument (nTerm/pTerm) is the split-key - a term that
................................................................................
  4905   4909     if( p2->n ){
  4906   4910       i64 iLastRowid = 0;
  4907   4911       Fts5DoclistIter i1;
  4908   4912       Fts5DoclistIter i2;
  4909   4913       Fts5Buffer out = {0, 0, 0};
  4910   4914       Fts5Buffer tmp = {0, 0, 0};
  4911   4915   
  4912         -    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
         4916  +    /* The maximum size of the output is equal to the sum of the two 
         4917  +    ** input sizes + 1 varint (9 bytes). The extra varint is because if the
         4918  +    ** first rowid in one input is a large negative number, and the first in
         4919  +    ** the other a non-negative number, the delta for the non-negative
         4920  +    ** number will be larger on disk than the literal integer value
         4921  +    ** was.  */
         4922  +    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return;
  4913   4923       fts5DoclistIterInit(p1, &i1);
  4914   4924       fts5DoclistIterInit(p2, &i2);
  4915   4925   
  4916   4926       while( 1 ){
  4917   4927         if( i1.iRowid<i2.iRowid ){
  4918   4928           /* Copy entry from i1 */
  4919   4929           fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
................................................................................
  4999   5009         fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  5000   5010         fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
  5001   5011       }
  5002   5012       else if( i2.aPoslist ){
  5003   5013         fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  5004   5014         fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
  5005   5015       }
         5016  +    assert( out.n<=(p1->n+p2->n+9) );
  5006   5017   
  5007   5018       fts5BufferSet(&p->rc, p1, out.n, out.p);
  5008   5019       fts5BufferFree(&tmp);
  5009   5020       fts5BufferFree(&out);
  5010   5021     }
  5011   5022   }
  5012   5023

Changes to ext/fts5/fts5_storage.c.

   454    454       sqlite3_stmt *pReplace = 0;
   455    455       rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
   456    456       if( rc==SQLITE_OK ){
   457    457         sqlite3_bind_int64(pReplace, 1, iRowid);
   458    458         sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
   459    459         sqlite3_step(pReplace);
   460    460         rc = sqlite3_reset(pReplace);
          461  +      sqlite3_bind_null(pReplace, 2);
   461    462       }
   462    463     }
   463    464     return rc;
   464    465   }
   465    466   
   466    467   /*
   467    468   ** Load the contents of the "averages" record from disk into the 
................................................................................
  1114   1115       if( pVal ){
  1115   1116         sqlite3_bind_value(pReplace, 2, pVal);
  1116   1117       }else{
  1117   1118         sqlite3_bind_int(pReplace, 2, iVal);
  1118   1119       }
  1119   1120       sqlite3_step(pReplace);
  1120   1121       rc = sqlite3_reset(pReplace);
         1122  +    sqlite3_bind_null(pReplace, 1);
  1121   1123     }
  1122   1124     if( rc==SQLITE_OK && pVal ){
  1123   1125       int iNew = p->pConfig->iCookie + 1;
  1124   1126       rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
  1125   1127       if( rc==SQLITE_OK ){
  1126   1128         p->pConfig->iCookie = iNew;
  1127   1129       }
  1128   1130     }
  1129   1131     return rc;
  1130   1132   }

Changes to ext/fts5/fts5_tcl.c.

   429    429         rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0);
   430    430         break;
   431    431       }
   432    432       CASE(15, "xGetAuxdataInt") {
   433    433         int iVal;
   434    434         int bClear;
   435    435         if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
   436         -      iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
          436  +      iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
   437    437         Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
   438    438         break;
   439    439       }
   440    440   
   441    441       CASE(16, "xPhraseForeach") {
   442    442         int iPhrase;
   443    443         int iCol;
................................................................................
   478    478         Fts5PhraseIter iter;
   479    479   
   480    480         if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
   481    481         zColvar = Tcl_GetString(objv[3]);
   482    482   
   483    483         rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
   484    484         if( rc!=SQLITE_OK ){
   485         -        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
          485  +        Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
   486    486           return TCL_ERROR;
   487    487         }
   488    488         for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
   489    489           Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
   490    490           rc = Tcl_EvalObjEx(interp, pScript, 0);
   491    491           if( rc==TCL_CONTINUE ) rc = TCL_OK;
   492    492           if( rc!=TCL_OK ){
................................................................................
   920    920           "sqlite3_fts5_token may only be used by tokenizer callback", 0
   921    921       );
   922    922       return TCL_ERROR;
   923    923     }
   924    924   
   925    925     rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
   926    926     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
   927         -  return TCL_OK;
          927  +  return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
   928    928   
   929    929    usage:
   930    930     Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
   931    931     return TCL_ERROR;
   932    932   }
   933    933   
   934    934   static void f5tDelTokenizer(void *pCtx){

Changes to ext/fts5/fts5parse.y.

   144    144   
   145    145   
   146    146   %type nearset     {Fts5ExprNearset*}
   147    147   %type nearphrases {Fts5ExprNearset*}
   148    148   %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
   149    149   %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
   150    150   
   151         -nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
          151  +nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
          152  +nearset(A) ::= CARET phrase(Y). { 
          153  +  sqlite3Fts5ParseSetCaret(Y);
          154  +  A = sqlite3Fts5ParseNearset(pParse, 0, Y); 
          155  +}
   152    156   nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
   153    157     sqlite3Fts5ParseNear(pParse, &X);
   154    158     sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
   155    159     A = Y;
   156    160   }
   157    161   
   158    162   nearphrases(A) ::= phrase(X). { 
................................................................................
   185    189     A = sqlite3Fts5ParseTerm(pParse, 0, &Y, Z);
   186    190   }
   187    191   
   188    192   /*
   189    193   ** Optional "*" character.
   190    194   */
   191    195   %type star_opt {int}
   192         -
   193    196   star_opt(A) ::= STAR. { A = 1; }
   194    197   star_opt(A) ::= . { A = 0; }

Changes to ext/fts5/test/fts5aa.test.

   589    589   
   590    590   do_execsql_test 22.1 {
   591    591     SELECT rowid FROM t9('a*')
   592    592   } {1}
   593    593   
   594    594   }
   595    595   
   596         -
          596  +expand_all_sql db
   597    597   finish_test

Changes to ext/fts5/test/fts5af.test.

   170    170       'x a a a a a a a a a a',
   171    171       'a a a a a a a a a a a a a a a a a a a x'
   172    172     );
   173    173   }
   174    174   do_execsql_test 5.1 {
   175    175     SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
   176    176   } {{[x] a a a a a...}}
          177  +
          178  +do_execsql_test 5.2 {
          179  +  SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
          180  +} {{[x] a a a a a}}
          181  +do_execsql_test 5.3 {
          182  +  SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
          183  +} {{x] a a a a a...}}
          184  +do_execsql_test 5.4 {
          185  +  SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
          186  +} {{[x a a a a a...}}
   177    187   
   178    188   } ;# foreach_detail_mode 
   179    189   
   180    190   finish_test

Changes to ext/fts5/test/fts5fault6.test.

   249    249     sqlite3_fts5_register_matchinfo db
   250    250     db func mit mit
   251    251   } -body {
   252    252     db eval { 
   253    253       SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
   254    254     }
   255    255   } -test {
   256         -  faultsim_test_result [list 0 $::res]
          256  +  faultsim_test_result [list 0 $::res] {1 {SQL logic error}}
   257    257   }
   258    258   
   259    259   do_faultsim_test 5.3 -faults oom* -prep {
   260    260     faultsim_restore_and_reopen
   261    261     sqlite3_fts5_create_tokenizer db tcl tcl_create
   262    262   } -body {
   263    263     db eval { 
   264    264       SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
   265    265     }
   266    266   } -test {
   267         -  faultsim_test_result {0 29}
          267  +  faultsim_test_result {0 29} {1 {SQL logic error}}
   268    268   }
   269    269   
   270    270   do_faultsim_test 5.4 -faults oom* -prep {
   271    271     faultsim_restore_and_reopen
   272    272     sqlite3_fts5_create_tokenizer db tcl tcl_create
   273    273   } -body {
   274    274     db eval { 
   275    275       SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
   276    276     }
   277    277   } -test {
   278         -  faultsim_test_result {0 1}
          278  +  faultsim_test_result {0 1} {1 {SQL logic error}}
   279    279   }
   280    280   
   281    281   #-------------------------------------------------------------------------
   282    282   catch { db close }
   283    283   do_faultsim_test 6 -faults oom* -prep {
   284    284     sqlite_orig db test.db
   285    285     sqlite3_db_config_lookaside db 0 0 0

Changes to ext/fts5/test/fts5fault9.test.

    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   foreach_detail_mode $testprefix {
    26     26   
           27  +if {"%DETAIL%" != "none"} continue
           28  +
    27     29   fts5_aux_test_functions db
    28     30   
    29     31   do_execsql_test 1.0 {
    30     32     CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    31     33     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    32     34     WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
    33     35     INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
................................................................................
    94     96     INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
    95     97   }
    96     98   
    97     99   do_faultsim_test 4.1 -faults oom-t* -body {
    98    100     execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
    99    101   } -test {
   100    102     faultsim_test_result \
   101         -      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
          103  +      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} \
          104  +      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
   102    105   }
   103    106   
   104    107   do_faultsim_test 4.2 -faults oom-t* -body {
   105    108     execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
   106    109   } -test {
   107    110     faultsim_test_result \
   108         -      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
          111  +      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} \
          112  +      {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
   109    113   }
   110    114   
   111    115   
   112    116   #-------------------------------------------------------------------------
   113    117   # An OOM within an "ORDER BY rank" query.
   114    118   #
   115    119   db func rnddoc fts5_rnddoc

Changes to ext/fts5/test/fts5faultB.test.

   125    125   }
   126    126   
   127    127   do_faultsim_test 4.2 -faults oom* -body {
   128    128     execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
   129    129   } -test {
   130    130     faultsim_test_result {0 {2 3}}
   131    131   }
          132  +
          133  +#-------------------------------------------------------------------------
          134  +# Test OOM injection while parsing a CARET expression
          135  +#
          136  +reset_db
          137  +do_execsql_test 5.0 {
          138  +  CREATE VIRTUAL TABLE t1 USING fts5(a);
          139  +  INSERT INTO t1 VALUES('a b c d');  -- 1
          140  +  INSERT INTO t1 VALUES('d a b c');  -- 2
          141  +  INSERT INTO t1 VALUES('c d a b');  -- 3
          142  +  INSERT INTO t1 VALUES('b c d a');  -- 4
          143  +}
          144  +do_faultsim_test 5.1 -faults oom* -body {
          145  +  execsql { SELECT rowid FROM t1('^a OR ^b') }
          146  +} -test {
          147  +  faultsim_test_result {0 {1 4}}
          148  +}
   132    149   
   133    150   
   134    151   finish_test

Added ext/fts5/test/fts5first.test.

            1  +# 2017 November 25
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +
           12  +source [file join [file dirname [info script]] fts5_common.tcl]
           13  +set testprefix fts5first
           14  +
           15  +ifcapable !fts5 {
           16  +  finish_test
           17  +  return
           18  +}
           19  +
           20  +
           21  +do_execsql_test 1.0 {
           22  +  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
           23  +}
           24  +
           25  +foreach {tn expr ok} {
           26  +  1 {^abc}           1
           27  +  2 {^abc + def}     1
           28  +  3 {^ "abc def"}    1
           29  +  4 {^"abc def"}     1
           30  +  5 {abc ^def}       1
           31  +  6 {abc + ^def}     0
           32  +  7 {abc ^+ def}     0
           33  +  8 {"^abc"}         1
           34  +  9 {NEAR(^abc def)} 0
           35  +} {
           36  +  set res(0) {/1 {fts5: syntax error near .*}/}
           37  +  set res(1) {0 {}}
           38  +
           39  +  do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok)
           40  +}
           41  +
           42  +#-------------------------------------------------------------------------
           43  +# 
           44  +do_execsql_test 2.0 {
           45  +  INSERT INTO x1 VALUES('a b c', 'b c a');
           46  +}
           47  +
           48  +foreach {tn expr match} {
           49  +  1 {^a} 1
           50  +  2 {^b} 1
           51  +  3 {^c} 0
           52  +  4 {^a + b} 1
           53  +  5 {^b + c} 1
           54  +  6 {^c + a} 0
           55  +  7 {^"c a"} 0
           56  +  8 {a:^a} 1
           57  +  9 {a:^b} 0
           58  +  10 {a:^"a b"} 1
           59  +} {
           60  +  do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match
           61  +}
           62  +
           63  +#-------------------------------------------------------------------------
           64  +# 
           65  +do_execsql_test 3.0 {
           66  +  DELETE FROM x1;
           67  +  INSERT INTO x1 VALUES('b a', 'c a');
           68  +  INSERT INTO x1 VALUES('a a', 'c c');
           69  +  INSERT INTO x1 VALUES('a b', 'a a');
           70  +}
           71  +fts5_aux_test_functions db
           72  +
           73  +foreach {tn expr expect} {
           74  +  1 {^a} {{2 1}}
           75  +  2 {^c AND ^b} {{0 2} {1 0}}
           76  +} {
           77  +  do_execsql_test 3.$tn {
           78  +    SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1
           79  +  } [list $expect]
           80  +}
           81  +
           82  +#-------------------------------------------------------------------------
           83  +# 
           84  +do_execsql_test 3.1 {
           85  +  CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column);
           86  +}
           87  +
           88  +do_catchsql_test 3.2 {
           89  +  SELECT * FROM x2('a + b');
           90  +} {1 {fts5: phrase queries are not supported (detail!=full)}}
           91  +
           92  +do_catchsql_test 3.3 {
           93  +  SELECT * FROM x2('^a');
           94  +} {1 {fts5: phrase queries are not supported (detail!=full)}}
           95  +finish_test
           96  +

Changes to ext/fts5/test/fts5query.test.

    60     60       foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] {
    61     61         set doc [string repeat "$x " 30]
    62     62         execsql { INSERT INTO t1 VALUES($doc) }
    63     63       }
    64     64       execsql COMMIT
    65     65     } {}
    66     66   
    67         -  do_execsql_test 1.$tn.2 {
           67  +  do_execsql_test 2.$tn.2 {
    68     68       INSERT INTO t1(t1) VALUES('integrity-check');
    69     69     }
    70     70   
    71     71     set ret 1
    72     72     foreach x [list a c e g i k m o q s u] {
    73     73       do_execsql_test 2.$tn.3.$ret {
    74     74         SELECT rowid FROM t1 WHERE t1 MATCH $x || '*';
    75     75       } {}
    76     76       incr ret
    77     77     }
    78     78   }
    79     79   
           80  +reset_db
           81  +do_execsql_test 3.0 {
           82  +  CREATE VIRTUAL TABLE x1 USING fts5(a);
           83  +  INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota');
           84  +  INSERT INTO x1(rowid, a) VALUES(1, 'tarago');
           85  +}
           86  +do_execsql_test 3.1 {
           87  +  SELECT rowid FROM x1('t*');
           88  +} {-1000000000000 1}
           89  +
    80     90   
    81     91   finish_test

Changes to ext/icu/icu.c.

    24     24   **
    25     25   **   * Integration of ICU and SQLite collation sequences.
    26     26   **
    27     27   **   * An implementation of the LIKE operator that uses ICU to 
    28     28   **     provide case-independent matching.
    29     29   */
    30     30   
    31         -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
           31  +#if !defined(SQLITE_CORE)                  \
           32  + || defined(SQLITE_ENABLE_ICU)             \
           33  + || defined(SQLITE_ENABLE_ICU_COLLATIONS)
    32     34   
    33     35   /* Include ICU headers */
    34     36   #include <unicode/utypes.h>
    35     37   #include <unicode/uregex.h>
    36     38   #include <unicode/ustring.h>
    37     39   #include <unicode/ucol.h>
    38     40   
................................................................................
    41     43   #ifndef SQLITE_CORE
    42     44     #include "sqlite3ext.h"
    43     45     SQLITE_EXTENSION_INIT1
    44     46   #else
    45     47     #include "sqlite3.h"
    46     48   #endif
    47     49   
           50  +/*
           51  +** This function is called when an ICU function called from within
           52  +** the implementation of an SQL scalar function returns an error.
           53  +**
           54  +** The scalar function context passed as the first argument is 
           55  +** loaded with an error message based on the following two args.
           56  +*/
           57  +static void icuFunctionError(
           58  +  sqlite3_context *pCtx,       /* SQLite scalar function context */
           59  +  const char *zName,           /* Name of ICU function that failed */
           60  +  UErrorCode e                 /* Error code returned by ICU function */
           61  +){
           62  +  char zBuf[128];
           63  +  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
           64  +  zBuf[127] = '\0';
           65  +  sqlite3_result_error(pCtx, zBuf, -1);
           66  +}
           67  +
           68  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
           69  +
    48     70   /*
    49     71   ** Maximum length (in bytes) of the pattern in a LIKE or GLOB
    50     72   ** operator.
    51     73   */
    52     74   #ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
    53     75   # define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000
    54     76   #endif
................................................................................
   220    242     }
   221    243   
   222    244     if( zA && zB ){
   223    245       sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc));
   224    246     }
   225    247   }
   226    248   
   227         -/*
   228         -** This function is called when an ICU function called from within
   229         -** the implementation of an SQL scalar function returns an error.
   230         -**
   231         -** The scalar function context passed as the first argument is 
   232         -** loaded with an error message based on the following two args.
   233         -*/
   234         -static void icuFunctionError(
   235         -  sqlite3_context *pCtx,       /* SQLite scalar function context */
   236         -  const char *zName,           /* Name of ICU function that failed */
   237         -  UErrorCode e                 /* Error code returned by ICU function */
   238         -){
   239         -  char zBuf[128];
   240         -  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
   241         -  zBuf[127] = '\0';
   242         -  sqlite3_result_error(pCtx, zBuf, -1);
   243         -}
   244         -
   245    249   /*
   246    250   ** Function to delete compiled regexp objects. Registered as
   247    251   ** a destructor function with sqlite3_set_auxdata().
   248    252   */
   249    253   static void icuRegexpDelete(void *p){
   250    254     URegularExpression *pExpr = (URegularExpression *)p;
   251    255     uregex_close(pExpr);
................................................................................
   403    407         icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status);
   404    408       }
   405    409       return;
   406    410     }
   407    411     assert( 0 );     /* Unreachable */
   408    412   }
   409    413   
          414  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
          415  +
   410    416   /*
   411    417   ** Collation sequence destructor function. The pCtx argument points to
   412    418   ** a UCollator structure previously allocated using ucol_open().
   413    419   */
   414    420   static void icuCollationDel(void *pCtx){
   415    421     UCollator *p = (UCollator *)pCtx;
   416    422     ucol_close(p);
................................................................................
   497    503       const char *zName;                        /* Function name */
   498    504       unsigned char nArg;                       /* Number of arguments */
   499    505       unsigned short enc;                       /* Optimal text encoding */
   500    506       unsigned char iContext;                   /* sqlite3_user_data() context */
   501    507       void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
   502    508     } scalars[] = {
   503    509       {"icu_load_collation",  2, SQLITE_UTF8,                1, icuLoadCollation},
          510  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
   504    511       {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC,         0, icuRegexpFunc},
   505    512       {"lower",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
   506    513       {"lower",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       0, icuCaseFunc16},
   507    514       {"upper",  1, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
   508    515       {"upper",  2, SQLITE_UTF16|SQLITE_DETERMINISTIC,       1, icuCaseFunc16},
   509    516       {"lower",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
   510    517       {"lower",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuCaseFunc16},
   511    518       {"upper",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
   512    519       {"upper",  2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        1, icuCaseFunc16},
   513    520       {"like",   2, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
   514    521       {"like",   3, SQLITE_UTF8|SQLITE_DETERMINISTIC,        0, icuLikeFunc},
          522  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
   515    523     };
   516    524     int rc = SQLITE_OK;
   517    525     int i;
   518         -
   519    526     
   520    527     for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
   521    528       const struct IcuScalar *p = &scalars[i];
   522    529       rc = sqlite3_create_function(
   523    530           db, p->zName, p->nArg, p->enc, 
   524    531           p->iContext ? (void*)db : (void*)0,
   525    532           p->xFunc, 0, 0

Changes to ext/lsm1/lsm-test/lsmtest.h.

   117    117   ** Functions in wrapper3.c. This file contains the tdb wrapper for lsm.
   118    118   ** The wrapper for lsm is a bit more involved than the others, as it 
   119    119   ** includes code for a couple of different lsm configurations, and for
   120    120   ** various types of fault injection and robustness testing.
   121    121   */
   122    122   int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
   123    123   int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
          124  +int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
   124    125   int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
   125    126   int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
   126    127   int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
   127    128   int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
   128    129   
   129    130   int tdb_lsm_configure(lsm_db *, const char *);
   130    131

Changes to ext/lsm1/lsm-test/lsmtest_tdb.c.

   717    717     const char *zName;
   718    718     const char *zDefaultDb;
   719    719     int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb);
   720    720   } aLib[] = {
   721    721     { "sqlite3",      "testdb.sqlite",    sql_open },
   722    722     { "lsm_small",    "testdb.lsm_small", test_lsm_small_open },
   723    723     { "lsm_lomem",    "testdb.lsm_lomem", test_lsm_lomem_open },
          724  +  { "lsm_lomem2",   "testdb.lsm_lomem2", test_lsm_lomem2_open },
   724    725   #ifdef HAVE_ZLIB
   725    726     { "lsm_zip",      "testdb.lsm_zip",   test_lsm_zip_open },
   726    727   #endif
   727    728     { "lsm",          "testdb.lsm",       test_lsm_open },
   728    729   #ifdef LSM_MUTEX_PTHREADS
   729    730     { "lsm_mt2",      "testdb.lsm_mt2",   test_lsm_mt2 },
   730    731     { "lsm_mt3",      "testdb.lsm_mt3",   test_lsm_mt3 },

Changes to ext/lsm1/lsm-test/lsmtest_tdb3.c.

  1028   1028     const char *zCfg = 
  1029   1029       "page_size=256 block_size=64 autoflush=16 "
  1030   1030       "autocheckpoint=32"
  1031   1031       "mmap=0 "
  1032   1032     ;
  1033   1033     return testLsmOpen(zCfg, zFilename, bClear, ppDb);
  1034   1034   }
         1035  +
         1036  +int test_lsm_lomem2_open(
         1037  +  const char *zSpec, 
         1038  +  const char *zFilename, 
         1039  +  int bClear, 
         1040  +  TestDb **ppDb
         1041  +){
         1042  +    /* "max_freelist=4 autocheckpoint=32" */
         1043  +  const char *zCfg = 
         1044  +    "page_size=512 block_size=64 autoflush=0 mmap=0 "
         1045  +  ;
         1046  +  return testLsmOpen(zCfg, zFilename, bClear, ppDb);
         1047  +}
  1035   1048   
  1036   1049   int test_lsm_zip_open(
  1037   1050     const char *zSpec, 
  1038   1051     const char *zFilename, 
  1039   1052     int bClear, 
  1040   1053     TestDb **ppDb
  1041   1054   ){

Changes to ext/lsm1/lsmInt.h.

   106    106   typedef unsigned short int u16;
   107    107   typedef unsigned int u32;
   108    108   typedef lsm_i64 i64;
   109    109   typedef unsigned long long int u64;
   110    110   #endif
   111    111   
   112    112   /* A page number is a 64-bit integer. */
   113         -typedef i64 Pgno;
          113  +typedef i64 LsmPgno;
   114    114   
   115    115   #ifdef LSM_DEBUG
   116    116   int lsmErrorBkpt(int);
   117    117   #else
   118    118   # define lsmErrorBkpt(x) (x)
   119    119   #endif
   120    120   
................................................................................
   398    398     void **apShm;                   /* Shared memory chunks */
   399    399     ShmHeader *pShmhdr;             /* Live shared-memory header */
   400    400     TreeHeader treehdr;             /* Local copy of tree-header */
   401    401     u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)];
   402    402   };
   403    403   
   404    404   struct Segment {
   405         -  Pgno iFirst;                     /* First page of this run */
   406         -  Pgno iLastPg;                    /* Last page of this run */
   407         -  Pgno iRoot;                      /* Root page number (if any) */
          405  +  LsmPgno iFirst;                  /* First page of this run */
          406  +  LsmPgno iLastPg;                 /* Last page of this run */
          407  +  LsmPgno iRoot;                   /* Root page number (if any) */
   408    408     int nSize;                       /* Size of this run in pages */
   409    409   
   410    410     Redirect *pRedirect;             /* Block redirects (or NULL) */
   411    411   };
   412    412   
   413    413   /*
   414    414   ** iSplitTopic/pSplitKey/nSplitKey:
................................................................................
   452    452   ** access to the associated Level struct.
   453    453   **
   454    454   ** iOutputOff:
   455    455   **   The byte offset to write to next within the last page of the 
   456    456   **   output segment.
   457    457   */
   458    458   struct MergeInput {
   459         -  Pgno iPg;                       /* Page on which next input is stored */
          459  +  LsmPgno iPg;                    /* Page on which next input is stored */
   460    460     int iCell;                      /* Cell containing next input to merge */
   461    461   };
   462    462   struct Merge {
   463    463     int nInput;                     /* Number of input runs being merged */
   464    464     MergeInput *aInput;             /* Array nInput entries in size */
   465    465     MergeInput splitkey;            /* Location in file of current splitkey */
   466    466     int nSkip;                      /* Number of separators entries to skip */
   467    467     int iOutputOff;                 /* Write offset on output page */
   468         -  Pgno iCurrentPtr;               /* Current pointer value */
          468  +  LsmPgno iCurrentPtr;            /* Current pointer value */
   469    469   };
   470    470   
   471    471   /* 
   472    472   ** The first argument to this macro is a pointer to a Segment structure.
   473    473   ** Returns true if the structure instance indicates that the separators
   474    474   ** array is valid.
   475    475   */
................................................................................
   575    575     u32 iCmpId;                     /* Id of compression scheme */
   576    576     Level *pLevel;                  /* Pointer to level 0 of snapshot (or NULL) */
   577    577     i64 iId;                        /* Snapshot id */
   578    578     i64 iLogOff;                    /* Log file offset */
   579    579     Redirect redirect;              /* Block redirection array */
   580    580   
   581    581     /* Used by worker snapshots only */
   582         -  int nBlock;                     /* Number of blocks in database file */
   583         -  Pgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
   584         -  Freelist freelist;              /* Free block list */
   585         -  u32 nWrite;                     /* Total number of pages written to disk */
          582  +  int nBlock;                        /* Number of blocks in database file */
          583  +  LsmPgno aiAppend[LSM_APPLIST_SZ];  /* Append point list */
          584  +  Freelist freelist;                 /* Free block list */
          585  +  u32 nWrite;                        /* Total number of pages written to disk */
   586    586   };
   587    587   #define LSM_INITIAL_SNAPSHOT_ID 11
   588    588   
   589    589   /*
   590    590   ** Functions from file "lsm_ckpt.c".
   591    591   */
   592    592   int lsmCheckpointWrite(lsm_db *, u32 *);
................................................................................
   706    706   
   707    707   int lsmFsPageSize(FileSystem *);
   708    708   void lsmFsSetPageSize(FileSystem *, int);
   709    709   
   710    710   int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId);
   711    711   
   712    712   /* Creating, populating, gobbling and deleting sorted runs. */
   713         -void lsmFsGobble(lsm_db *, Segment *, Pgno *, int);
          713  +void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int);
   714    714   int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *);
   715    715   int lsmFsSortedFinish(FileSystem *, Segment *);
   716    716   int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **);
   717    717   int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *);
   718    718   
   719    719   /* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */
   720    720   lsm_env *lsmFsEnv(FileSystem *);
................................................................................
   723    723   
   724    724   int lsmFsSectorSize(FileSystem *);
   725    725   
   726    726   void lsmSortedSplitkey(lsm_db *, Level *, int *);
   727    727   
   728    728   /* Reading sorted run content. */
   729    729   int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg);
   730         -int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **);
          730  +int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **);
   731    731   int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **);
   732    732   
   733    733   u8 *lsmFsPageData(Page *, int *);
   734    734   int lsmFsPageRelease(Page *);
   735    735   int lsmFsPagePersist(Page *);
   736    736   void lsmFsPageRef(Page *);
   737         -Pgno lsmFsPageNumber(Page *);
          737  +LsmPgno lsmFsPageNumber(Page *);
   738    738   
   739    739   int lsmFsNRead(FileSystem *);
   740    740   int lsmFsNWrite(FileSystem *);
   741    741   
   742    742   int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **);
   743    743   int lsmFsMetaPageRelease(MetaPage *);
   744    744   u8 *lsmFsMetaPageData(MetaPage *, int *);
   745    745   
   746    746   #ifdef LSM_DEBUG
   747    747   int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg);
   748    748   int lsmFsIntegrityCheck(lsm_db *);
   749    749   #endif
   750    750   
   751         -Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);
          751  +LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno);
   752    752   
   753    753   int lsmFsPageWritable(Page *);
   754    754   
   755    755   /* Functions to read, write and sync the log file. */
   756    756   int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr);
   757    757   int lsmFsSyncLog(FileSystem *pFS);
   758    758   int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr);
................................................................................
   764    764   
   765    765   /* And to sync the db file */
   766    766   int lsmFsSyncDb(FileSystem *, int);
   767    767   
   768    768   void lsmFsFlushWaiting(FileSystem *, int *);
   769    769   
   770    770   /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */
   771         -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut);
   772         -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut);
          771  +int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz);
          772  +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut);
   773    773   int lsmConfigMmap(lsm_db *pDb, int *piParam);
   774    774   
   775    775   int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **);
   776    776   int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile);
   777    777   int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock);
   778    778   int lsmEnvTestLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int nLock, int);
   779    779   
................................................................................
   781    781   void lsmEnvShmBarrier(lsm_env *);
   782    782   void lsmEnvShmUnmap(lsm_env *, lsm_file *, int);
   783    783   
   784    784   void lsmEnvSleep(lsm_env *, int);
   785    785   
   786    786   int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);
   787    787   
   788         -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
          788  +int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *);
   789    789   
   790    790   void lsmFsPurgeCache(FileSystem *);
   791    791   
   792    792   /*
   793    793   ** End of functions from "lsm_file.c".
   794    794   **************************************************************************/
   795    795   
   796    796   /* 
   797    797   ** Functions from file "lsm_sorted.c".
   798    798   */
   799         -int lsmInfoPageDump(lsm_db *, Pgno, int, char **);
          799  +int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **);
   800    800   void lsmSortedCleanup(lsm_db *);
   801    801   int lsmSortedAutoWork(lsm_db *, int nUnit);
   802    802   
   803    803   int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *);
   804    804   
   805    805   int lsmSaveWorker(lsm_db *, int);
   806    806

Changes to ext/lsm1/lsm_ckpt.c.

   385    385   static void ckptExportAppendlist(
   386    386     lsm_db *db,                     /* Database connection */
   387    387     CkptBuffer *p,                  /* Checkpoint buffer to write to */
   388    388     int *piOut,                     /* IN/OUT: Offset within checkpoint buffer */
   389    389     int *pRc                        /* IN/OUT: Error code */
   390    390   ){
   391    391     int i;
   392         -  Pgno *aiAppend = db->pWorker->aiAppend;
          392  +  LsmPgno *aiAppend = db->pWorker->aiAppend;
   393    393   
   394    394     for(i=0; i<LSM_APPLIST_SZ; i++){
   395    395       ckptAppend64(p, piOut, aiAppend[i], pRc);
   396    396     }
   397    397   };
   398    398   
   399    399   static int ckptExportSnapshot(

Changes to ext/lsm1/lsm_file.c.

   265    265   **   The lsmFsSortedAppend() function sets the pSeg pointer to point to the
   266    266   **   segment that the new page will be a part of. It is unset by
   267    267   **   lsmFsPagePersist() after the page is written to disk.
   268    268   */
   269    269   struct Page {
   270    270     u8 *aData;                      /* Buffer containing page data */
   271    271     int nData;                      /* Bytes of usable data at aData[] */
   272         -  Pgno iPg;                       /* Page number */
          272  +  LsmPgno iPg;                    /* Page number */
   273    273     int nRef;                       /* Number of outstanding references */
   274    274     int flags;                      /* Combination of PAGE_XXX flags */
   275    275     Page *pHashNext;                /* Next page in hash table slot */
   276    276     Page *pLruNext;                 /* Next page in LRU list */
   277    277     Page *pLruPrev;                 /* Previous page in LRU list */
   278    278     FileSystem *pFS;                /* File system that owns this page */
   279    279   
................................................................................
   328    328   #else
   329    329   # define IOERR_WRAPPER(rc) (rc)
   330    330   #endif
   331    331   
   332    332   #ifdef NDEBUG
   333    333   # define assert_lists_are_ok(x)
   334    334   #else
   335         -static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash);
          335  +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash);
   336    336   
   337    337   static void assert_lists_are_ok(FileSystem *pFS){
   338    338   #if 0
   339    339     Page *p;
   340    340   
   341    341     assert( pFS->nMapLimit>=0 );
   342    342   
................................................................................
   528    528     return LSM_OK;
   529    529   }
   530    530   
   531    531   /*
   532    532   ** Return true if page iReal of the database should be accessed using mmap.
   533    533   ** False otherwise.
   534    534   */
   535         -static int fsMmapPage(FileSystem *pFS, Pgno iReal){
          535  +static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){
   536    536     return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit);
   537    537   }
   538    538   
   539    539   /*
   540    540   ** Given that there are currently nHash slots in the hash table, return 
   541    541   ** the hash key for file iFile, page iPg.
   542    542   */
   543         -static int fsHashKey(int nHash, Pgno iPg){
          543  +static int fsHashKey(int nHash, LsmPgno iPg){
   544    544     return (iPg % nHash);
   545    545   }
   546    546   
   547    547   /*
   548    548   ** This is a helper function for lsmFsOpen(). It opens a single file on
   549    549   ** disk (either the database or log file).
   550    550   */
................................................................................
   876    876   ** Return the page number of the first page on block iBlock. Blocks are
   877    877   ** numbered starting from 1.
   878    878   **
   879    879   ** For a compressed database, page numbers are byte offsets. The first
   880    880   ** page on each block is the byte offset immediately following the 4-byte
   881    881   ** "previous block" pointer at the start of each block.
   882    882   */
   883         -static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
   884         -  Pgno iPg;
          883  +static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
          884  +  LsmPgno iPg;
   885    885     if( pFS->pCompress ){
   886    886       if( iBlock==1 ){
   887    887         iPg = pFS->nMetasize * 2 + 4;
   888    888       }else{
   889         -      iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4;
          889  +      iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
   890    890       }
   891    891     }else{
   892    892       const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   893    893       if( iBlock==1 ){
   894    894         iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize);
   895    895       }else{
   896    896         iPg = 1 + (iBlock-1) * nPagePerBlock;
................................................................................
   903    903   ** Return the page number of the last page on block iBlock. Blocks are
   904    904   ** numbered starting from 1.
   905    905   **
   906    906   ** For a compressed database, page numbers are byte offsets. The first
   907    907   ** page on each block is the byte offset of the byte immediately before 
   908    908   ** the 4-byte "next block" pointer at the end of each block.
   909    909   */
   910         -static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
          910  +static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
   911    911     if( pFS->pCompress ){
   912         -    return pFS->nBlocksize * (Pgno)iBlock - 1 - 4;
          912  +    return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4;
   913    913     }else{
   914    914       const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   915    915       return iBlock * nPagePerBlock;
   916    916     }
   917    917   }
   918    918   
   919    919   /*
   920    920   ** Return the block number of the block that page iPg is located on. 
   921    921   ** Blocks are numbered starting from 1.
   922    922   */
   923         -static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
          923  +static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){
   924    924     if( pFS->pCompress ){
   925    925       return (int)((iPg / pFS->nBlocksize) + 1);
   926    926     }else{
   927    927       return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize)));
   928    928     }
   929    929   }
   930    930   
   931    931   /*
   932    932   ** Return true if page iPg is the last page on its block.
   933    933   **
   934    934   ** This function is only called in non-compressed database mode.
   935    935   */
   936         -static int fsIsLast(FileSystem *pFS, Pgno iPg){
          936  +static int fsIsLast(FileSystem *pFS, LsmPgno iPg){
   937    937     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   938    938     assert( !pFS->pCompress );
   939    939     return ( iPg && (iPg % nPagePerBlock)==0 );
   940    940   }
   941    941   
   942    942   /*
   943    943   ** Return true if page iPg is the first page on its block.
   944    944   **
   945    945   ** This function is only called in non-compressed database mode.
   946    946   */
   947         -static int fsIsFirst(FileSystem *pFS, Pgno iPg){
          947  +static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){
   948    948     const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
   949    949     assert( !pFS->pCompress );
   950    950     return ( (iPg % nPagePerBlock)==1
   951    951           || (iPg<nPagePerBlock && iPg==fsFirstPageOnBlock(pFS, 1))
   952    952     );
   953    953   }
   954    954   
................................................................................
   963    963     }
   964    964     return pPage->aData;
   965    965   }
   966    966   
   967    967   /*
   968    968   ** Return the page number of a page.
   969    969   */
   970         -Pgno lsmFsPageNumber(Page *pPage){
          970  +LsmPgno lsmFsPageNumber(Page *pPage){
   971    971     /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */
   972    972     return pPage ? pPage->iPg : 0;
   973    973   }
   974    974   
   975    975   /*
   976    976   ** Page pPg is currently part of the LRU list belonging to pFS. Remove
   977    977   ** it from the list. pPg->pLruNext and pPg->pLruPrev are cleared by this
................................................................................
  1054   1054   /*
  1055   1055   ** Search the hash-table for page iPg. If an entry is round, return a pointer
  1056   1056   ** to it. Otherwise, return NULL.
  1057   1057   **
  1058   1058   ** Either way, if argument piHash is not NULL set *piHash to the hash slot
  1059   1059   ** number that page iPg would be stored in before returning.
  1060   1060   */
  1061         -static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
         1061  +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){
  1062   1062     Page *p;                        /* Return value */
  1063   1063     int iHash = fsHashKey(pFS->nHash, iPg);
  1064   1064   
  1065   1065     if( piHash ) *piHash = iHash;
  1066   1066     for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
  1067   1067       if( p->iPg==iPg) break;
  1068   1068     }
................................................................................
  1185   1185   }
  1186   1186   
  1187   1187   /*
  1188   1188   ** If page iPg has been redirected according to the redirections in the
  1189   1189   ** object passed as the second argument, return the destination page to
  1190   1190   ** which it is redirected. Otherwise, return a copy of iPg.
  1191   1191   */
  1192         -Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
  1193         -  Pgno iReal = iPg;
         1192  +LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){
         1193  +  LsmPgno iReal = iPg;
  1194   1194   
  1195   1195     if( pRedir ){
  1196   1196       const int nPagePerBlock = (
  1197   1197           pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
  1198   1198       );
  1199   1199       int iBlk = fsPageToBlock(pFS, iPg);
  1200   1200       int i;
  1201   1201       for(i=0; i<pRedir->n; i++){
  1202   1202         int iFrom = pRedir->a[i].iFrom;
  1203   1203         if( iFrom>iBlk ) break;
  1204   1204         if( iFrom==iBlk ){
  1205   1205           int iTo = pRedir->a[i].iTo;
  1206         -        iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
         1206  +        iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
  1207   1207           if( iTo==1 ){
  1208   1208             iReal += (fsFirstPageOnBlock(pFS, 1)-1);
  1209   1209           }
  1210   1210           break;
  1211   1211         }
  1212   1212       }
  1213   1213     }
  1214   1214   
  1215   1215     assert( iReal!=0 );
  1216   1216     return iReal;
  1217   1217   }
  1218   1218   
  1219   1219   /* Required by the circular fsBlockNext<->fsPageGet dependency. */
  1220         -static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *);
         1220  +static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *);
  1221   1221   
  1222   1222   /*
  1223   1223   ** Parameter iBlock is a database file block. This function reads the value 
  1224   1224   ** stored in the blocks "next block" pointer and stores it in *piNext.
  1225   1225   ** LSM_OK is returned if everything is successful, or an LSM error code
  1226   1226   ** otherwise.
  1227   1227   */
................................................................................
  1265   1265     }
  1266   1266     return rc;
  1267   1267   }
  1268   1268   
  1269   1269   /*
  1270   1270   ** Return the page number of the last page on the same block as page iPg.
  1271   1271   */
  1272         -Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){
         1272  +LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){
  1273   1273     return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg));
  1274   1274   }
  1275   1275   
  1276   1276   /*
  1277   1277   ** Read nData bytes of data from offset iOff of the database file into
  1278   1278   ** buffer aData. If this means reading past the end of a block, follow
  1279   1279   ** the block pointer to the next block and continue reading.
................................................................................
  1533   1533   ** to the total number of free bytes before returning.
  1534   1534   **
  1535   1535   ** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code.
  1536   1536   */
  1537   1537   static int fsPageGet(
  1538   1538     FileSystem *pFS,                /* File-system handle */
  1539   1539     Segment *pSeg,                  /* Block redirection to use (or NULL) */
  1540         -  Pgno iPg,                       /* Page id */
         1540  +  LsmPgno iPg,                    /* Page id */
  1541   1541     int noContent,                  /* True to not load content from disk */
  1542   1542     Page **ppPg,                    /* OUT: New page handle */
  1543   1543     int *pnSpace                    /* OUT: Bytes of free space */
  1544   1544   ){
  1545   1545     Page *p;
  1546   1546     int iHash;
  1547   1547     int rc = LSM_OK;
  1548   1548   
  1549   1549     /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is 
  1550   1550     ** not NULL, and the block containing iPg has been redirected, then iReal
  1551   1551     ** is the page number after redirection.  */
  1552         -  Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
         1552  +  LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
  1553   1553   
  1554   1554     assert_lists_are_ok(pFS);
  1555   1555     assert( iPg>=fsFirstPageOnBlock(pFS, 1) );
  1556   1556     assert( iReal>=fsFirstPageOnBlock(pFS, 1) );
  1557   1557     *ppPg = 0;
  1558   1558   
  1559   1559     /* Search the hash-table for the page */
................................................................................
  1685   1685   /*
  1686   1686   ** Return true if the first or last page of segment pRun falls between iFirst
  1687   1687   ** and iLast, inclusive, and pRun is not equal to pIgnore.
  1688   1688   */
  1689   1689   static int fsRunEndsBetween(
  1690   1690     Segment *pRun, 
  1691   1691     Segment *pIgnore, 
  1692         -  Pgno iFirst, 
  1693         -  Pgno iLast
         1692  +  LsmPgno iFirst, 
         1693  +  LsmPgno iLast
  1694   1694   ){
  1695   1695     return (pRun!=pIgnore && (
  1696   1696           (pRun->iFirst>=iFirst && pRun->iFirst<=iLast)
  1697   1697        || (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast)
  1698   1698     ));
  1699   1699   }
  1700   1700   
................................................................................
  1701   1701   /*
  1702   1702   ** Return true if level pLevel contains a segment other than pIgnore for
  1703   1703   ** which the first or last page is between iFirst and iLast, inclusive.
  1704   1704   */
  1705   1705   static int fsLevelEndsBetween(
  1706   1706     Level *pLevel, 
  1707   1707     Segment *pIgnore, 
  1708         -  Pgno iFirst, 
  1709         -  Pgno iLast
         1708  +  LsmPgno iFirst, 
         1709  +  LsmPgno iLast
  1710   1710   ){
  1711   1711     int i;
  1712   1712   
  1713   1713     if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){
  1714   1714       return 1;
  1715   1715     }
  1716   1716     for(i=0; i<pLevel->nRight; i++){
................................................................................
  1729   1729   static int fsFreeBlock(
  1730   1730     FileSystem *pFS,                /* File system object */
  1731   1731     Snapshot *pSnapshot,            /* Worker snapshot */
  1732   1732     Segment *pIgnore,               /* Ignore this run when searching */
  1733   1733     int iBlk                        /* Block number of block to free */
  1734   1734   ){
  1735   1735     int rc = LSM_OK;                /* Return code */
  1736         -  Pgno iFirst;                    /* First page on block iBlk */
  1737         -  Pgno iLast;                     /* Last page on block iBlk */
         1736  +  LsmPgno iFirst;                 /* First page on block iBlk */
         1737  +  LsmPgno iLast;                  /* Last page on block iBlk */
  1738   1738     Level *pLevel;                  /* Used to iterate through levels */
  1739   1739   
  1740   1740     int iIn;                        /* Used to iterate through append points */
  1741   1741     int iOut = 0;                   /* Used to output append points */
  1742         -  Pgno *aApp = pSnapshot->aiAppend;
         1742  +  LsmPgno *aApp = pSnapshot->aiAppend;
  1743   1743   
  1744   1744     iFirst = fsFirstPageOnBlock(pFS, iBlk);
  1745   1745     iLast = fsLastPageOnBlock(pFS, iBlk);
  1746   1746   
  1747   1747     /* Check if any other run in the snapshot has a start or end page 
  1748   1748     ** within this block. If there is such a run, return early. */
  1749   1749     for(pLevel=lsmDbSnapshotLevel(pSnapshot); pLevel; pLevel=pLevel->pNext){
................................................................................
  1807   1807   }
  1808   1808   
  1809   1809   /*
  1810   1810   ** aPgno is an array containing nPgno page numbers. Return the smallest page
  1811   1811   ** number from the array that falls on block iBlk. Or, if none of the pages
  1812   1812   ** in aPgno[] fall on block iBlk, return 0.
  1813   1813   */
  1814         -static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
  1815         -  Pgno iRet = 0;
         1814  +static LsmPgno firstOnBlock(
         1815  +  FileSystem *pFS, 
         1816  +  int iBlk, 
         1817  +  LsmPgno *aPgno, 
         1818  +  int nPgno
         1819  +){
         1820  +  LsmPgno iRet = 0;
  1816   1821     int i;
  1817   1822     for(i=0; i<nPgno; i++){
  1818         -    Pgno iPg = aPgno[i];
         1823  +    LsmPgno iPg = aPgno[i];
  1819   1824       if( fsPageToBlock(pFS, iPg)==iBlk && (iRet==0 || iPg<iRet) ){
  1820   1825         iRet = iPg;
  1821   1826       }
  1822   1827     }
  1823   1828     return iRet;
  1824   1829   }
  1825   1830   
  1826   1831   #ifndef NDEBUG
  1827   1832   /*
  1828   1833   ** Return true if page iPg, which is a part of segment p, lies on
  1829   1834   ** a redirected block. 
  1830   1835   */
  1831         -static int fsPageRedirects(FileSystem *pFS, Segment *p, Pgno iPg){
         1836  +static int fsPageRedirects(FileSystem *pFS, Segment *p, LsmPgno iPg){
  1832   1837     return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg));
  1833   1838   }
  1834   1839   
  1835   1840   /*
  1836   1841   ** Return true if the second argument is not NULL and any of the first
  1837   1842   ** last or root pages lie on a redirected block. 
  1838   1843   */
................................................................................
  1850   1855   ** the segment pRun. This function gobbles from the start of the run to the
  1851   1856   ** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is
  1852   1857   ** the new first page of the run).
  1853   1858   */
  1854   1859   void lsmFsGobble(
  1855   1860     lsm_db *pDb,
  1856   1861     Segment *pRun, 
  1857         -  Pgno *aPgno,
         1862  +  LsmPgno *aPgno,
  1858   1863     int nPgno
  1859   1864   ){
  1860   1865     int rc = LSM_OK;
  1861   1866     FileSystem *pFS = pDb->pFS;
  1862   1867     Snapshot *pSnapshot = pDb->pWorker;
  1863   1868     int iBlk;
  1864   1869   
................................................................................
  1867   1872     assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) );
  1868   1873   
  1869   1874     iBlk = fsPageToBlock(pFS, pRun->iFirst);
  1870   1875     pRun->nSize += (int)(pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk));
  1871   1876   
  1872   1877     while( rc==LSM_OK ){
  1873   1878       int iNext = 0;
  1874         -    Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
         1879  +    LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
  1875   1880       if( iFirst ){
  1876   1881         pRun->iFirst = iFirst;
  1877   1882         break;
  1878   1883       }
  1879   1884       rc = fsBlockNext(pFS, pRun, iBlk, &iNext);
  1880   1885       if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk);
  1881   1886       pRun->nSize -= (int)(
................................................................................
  1901   1906   **   *piNext = iPg + nByte;
  1902   1907   **
  1903   1908   ** But take block overflow and redirection into account.
  1904   1909   */
  1905   1910   static int fsNextPageOffset(
  1906   1911     FileSystem *pFS,                /* File system object */
  1907   1912     Segment *pSeg,                  /* Segment to move within */
  1908         -  Pgno iPg,                       /* Offset of current page */
         1913  +  LsmPgno iPg,                    /* Offset of current page */
  1909   1914     int nByte,                      /* Size of current page including headers */
  1910         -  Pgno *piNext                    /* OUT: Offset of next page. Or zero (EOF) */
         1915  +  LsmPgno *piNext                 /* OUT: Offset of next page. Or zero (EOF) */
  1911   1916   ){
  1912         -  Pgno iNext;
         1917  +  LsmPgno iNext;
  1913   1918     int rc;
  1914   1919   
  1915   1920     assert( pFS->pCompress );
  1916   1921   
  1917   1922     rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext);
  1918   1923     if( pSeg && iNext==pSeg->iLastPg ){
  1919   1924       iNext = 0;
................................................................................
  1935   1940   ** LSM_OK is returned if no error occurs. Otherwise, an lsm error code.
  1936   1941   ** If any value other than LSM_OK is returned, then the final value of
  1937   1942   ** *piPrev is undefined.
  1938   1943   */
  1939   1944   static int fsGetPageBefore(
  1940   1945     FileSystem *pFS, 
  1941   1946     Segment *pSeg, 
  1942         -  Pgno iPg, 
  1943         -  Pgno *piPrev
         1947  +  LsmPgno iPg, 
         1948  +  LsmPgno *piPrev
  1944   1949   ){
  1945   1950     u8 aSz[3];
  1946   1951     int rc;
  1947   1952     i64 iRead;
  1948   1953   
  1949   1954     assert( pFS->pCompress );
  1950   1955   
................................................................................
  1986   1991   **
  1987   1992   ** Page references returned by this function should be released by the 
  1988   1993   ** caller using lsmFsPageRelease().
  1989   1994   */
  1990   1995   int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
  1991   1996     int rc = LSM_OK;
  1992   1997     FileSystem *pFS = pPg->pFS;
  1993         -  Pgno iPg = pPg->iPg;
         1998  +  LsmPgno iPg = pPg->iPg;
  1994   1999   
  1995   2000     assert( 0==fsSegmentRedirects(pFS, pRun) );
  1996   2001     if( pFS->pCompress ){
  1997   2002       int nSpace = pPg->nCompress + 2*3;
  1998   2003   
  1999   2004       do {
  2000   2005         if( eDir>0 ){
................................................................................
  2058   2063   ** already allocated block. If it is possible, the page number of the first
  2059   2064   ** page to use for the new segment is returned. Otherwise zero.
  2060   2065   **
  2061   2066   ** If argument pLvl is not NULL, then this function will not attempt to
  2062   2067   ** start the new segment immediately following any segment that is part
  2063   2068   ** of the right-hand-side of pLvl.
  2064   2069   */
  2065         -static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){
         2070  +static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){
  2066   2071     int i;
  2067         -  Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
  2068         -  Pgno iRet = 0;
         2072  +  LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
         2073  +  LsmPgno iRet = 0;
  2069   2074   
  2070   2075     for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){
  2071   2076       if( (iRet = aiAppend[i]) ){
  2072   2077         if( pLvl ){
  2073   2078           int iBlk = fsPageToBlock(pFS, iRet);
  2074   2079           int j;
  2075   2080           for(j=0; iRet && j<pLvl->nRight; j++){
................................................................................
  2094   2099     Snapshot *pSnapshot,
  2095   2100     Level *pLvl,
  2096   2101     int bDefer,
  2097   2102     Page **ppOut
  2098   2103   ){
  2099   2104     int rc = LSM_OK;
  2100   2105     Page *pPg = 0;
  2101         -  Pgno iApp = 0;
  2102         -  Pgno iNext = 0;
         2106  +  LsmPgno iApp = 0;
         2107  +  LsmPgno iNext = 0;
  2103   2108     Segment *p = &pLvl->lhs;
  2104         -  Pgno iPrev = p->iLastPg;
         2109  +  LsmPgno iPrev = p->iLastPg;
  2105   2110   
  2106   2111     *ppOut = 0;
  2107   2112     assert( p->pRedirect==0 );
  2108   2113   
  2109   2114     if( pFS->pCompress || bDefer ){
  2110   2115       /* In compressed database mode the page is not assigned a page number
  2111   2116       ** or location in the database file at this point. This will be done
................................................................................
  2191   2196       ** Shift this extra block back to the free-block list. 
  2192   2197       **
  2193   2198       ** Otherwise, add the first free page in the last block used by the run
  2194   2199       ** to the lAppend list.
  2195   2200       */
  2196   2201       if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){
  2197   2202         int i;
  2198         -      Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
         2203  +      LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
  2199   2204         for(i=0; i<LSM_APPLIST_SZ; i++){
  2200   2205           if( aiAppend[i]==0 ){
  2201   2206             aiAppend[i] = p->iLastPg+1;
  2202   2207             break;
  2203   2208           }
  2204   2209         }
  2205   2210       }else if( pFS->pCompress==0 ){
................................................................................
  2222   2227   }
  2223   2228   
  2224   2229   /*
  2225   2230   ** Obtain a reference to page number iPg.
  2226   2231   **
  2227   2232   ** Return LSM_OK if successful, or an lsm error code if an error occurs.
  2228   2233   */
  2229         -int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
         2234  +int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){
  2230   2235     return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0);
  2231   2236   }
  2232   2237   
  2233   2238   /*
  2234   2239   ** Obtain a reference to the last page in the segment passed as the 
  2235   2240   ** second argument.
  2236   2241   **
  2237   2242   ** Return LSM_OK if successful, or an lsm error code if an error occurs.
  2238   2243   */
  2239   2244   int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){
  2240   2245     int rc;
  2241         -  Pgno iPg = pSeg->iLastPg;
         2246  +  LsmPgno iPg = pSeg->iLastPg;
  2242   2247     if( pFS->pCompress ){
  2243   2248       int nSpace;
  2244   2249       iPg++;
  2245   2250       do {
  2246   2251         nSpace = 0;
  2247   2252         rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg);
  2248   2253         if( rc==LSM_OK ){
................................................................................
  2362   2367   ** number (*piPg) lies on block iFrom, then calculate the equivalent
  2363   2368   ** page on block iTo and set *piPg to this value before returning.
  2364   2369   */
  2365   2370   static void fsMovePage(
  2366   2371     FileSystem *pFS,                /* File system object */
  2367   2372     int iTo,                        /* Destination block */
  2368   2373     int iFrom,                      /* Source block */
  2369         -  Pgno *piPg                      /* IN/OUT: Page number */
         2374  +  LsmPgno *piPg                   /* IN/OUT: Page number */
  2370   2375   ){
  2371         -  Pgno iPg = *piPg;
         2376  +  LsmPgno iPg = *piPg;
  2372   2377     if( iFrom==fsPageToBlock(pFS, iPg) ){
  2373   2378       const int nPagePerBlock = (
  2374   2379           pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
  2375   2380       );
  2376         -    *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
         2381  +    *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
  2377   2382     }
  2378   2383   }
  2379   2384   
  2380   2385   /*
  2381   2386   ** Copy the contents of block iFrom to block iTo. 
  2382   2387   **
  2383   2388   ** It is safe to assume that there are no outstanding references to pages 
................................................................................
  2453   2458   /*
  2454   2459   ** Append raw data to a segment. Return the database file offset that the
  2455   2460   ** data is written to (this may be used as the page number if the data
  2456   2461   ** being appended is a new page record).
  2457   2462   **
  2458   2463   ** This function is only used in compressed database mode.
  2459   2464   */
  2460         -static Pgno fsAppendData(
         2465  +static LsmPgno fsAppendData(
  2461   2466     FileSystem *pFS,                /* File-system handle */
  2462   2467     Segment *pSeg,                  /* Segment to append to */
  2463   2468     const u8 *aData,                /* Buffer containing data to write */
  2464   2469     int nData,                      /* Size of buffer aData[] in bytes */
  2465   2470     int *pRc                        /* IN/OUT: Error code */
  2466   2471   ){
  2467         -  Pgno iRet = 0;
         2472  +  LsmPgno iRet = 0;
  2468   2473     int rc = *pRc;
  2469   2474     assert( pFS->pCompress );
  2470   2475     if( rc==LSM_OK ){
  2471   2476       int nRem = 0;
  2472   2477       int nWrite = 0;
  2473         -    Pgno iLastOnBlock;
  2474         -    Pgno iApp = pSeg->iLastPg+1;
         2478  +    LsmPgno iLastOnBlock;
         2479  +    LsmPgno iApp = pSeg->iLastPg+1;
  2475   2480   
  2476   2481       /* If this is the first data written into the segment, find an append-point
  2477   2482       ** or allocate a new block.  */
  2478   2483       if( iApp==1 ){
  2479   2484         pSeg->iFirst = iApp = findAppendPoint(pFS, 0);
  2480   2485         if( iApp==0 ){
  2481   2486           int iBlk;
................................................................................
  2515   2520             assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 );
  2516   2521             lsmPutU32(aPtr, iBlk);
  2517   2522             rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr));
  2518   2523           }
  2519   2524   
  2520   2525           /* Set the "prev" pointer on the new block */
  2521   2526           if( rc==LSM_OK ){
  2522         -          Pgno iWrite;
         2527  +          LsmPgno iWrite;
  2523   2528             lsmPutU32(aPtr, fsPageToBlock(pFS, iApp));
  2524   2529             iWrite = fsFirstPageOnBlock(pFS, iBlk);
  2525   2530             rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr));
  2526   2531             if( nRem>0 ) iApp = iWrite;
  2527   2532           }
  2528   2533         }else{
  2529   2534           /* The next block is already allocated. */
................................................................................
  2584   2589   ** LSM_OK is returned if successful, or an lsm error code otherwise. If
  2585   2590   ** any value other than LSM_OK is returned, then the final value of all
  2586   2591   ** output variables is undefined.
  2587   2592   */
  2588   2593   static int fsAppendPage(
  2589   2594     FileSystem *pFS, 
  2590   2595     Segment *pSeg,
  2591         -  Pgno *piNew,
         2596  +  LsmPgno *piNew,
  2592   2597     int *piPrev,
  2593   2598     int *piNext
  2594   2599   ){
  2595         -  Pgno iPrev = pSeg->iLastPg;
         2600  +  LsmPgno iPrev = pSeg->iLastPg;
  2596   2601     int rc;
  2597   2602     assert( iPrev!=0 );
  2598   2603   
  2599   2604     *piPrev = 0;
  2600   2605     *piNext = 0;
  2601   2606   
  2602   2607     if( fsIsLast(pFS, iPrev) ){
................................................................................
  2646   2651     }
  2647   2652     *pRc = rc;
  2648   2653   }
  2649   2654   
  2650   2655   /*
  2651   2656   ** If there exists a hash-table entry associated with page iPg, remove it.
  2652   2657   */
  2653         -static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
         2658  +static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){
  2654   2659     Page *p;
  2655   2660     int iHash = fsHashKey(pFS->nHash, iPg);
  2656   2661   
  2657   2662     for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext);
  2658   2663   
  2659   2664     if( p ){
  2660   2665       assert( p->nRef==0 || (p->flags & PAGE_FREE)==0 );
................................................................................
  2800   2805   int lsmFsSortedPadding(
  2801   2806     FileSystem *pFS, 
  2802   2807     Snapshot *pSnapshot,
  2803   2808     Segment *pSeg
  2804   2809   ){
  2805   2810     int rc = LSM_OK;
  2806   2811     if( pFS->pCompress && pSeg->iFirst ){
  2807         -    Pgno iLast2;
  2808         -    Pgno iLast = pSeg->iLastPg;     /* Current last page of segment */
         2812  +    LsmPgno iLast2;
         2813  +    LsmPgno iLast = pSeg->iLastPg;  /* Current last page of segment */
  2809   2814       int nPad;                       /* Bytes of padding required */
  2810   2815       u8 aSz[3];
  2811   2816   
  2812   2817       iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1;
  2813   2818       assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) );
  2814   2819       nPad = (int)(iLast2 - iLast);
  2815   2820   
................................................................................
  2931   2936   int lsmFsSectorSize(FileSystem *pFS){
  2932   2937     return pFS->szSector;
  2933   2938   }
  2934   2939   
  2935   2940   /*
  2936   2941   ** Helper function for lsmInfoArrayStructure().
  2937   2942   */
  2938         -static Segment *startsWith(Segment *pRun, Pgno iFirst){
         2943  +static Segment *startsWith(Segment *pRun, LsmPgno iFirst){
  2939   2944     return (iFirst==pRun->iFirst) ? pRun : 0;
  2940   2945   }
  2941   2946   
  2942   2947   /*
  2943   2948   ** Return the segment that starts with page iFirst, if any. If no such segment
  2944   2949   ** can be found, return NULL.
  2945   2950   */
  2946         -static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
         2951  +static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){
  2947   2952     Level *pLvl;                    /* Used to iterate through db levels */
  2948   2953     Segment *pSeg = 0;              /* Pointer to segment to return */
  2949   2954   
  2950   2955     for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){
  2951   2956       if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){
  2952   2957         int i;
  2953   2958         for(i=0; i<pLvl->nRight; i++){
................................................................................
  2966   2971   ** eventually free the string using lsmFree().
  2967   2972   **
  2968   2973   ** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
  2969   2974   */
  2970   2975   int lsmInfoArrayStructure(
  2971   2976     lsm_db *pDb, 
  2972   2977     int bBlock,                     /* True for block numbers only */
  2973         -  Pgno iFirst,
         2978  +  LsmPgno iFirst,
  2974   2979     char **pzOut
  2975   2980   ){
  2976   2981     int rc = LSM_OK;
  2977   2982     Snapshot *pWorker;              /* Worker snapshot */
  2978   2983     Segment *pArray = 0;            /* Array to report on */
  2979   2984     int bUnlock = 0;
  2980   2985   
................................................................................
  3031   3036     }
  3032   3037     return rc;
  3033   3038   }
  3034   3039   
  3035   3040   int lsmFsSegmentContainsPg(
  3036   3041     FileSystem *pFS, 
  3037   3042     Segment *pSeg, 
  3038         -  Pgno iPg, 
         3043  +  LsmPgno iPg, 
  3039   3044     int *pbRes
  3040   3045   ){
  3041   3046     Redirect *pRedir = pSeg->pRedirect;
  3042   3047     int rc = LSM_OK;
  3043   3048     int iBlk;
  3044   3049     int iLastBlk;
  3045   3050     int iPgBlock;                   /* Block containing page iPg */
................................................................................
  3060   3065   ** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request.
  3061   3066   ** If successful, *pzOut is set to point to a nul-terminated string 
  3062   3067   ** containing the array structure and LSM_OK is returned. The caller should
  3063   3068   ** eventually free the string using lsmFree().
  3064   3069   **
  3065   3070   ** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
  3066   3071   */
  3067         -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){
         3072  +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){
  3068   3073     int rc = LSM_OK;
  3069   3074     Snapshot *pWorker;              /* Worker snapshot */
  3070   3075     Segment *pSeg = 0;              /* Array to report on */
  3071   3076     int bUnlock = 0;
  3072   3077   
  3073   3078     *pzOut = 0;
  3074   3079     if( iFirst==0 ) return LSM_ERROR;
................................................................................
  3293   3298   #ifndef NDEBUG
  3294   3299   /*
  3295   3300   ** Return true if pPg happens to be the last page in segment pSeg. Or false
  3296   3301   ** otherwise. This function is only invoked as part of assert() conditions.
  3297   3302   */
  3298   3303   int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){
  3299   3304     if( pPg->pFS->pCompress ){
  3300         -    Pgno iNext = 0;
         3305  +    LsmPgno iNext = 0;
  3301   3306       int rc;
  3302   3307       rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext);
  3303   3308       return (rc!=LSM_OK || iNext==0);
  3304   3309     }
  3305   3310     return (pPg->iPg==pSeg->iLastPg);
  3306   3311   }
  3307   3312   #endif

Changes to ext/lsm1/lsm_main.c.

   579    579       case LSM_INFO_DB_STRUCTURE: {
   580    580         char **pzVal = va_arg(ap, char **);
   581    581         rc = lsmStructList(pDb, pzVal);
   582    582         break;
   583    583       }
   584    584   
   585    585       case LSM_INFO_ARRAY_STRUCTURE: {
   586         -      Pgno pgno = va_arg(ap, Pgno);
          586  +      LsmPgno pgno = va_arg(ap, LsmPgno);
   587    587         char **pzVal = va_arg(ap, char **);
   588    588         rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal);
   589    589         break;
   590    590       }
   591    591   
   592    592       case LSM_INFO_ARRAY_PAGES: {
   593         -      Pgno pgno = va_arg(ap, Pgno);
          593  +      LsmPgno pgno = va_arg(ap, LsmPgno);
   594    594         char **pzVal = va_arg(ap, char **);
   595    595         rc = lsmInfoArrayPages(pDb, pgno, pzVal);
   596    596         break;
   597    597       }
   598    598   
   599    599       case LSM_INFO_PAGE_HEX_DUMP:
   600    600       case LSM_INFO_PAGE_ASCII_DUMP: {
   601         -      Pgno pgno = va_arg(ap, Pgno);
          601  +      LsmPgno pgno = va_arg(ap, LsmPgno);
   602    602         char **pzVal = va_arg(ap, char **);
   603    603         int bUnlock = 0;
   604    604         rc = infoGetWorker(pDb, 0, &bUnlock);
   605    605         if( rc==LSM_OK ){
   606    606           int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP);
   607    607           rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal);
   608    608         }
................................................................................
   679    679       int pgsz = lsmFsPageSize(pDb->pFS);
   680    680       int nQuant = LSM_AUTOWORK_QUANT * pgsz;
   681    681       int nBefore;
   682    682       int nAfter;
   683    683       int nDiff;
   684    684   
   685    685       if( nQuant>pDb->nTreeLimit ){
   686         -      nQuant = pDb->nTreeLimit;
          686  +      nQuant = LSM_MAX(pDb->nTreeLimit, pgsz);
   687    687       }
   688    688   
   689    689       nBefore = lsmTreeSize(pDb);
   690    690       if( bDeleteRange ){
   691    691         rc = lsmTreeDelete(pDb, (void *)pKey, nKey, (void *)pVal, nVal);
   692    692       }else{
   693    693         rc = lsmTreeInsert(pDb, (void *)pKey, nKey, (void *)pVal, nVal);

Changes to ext/lsm1/lsm_sorted.c.

    88     88   ** The following macros are used to access a page footer.
    89     89   */
    90     90   #define SEGMENT_NRECORD_OFFSET(pgsz)        ((pgsz) - 2)
    91     91   #define SEGMENT_FLAGS_OFFSET(pgsz)          ((pgsz) - 2 - 2)
    92     92   #define SEGMENT_POINTER_OFFSET(pgsz)        ((pgsz) - 2 - 2 - 8)
    93     93   #define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2)
    94     94   
    95         -#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)
           95  +#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1)
    96     96   
    97     97   #define SEGMENT_BTREE_FLAG     0x0001
    98     98   #define PGFTR_SKIP_NEXT_FLAG   0x0002
    99     99   #define PGFTR_SKIP_THIS_FLAG   0x0004
   100    100   
   101    101   
   102    102   #ifndef LSM_SEGMENTPTR_FREE_THRESHOLD
   103    103   # define LSM_SEGMENTPTR_FREE_THRESHOLD 1024
   104    104   #endif
   105    105   
   106    106   typedef struct SegmentPtr SegmentPtr;
   107         -typedef struct Blob Blob;
          107  +typedef struct LsmBlob LsmBlob;
   108    108   
   109         -struct Blob {
          109  +struct LsmBlob {
   110    110     lsm_env *pEnv;
   111    111     void *pData;
   112    112     int nData;
   113    113     int nAlloc;
   114    114   };
   115    115   
   116    116   /*
................................................................................
   125    125     Level *pLevel;                /* Level object segment is part of */
   126    126     Segment *pSeg;                /* Segment to access */
   127    127   
   128    128     /* Current page. See segmentPtrLoadPage(). */
   129    129     Page *pPg;                    /* Current page */
   130    130     u16 flags;                    /* Copy of page flags field */
   131    131     int nCell;                    /* Number of cells on pPg */
   132         -  Pgno iPtr;                    /* Base cascade pointer */
          132  +  LsmPgno iPtr;                 /* Base cascade pointer */
   133    133   
   134    134     /* Current cell. See segmentPtrLoadCell() */
   135    135     int iCell;                    /* Current record within page pPg */
   136    136     int eType;                    /* Type of current record */
   137         -  Pgno iPgPtr;                  /* Cascade pointer offset */
          137  +  LsmPgno iPgPtr;               /* Cascade pointer offset */
   138    138     void *pKey; int nKey;         /* Key associated with current record */
   139    139     void *pVal; int nVal;         /* Current record value (eType==WRITE only) */
   140    140   
   141    141     /* Blobs used to allocate buffers for pKey and pVal as required */
   142         -  Blob blob1;
   143         -  Blob blob2;
          142  +  LsmBlob blob1;
          143  +  LsmBlob blob2;
   144    144   };
   145    145   
   146    146   /*
   147    147   ** Used to iterate through the keys stored in a b-tree hierarchy from start
   148    148   ** to finish. Only First() and Next() operations are required.
   149    149   **
   150    150   **   btreeCursorNew()
................................................................................
   167    167     int iPg;                        /* Current entry in aPg[]. -1 -> EOF. */
   168    168     BtreePg *aPg;                   /* Pages from root to current location */
   169    169   
   170    170     /* Cache of current entry. pKey==0 for EOF. */
   171    171     void *pKey;
   172    172     int nKey;
   173    173     int eType;
   174         -  Pgno iPtr;
          174  +  LsmPgno iPtr;
   175    175   
   176    176     /* Storage for key, if not local */
   177         -  Blob blob;
          177  +  LsmBlob blob;
   178    178   };
   179    179   
   180    180   
   181    181   /*
   182    182   ** A cursor used for merged searches or iterations through up to one
   183    183   ** Tree structure and any number of sorted files.
   184    184   **
................................................................................
   199    199   */
   200    200   struct MultiCursor {
   201    201     lsm_db *pDb;                    /* Connection that owns this cursor */
   202    202     MultiCursor *pNext;             /* Next cursor owned by connection pDb */
   203    203     int flags;                      /* Mask of CURSOR_XXX flags */
   204    204   
   205    205     int eType;                      /* Cache of current key type */
   206         -  Blob key;                       /* Cache of current key (or NULL) */
   207         -  Blob val;                       /* Cache of current value */
          206  +  LsmBlob key;                    /* Cache of current key (or NULL) */
          207  +  LsmBlob val;                    /* Cache of current value */
   208    208   
   209    209     /* All the component cursors: */
   210    210     TreeCursor *apTreeCsr[2];       /* Up to two tree cursors */
   211    211     int iFree;                      /* Next element of free-list (-ve for eof) */
   212    212     SegmentPtr *aPtr;               /* Array of segment pointers */
   213    213     int nPtr;                       /* Size of array aPtr[] */
   214    214     BtreeCursor *pBtCsr;            /* b-tree cursor (db writes only) */
................................................................................
   217    217     int nTree;                      /* Size of aTree[] array */
   218    218     int *aTree;                     /* Array of comparison results */
   219    219   
   220    220     /* Used by cursors flushing the in-memory tree only */
   221    221     void *pSystemVal;               /* Pointer to buffer to free */
   222    222   
   223    223     /* Used by worker cursors only */
   224         -  Pgno *pPrevMergePtr;
          224  +  LsmPgno *pPrevMergePtr;
   225    225   };
   226    226   
   227    227   /*
   228    228   ** The following constants are used to assign integers to each component
   229    229   ** cursor of a multi-cursor.
   230    230   */
   231    231   #define CURSOR_DATA_TREE0     0   /* Current tree cursor (apTreeCsr[0]) */
................................................................................
   291    291     lsm_db *pDb;                    /* Database handle */
   292    292     Level *pLevel;                  /* Worker snapshot Level being merged */
   293    293     MultiCursor *pCsr;              /* Cursor to read new segment contents from */
   294    294     int bFlush;                     /* True if this is an in-memory tree flush */
   295    295     Hierarchy hier;                 /* B-tree hierarchy under construction */
   296    296     Page *pPage;                    /* Current output page */
   297    297     int nWork;                      /* Number of calls to mergeWorkerNextPage() */
   298         -  Pgno *aGobble;                  /* Gobble point for each input segment */
          298  +  LsmPgno *aGobble;               /* Gobble point for each input segment */
   299    299   
   300         -  Pgno iIndirect;
          300  +  LsmPgno iIndirect;
   301    301     struct SavedPgno {
   302         -    Pgno iPgno;
          302  +    LsmPgno iPgno;
   303    303       int bStore;
   304    304     } aSave[2];
   305    305   };
   306    306   
   307    307   #ifdef LSM_DEBUG_EXPENSIVE
   308    308   static int assertPointersOk(lsm_db *, Segment *, Segment *, int);
   309    309   static int assertBtreeOk(lsm_db *, Segment *);
................................................................................
   367    367     aOut[3] = (u8)((nVal>>32) & 0xFF);
   368    368     aOut[4] = (u8)((nVal>>24) & 0xFF);
   369    369     aOut[5] = (u8)((nVal>>16) & 0xFF);
   370    370     aOut[6] = (u8)((nVal>> 8) & 0xFF);
   371    371     aOut[7] = (u8)((nVal    ) & 0xFF);
   372    372   }
   373    373   
   374         -static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
          374  +static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){
   375    375     assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) );
   376    376     if( pBlob->nAlloc<nData ){
   377    377       pBlob->pData = lsmReallocOrFree(pEnv, pBlob->pData, nData);
   378    378       if( !pBlob->pData ) return LSM_NOMEM_BKPT;
   379    379       pBlob->nAlloc = nData;
   380    380       pBlob->pEnv = pEnv;
   381    381     }
   382    382     return LSM_OK;
   383    383   }
   384    384   
   385         -static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
          385  +static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){
   386    386     if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM;
   387    387     memcpy(pBlob->pData, pData, nData);
   388    388     pBlob->nData = nData;
   389    389     return LSM_OK;
   390    390   }
   391    391   
   392    392   #if 0
   393         -static int sortedBlobCopy(Blob *pDest, Blob *pSrc){
          393  +static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){
   394    394     return sortedBlobSet(pDest, pSrc->pData, pSrc->nData);
   395    395   }
   396    396   #endif
   397    397   
   398         -static void sortedBlobFree(Blob *pBlob){
          398  +static void sortedBlobFree(LsmBlob *pBlob){
   399    399     assert( pBlob->pEnv || pBlob->pData==0 );
   400    400     if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData);
   401         -  memset(pBlob, 0, sizeof(Blob));
          401  +  memset(pBlob, 0, sizeof(LsmBlob));
   402    402   }
   403    403   
   404    404   static int sortedReadData(
   405    405     Segment *pSeg,
   406    406     Page *pPg,
   407    407     int iOff,
   408    408     int nByte,
   409    409     void **ppData,
   410         -  Blob *pBlob
          410  +  LsmBlob *pBlob
   411    411   ){
   412    412     int rc = LSM_OK;
   413    413     int iEnd;
   414    414     int nData;
   415    415     int nCell;
   416    416     u8 *aData;
   417    417   
................................................................................
   477    477     return rc;
   478    478   }
   479    479   
   480    480   static int pageGetNRec(u8 *aData, int nData){
   481    481     return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]);
   482    482   }
   483    483   
   484         -static Pgno pageGetPtr(u8 *aData, int nData){
   485         -  return (Pgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
          484  +static LsmPgno pageGetPtr(u8 *aData, int nData){
          485  +  return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
   486    486   }
   487    487   
   488    488   static int pageGetFlags(u8 *aData, int nData){
   489    489     return (int)lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]);
   490    490   }
   491    491   
   492    492   static u8 *pageGetCell(u8 *aData, int nData, int iCell){
................................................................................
   502    502     return pageGetNRec(aData, nData);
   503    503   }
   504    504   
   505    505   /*
   506    506   ** Return the decoded (possibly relative) pointer value stored in cell 
   507    507   ** iCell from page aData/nData.
   508    508   */
   509         -static Pgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
   510         -  Pgno iRet;                      /* Return value */
          509  +static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
          510  +  LsmPgno iRet;                   /* Return value */
   511    511     u8 *aCell;                      /* Pointer to cell iCell */
   512    512   
   513    513     assert( iCell<pageGetNRec(aData, nData) && iCell>=0 );
   514    514     aCell = pageGetCell(aData, nData, iCell);
   515    515     lsmVarintGet64(&aCell[1], &iRet);
   516    516     return iRet;
   517    517   }
................................................................................
   518    518   
   519    519   static u8 *pageGetKey(
   520    520     Segment *pSeg,                  /* Segment pPg belongs to */
   521    521     Page *pPg,                      /* Page to read from */
   522    522     int iCell,                      /* Index of cell on page to read */
   523    523     int *piTopic,                   /* OUT: Topic associated with this key */
   524    524     int *pnKey,                     /* OUT: Size of key in bytes */
   525         -  Blob *pBlob                     /* If required, use this for dynamic memory */
          525  +  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
   526    526   ){
   527    527     u8 *pKey;
   528    528     int nDummy;
   529    529     int eType;
   530    530     u8 *aData;
   531    531     int nData;
   532    532   
................................................................................
   550    550   
   551    551   static int pageGetKeyCopy(
   552    552     lsm_env *pEnv,                  /* Environment handle */
   553    553     Segment *pSeg,                  /* Segment pPg belongs to */
   554    554     Page *pPg,                      /* Page to read from */
   555    555     int iCell,                      /* Index of cell on page to read */
   556    556     int *piTopic,                   /* OUT: Topic associated with this key */
   557         -  Blob *pBlob                     /* If required, use this for dynamic memory */
          557  +  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
   558    558   ){
   559    559     int rc = LSM_OK;
   560    560     int nKey;
   561    561     u8 *aKey;
   562    562   
   563    563     aKey = pageGetKey(pSeg, pPg, iCell, piTopic, &nKey, pBlob);
   564    564     assert( (void *)aKey!=pBlob->pData || nKey==pBlob->nData );
................................................................................
   565    565     if( (void *)aKey!=pBlob->pData ){
   566    566       rc = sortedBlobSet(pEnv, pBlob, aKey, nKey);
   567    567     }
   568    568   
   569    569     return rc;
   570    570   }
   571    571   
   572         -static Pgno pageGetBtreeRef(Page *pPg, int iKey){
   573         -  Pgno iRef;
          572  +static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){
          573  +  LsmPgno iRef;
   574    574     u8 *aData;
   575    575     int nData;
   576    576     u8 *aCell;
   577    577   
   578    578     aData = fsPageData(pPg, &nData);
   579    579     aCell = pageGetCell(aData, nData, iKey);
   580    580     assert( aCell[0]==0 );
................................................................................
   588    588   #define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i)))
   589    589   #define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i)))
   590    590   
   591    591   static int pageGetBtreeKey(
   592    592     Segment *pSeg,                  /* Segment page pPg belongs to */
   593    593     Page *pPg,
   594    594     int iKey, 
   595         -  Pgno *piPtr, 
          595  +  LsmPgno *piPtr, 
   596    596     int *piTopic, 
   597    597     void **ppKey,
   598    598     int *pnKey,
   599         -  Blob *pBlob
          599  +  LsmBlob *pBlob
   600    600   ){
   601    601     u8 *aData;
   602    602     int nData;
   603    603     u8 *aCell;
   604    604     int eType;
   605    605   
   606    606     aData = fsPageData(pPg, &nData);
................................................................................
   609    609   
   610    610     aCell = pageGetCell(aData, nData, iKey);
   611    611     eType = *aCell++;
   612    612     aCell += GETVARINT64(aCell, *piPtr);
   613    613   
   614    614     if( eType==0 ){
   615    615       int rc;
   616         -    Pgno iRef;                  /* Page number of referenced page */
          616  +    LsmPgno iRef;               /* Page number of referenced page */
   617    617       Page *pRef;
   618    618       aCell += GETVARINT64(aCell, iRef);
   619    619       rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef);
   620    620       if( rc!=LSM_OK ) return rc;
   621    621       pageGetKeyCopy(lsmPageEnv(pPg), pSeg, pRef, 0, &eType, pBlob);
   622    622       lsmFsPageRelease(pRef);
   623    623       *ppKey = pBlob->pData;
................................................................................
   634    634   static int btreeCursorLoadKey(BtreeCursor *pCsr){
   635    635     int rc = LSM_OK;
   636    636     if( pCsr->iPg<0 ){
   637    637       pCsr->pKey = 0;
   638    638       pCsr->nKey = 0;
   639    639       pCsr->eType = 0;
   640    640     }else{
   641         -    Pgno dummy;
          641  +    LsmPgno dummy;
   642    642       int iPg = pCsr->iPg;
   643    643       int iCell = pCsr->aPg[iPg].iCell;
   644    644       while( iCell<0 && (--iPg)>=0 ){
   645    645         iCell = pCsr->aPg[iPg].iCell-1;
   646    646       }
   647    647       if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT;
   648    648   
................................................................................
   679    679     assert( pCsr->iPg==pCsr->nDepth-1 );
   680    680   
   681    681     aData = fsPageData(pPg->pPage, &nData);
   682    682     nCell = pageGetNRec(aData, nData);
   683    683     assert( pPg->iCell<=nCell );
   684    684     pPg->iCell++;
   685    685     if( pPg->iCell==nCell ){
   686         -    Pgno iLoad;
          686  +    LsmPgno iLoad;
   687    687   
   688    688       /* Up to parent. */
   689    689       lsmFsPageRelease(pPg->pPage);
   690    690       pPg->pPage = 0;
   691    691       pCsr->iPg--;
   692    692       while( pCsr->iPg>=0 ){
   693    693         pPg = &pCsr->aPg[pCsr->iPg];
................................................................................
   838    838     MergeInput *p
   839    839   ){
   840    840     int rc = LSM_OK;
   841    841   
   842    842     if( p->iPg ){
   843    843       lsm_env *pEnv = lsmFsEnv(pCsr->pFS);
   844    844       int iCell;                    /* Current cell number on leaf page */
   845         -    Pgno iLeaf;                   /* Page number of current leaf page */
          845  +    LsmPgno iLeaf;                /* Page number of current leaf page */
   846    846       int nDepth;                   /* Depth of b-tree structure */
   847    847       Segment *pSeg = pCsr->pSeg;
   848    848   
   849    849       /* Decode the MergeInput structure */
   850    850       iLeaf = p->iPg;
   851    851       nDepth = (p->iCell & 0x00FF);
   852    852       iCell = (p->iCell >> 8) - 1;
................................................................................
   862    862         pCsr->nDepth = nDepth;
   863    863         pCsr->aPg[pCsr->iPg].iCell = iCell;
   864    864         rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp);
   865    865       }
   866    866   
   867    867       /* Populate any other aPg[] array entries */
   868    868       if( rc==LSM_OK && nDepth>1 ){
   869         -      Blob blob = {0,0,0};
          869  +      LsmBlob blob = {0,0,0};
   870    870         void *pSeek;
   871    871         int nSeek;
   872    872         int iTopicSeek;
   873    873         int iPg = 0;
   874    874         int iLoad = (int)pSeg->iRoot;
   875    875         Page *pPg = pCsr->aPg[nDepth-1].pPage;
   876    876    
................................................................................
   879    879           ** In this case, set the iTopicSeek/pSeek/nSeek key to a value
   880    880           ** greater than any real key.  */
   881    881           assert( iCell==-1 );
   882    882           iTopicSeek = 1000;
   883    883           pSeek = 0;
   884    884           nSeek = 0;
   885    885         }else{
   886         -        Pgno dummy;
          886  +        LsmPgno dummy;
   887    887           rc = pageGetBtreeKey(pSeg, pPg,
   888    888               0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob
   889    889           );
   890    890         }
   891    891   
   892    892         do {
   893    893           Page *pPg2;
................................................................................
   908    908             iMax = iCell2-1;
   909    909             iMin = 0;
   910    910   
   911    911             while( iMax>=iMin ){
   912    912               int iTry = (iMin+iMax)/2;
   913    913               void *pKey; int nKey;         /* Key for cell iTry */
   914    914               int iTopic;                   /* Topic for key pKeyT/nKeyT */
   915         -            Pgno iPtr;                    /* Pointer for cell iTry */
          915  +            LsmPgno iPtr;                 /* Pointer for cell iTry */
   916    916               int res;                      /* (pSeek - pKeyT) */
   917    917   
   918    918               rc = pageGetBtreeKey(
   919    919                   pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob
   920    920               );
   921    921               if( rc!=LSM_OK ) break;
   922    922   
................................................................................
   951    951         u8 *aData;
   952    952         int nData;
   953    953   
   954    954         pBtreePg = &pCsr->aPg[pCsr->iPg];
   955    955         aData = fsPageData(pBtreePg->pPage, &nData);
   956    956         pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1);
   957    957         if( pBtreePg->iCell<0 ){
   958         -        Pgno dummy;
          958  +        LsmPgno dummy;
   959    959           int i;
   960    960           for(i=pCsr->iPg-1; i>=0; i--){
   961    961             if( pCsr->aPg[i].iCell>0 ) break;
   962    962           }
   963    963           assert( i>=0 );
   964    964           rc = pageGetBtreeKey(pSeg,
   965    965               pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1,
................................................................................
  1026   1026   }
  1027   1027   
  1028   1028   static int segmentPtrReadData(
  1029   1029     SegmentPtr *pPtr,
  1030   1030     int iOff,
  1031   1031     int nByte,
  1032   1032     void **ppData,
  1033         -  Blob *pBlob
         1033  +  LsmBlob *pBlob
  1034   1034   ){
  1035   1035     return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob);
  1036   1036   }
  1037   1037   
  1038   1038   static int segmentPtrNextPage(
  1039   1039     SegmentPtr *pPtr,              /* Load page into this SegmentPtr object */
  1040   1040     int eDir                       /* +1 for next(), -1 for prev() */
................................................................................
  1119   1119   
  1120   1120     pSeg = sortedSplitkeySegment(pLevel);
  1121   1121     if( rc==LSM_OK ){
  1122   1122       rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg);
  1123   1123     }
  1124   1124     if( rc==LSM_OK ){
  1125   1125       int iTopic;
  1126         -    Blob blob = {0, 0, 0, 0};
         1126  +    LsmBlob blob = {0, 0, 0, 0};
  1127   1127       u8 *aData;
  1128   1128       int nData;
  1129   1129     
  1130   1130       aData = lsmFsPageData(pPg, &nData);
  1131   1131       if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){
  1132   1132         void *pKey;
  1133   1133         int nKey;
  1134         -      Pgno dummy;
         1134  +      LsmPgno dummy;
  1135   1135         rc = pageGetBtreeKey(pSeg,
  1136   1136             pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob
  1137   1137         );
  1138   1138         if( rc==LSM_OK && blob.pData!=pKey ){
  1139   1139           rc = sortedBlobSet(pEnv, &blob, pKey, nKey);
  1140   1140         }
  1141   1141       }else{
................................................................................
  1338   1338   */
  1339   1339   static int assertKeyLocation(
  1340   1340     MultiCursor *pCsr, 
  1341   1341     SegmentPtr *pPtr, 
  1342   1342     void *pKey, int nKey
  1343   1343   ){
  1344   1344     lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS);
  1345         -  Blob blob = {0, 0, 0};
         1345  +  LsmBlob blob = {0, 0, 0};
  1346   1346     int eDir;
  1347   1347     int iTopic = 0;                 /* TODO: Fix me */
  1348   1348   
  1349   1349     for(eDir=-1; eDir<=1; eDir+=2){
  1350   1350       Page *pTest = pPtr->pPg;
  1351   1351   
  1352   1352       lsmFsPageRef(pTest);
................................................................................
  1484   1484     return rc;
  1485   1485   }
  1486   1486   
  1487   1487   static int ptrFwdPointer(
  1488   1488     Page *pPage,
  1489   1489     int iCell,
  1490   1490     Segment *pSeg,
  1491         -  Pgno *piPtr,
         1491  +  LsmPgno *piPtr,
  1492   1492     int *pbFound
  1493   1493   ){
  1494   1494     Page *pPg = pPage;
  1495   1495     int iFirst = iCell;
  1496   1496     int rc = LSM_OK;
  1497   1497   
  1498   1498     do {
................................................................................
  1569   1569   **   much better if the multi-cursor could do this lazily - only seek to the
  1570   1570   **   level (N+1) page after the user has moved the cursor on level N passed
  1571   1571   **   the big range-delete.
  1572   1572   */
  1573   1573   static int segmentPtrFwdPointer(
  1574   1574     MultiCursor *pCsr,              /* Multi-cursor pPtr belongs to */
  1575   1575     SegmentPtr *pPtr,               /* Segment-pointer to extract FC ptr from */
  1576         -  Pgno *piPtr                     /* OUT: FC pointer value */
         1576  +  LsmPgno *piPtr                  /* OUT: FC pointer value */
  1577   1577   ){
  1578   1578     Level *pLvl = pPtr->pLevel;
  1579   1579     Level *pNext = pLvl->pNext;
  1580   1580     Page *pPg = pPtr->pPg;
  1581   1581     int rc;
  1582   1582     int bFound;
  1583         -  Pgno iOut = 0;
         1583  +  LsmPgno iOut = 0;
  1584   1584   
  1585   1585     if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){
  1586   1586       if( pNext==0 
  1587   1587           || (pNext->nRight==0 && pNext->lhs.iRoot)
  1588   1588           || (pNext->nRight!=0 && pNext->aRhs[0].iRoot)
  1589   1589         ){
  1590   1590         /* Do nothing. The pointer will not be used anyway. */
................................................................................
  1637   1637     int *pbStop
  1638   1638   ){
  1639   1639     int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp;
  1640   1640     int res = 0;                        /* Result of comparison operation */
  1641   1641     int rc = LSM_OK;
  1642   1642     int iMin;
  1643   1643     int iMax;
  1644         -  Pgno iPtrOut = 0;
         1644  +  LsmPgno iPtrOut = 0;
  1645   1645   
  1646   1646     /* If the current page contains an oversized entry, then there are no
  1647   1647     ** pointers to one or more of the subsequent pages in the sorted run.
  1648   1648     ** The following call ensures that the segment-ptr points to the correct 
  1649   1649     ** page in this case.  */
  1650   1650     rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey);
  1651   1651     iPtrOut = pPtr->iPtr;
................................................................................
  1764   1764   }
  1765   1765   
  1766   1766   static int seekInBtree(
  1767   1767     MultiCursor *pCsr,              /* Multi-cursor object */
  1768   1768     Segment *pSeg,                  /* Seek within this segment */
  1769   1769     int iTopic,
  1770   1770     void *pKey, int nKey,           /* Key to seek to */
  1771         -  Pgno *aPg,                      /* OUT: Page numbers */
         1771  +  LsmPgno *aPg,                   /* OUT: Page numbers */
  1772   1772     Page **ppPg                     /* OUT: Leaf (sorted-run) page reference */
  1773   1773   ){
  1774   1774     int i = 0;
  1775   1775     int rc;
  1776   1776     int iPg;
  1777   1777     Page *pPg = 0;
  1778         -  Blob blob = {0, 0, 0};
         1778  +  LsmBlob blob = {0, 0, 0};
  1779   1779   
  1780   1780     iPg = (int)pSeg->iRoot;
  1781   1781     do {
  1782         -    Pgno *piFirst = 0;
         1782  +    LsmPgno *piFirst = 0;
  1783   1783       if( aPg ){
  1784   1784         aPg[i++] = iPg;
  1785   1785         piFirst = &aPg[i];
  1786   1786       }
  1787   1787   
  1788   1788       rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg);
  1789   1789       assert( rc==LSM_OK || pPg==0 );
................................................................................
  1804   1804   
  1805   1805         iMin = 0;
  1806   1806         iMax = nRec-1;
  1807   1807         while( iMax>=iMin ){
  1808   1808           int iTry = (iMin+iMax)/2;
  1809   1809           void *pKeyT; int nKeyT;       /* Key for cell iTry */
  1810   1810           int iTopicT;                  /* Topic for key pKeyT/nKeyT */
  1811         -        Pgno iPtr;                    /* Pointer associated with cell iTry */
         1811  +        LsmPgno iPtr;                 /* Pointer associated with cell iTry */
  1812   1812           int res;                      /* (pKey - pKeyT) */
  1813   1813   
  1814   1814           rc = pageGetBtreeKey(
  1815   1815               pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob
  1816   1816           );
  1817   1817           if( rc!=LSM_OK ) break;
  1818   1818           if( piFirst && pKeyT==blob.pData ){
................................................................................
  1895   1895   */
  1896   1896   static int seekInLevel(
  1897   1897     MultiCursor *pCsr,              /* Sorted cursor object to seek */
  1898   1898     SegmentPtr *aPtr,               /* Pointer to array of (nRhs+1) SPs */
  1899   1899     int eSeek,                      /* Search bias - see above */
  1900   1900     int iTopic,                     /* Key topic to search for */
  1901   1901     void *pKey, int nKey,           /* Key to search for */
  1902         -  Pgno *piPgno,                   /* IN/OUT: fraction cascade pointer (or 0) */
         1902  +  LsmPgno *piPgno,                /* IN/OUT: fraction cascade pointer (or 0) */
  1903   1903     int *pbStop                     /* OUT: See above */
  1904   1904   ){
  1905   1905     Level *pLvl = aPtr[0].pLevel;   /* Level to seek within */
  1906   1906     int rc = LSM_OK;                /* Return code */
  1907   1907     int iOut = 0;                   /* Pointer to return to caller */
  1908   1908     int res = -1;                   /* Result of xCmp(pKey, split) */
  1909   1909     int nRhs = pLvl->nRight;        /* Number of right-hand-side segments */
................................................................................
  3051   3051     void *pKey, int nKey, 
  3052   3052     int eSeek
  3053   3053   ){
  3054   3054     int eESeek = eSeek;             /* Effective eSeek parameter */
  3055   3055     int bStop = 0;                  /* Set to true to halt search operation */
  3056   3056     int rc = LSM_OK;                /* Return code */
  3057   3057     int iPtr = 0;                   /* Used to iterate through pCsr->aPtr[] */
  3058         -  Pgno iPgno = 0;                 /* FC pointer value */
         3058  +  LsmPgno iPgno = 0;              /* FC pointer value */
  3059   3059   
  3060   3060     assert( pCsr->apTreeCsr[0]==0 || iTopic==0 );
  3061   3061     assert( pCsr->apTreeCsr[1]==0 || iTopic==0 );
  3062   3062   
  3063   3063     if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE;
  3064   3064   
  3065   3065     assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE );
................................................................................
  3533   3533   ** differences are:
  3534   3534   **
  3535   3535   **   1. The record format is (usually, see below) as follows:
  3536   3536   **
  3537   3537   **         + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR),
  3538   3538   **         + Absolute pointer value (varint),
  3539   3539   **         + Number of bytes in key (varint),
  3540         -**         + Blob containing key data.
         3540  +**         + LsmBlob containing key data.
  3541   3541   **
  3542   3542   **   2. All pointer values are stored as absolute values (not offsets 
  3543   3543   **      relative to the footer pointer value).
  3544   3544   **
  3545   3545   **   3. Each pointer that is part of a record points to a page that 
  3546   3546   **      contains keys smaller than the records key (note: not "equal to or
  3547   3547   **      smaller than - smaller than").
................................................................................
  3567   3567   **
  3568   3568   ** See function seekInBtree() for the code that traverses b-tree pages.
  3569   3569   */
  3570   3570   
  3571   3571   static int mergeWorkerBtreeWrite(
  3572   3572     MergeWorker *pMW,
  3573   3573     u8 eType,
  3574         -  Pgno iPtr,
  3575         -  Pgno iKeyPg,
         3574  +  LsmPgno iPtr,
         3575  +  LsmPgno iKeyPg,
  3576   3576     void *pKey,
  3577   3577     int nKey
  3578   3578   ){
  3579   3579     Hierarchy *p = &pMW->hier;
  3580   3580     lsm_db *pDb = pMW->pDb;         /* Database handle */
  3581   3581     int rc = LSM_OK;                /* Return Code */
  3582   3582     int iLevel;                     /* Level of b-tree hierachy to write to */
................................................................................
  3678   3678   
  3679   3679     return rc;
  3680   3680   }
  3681   3681   
  3682   3682   static int mergeWorkerBtreeIndirect(MergeWorker *pMW){
  3683   3683     int rc = LSM_OK;
  3684   3684     if( pMW->iIndirect ){
  3685         -    Pgno iKeyPg = pMW->aSave[1].iPgno;
         3685  +    LsmPgno iKeyPg = pMW->aSave[1].iPgno;
  3686   3686       rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0);
  3687   3687       pMW->iIndirect = 0;
  3688   3688     }
  3689   3689     return rc;
  3690   3690   }
  3691   3691   
  3692   3692   /*
................................................................................
  3699   3699   static int mergeWorkerPushHierarchy(
  3700   3700     MergeWorker *pMW,               /* Merge worker object */
  3701   3701     int iTopic,                     /* Topic value for this key */
  3702   3702     void *pKey,                     /* Pointer to key buffer */
  3703   3703     int nKey                        /* Size of pKey buffer in bytes */
  3704   3704   ){
  3705   3705     int rc = LSM_OK;                /* Return Code */
  3706         -  Pgno iPtr;                      /* Pointer value to accompany pKey/nKey */
         3706  +  LsmPgno iPtr;                   /* Pointer value to accompany pKey/nKey */
  3707   3707   
  3708   3708     assert( pMW->aSave[0].bStore==0 );
  3709   3709     assert( pMW->aSave[1].bStore==0 );
  3710   3710     rc = mergeWorkerBtreeIndirect(pMW);
  3711   3711   
  3712   3712     /* Obtain the absolute pointer value to store along with the key in the
  3713   3713     ** page body. This pointer points to a page that contains keys that are
................................................................................
  3730   3730   }
  3731   3731   
  3732   3732   static int mergeWorkerFinishHierarchy(
  3733   3733     MergeWorker *pMW                /* Merge worker object */
  3734   3734   ){
  3735   3735     int i;                          /* Used to loop through apHier[] */
  3736   3736     int rc = LSM_OK;                /* Return code */
  3737         -  Pgno iPtr;                      /* New right-hand-child pointer value */
         3737  +  LsmPgno iPtr;                   /* New right-hand-child pointer value */
  3738   3738   
  3739   3739     iPtr = pMW->aSave[0].iPgno;
  3740   3740     for(i=0; i<pMW->hier.nHier && rc==LSM_OK; i++){
  3741   3741       Page *pPg = pMW->hier.apHier[i];
  3742   3742       int nData;                    /* Size of aData[] in bytes */
  3743   3743       u8 *aData;                    /* Page data for pPg */
  3744   3744   
................................................................................
  3826   3826   ** zero records. The flags field is cleared. The page footer pointer field
  3827   3827   ** is set to iFPtr.
  3828   3828   **
  3829   3829   ** If successful, LSM_OK is returned. Otherwise, an error code.
  3830   3830   */
  3831   3831   static int mergeWorkerNextPage(
  3832   3832     MergeWorker *pMW,               /* Merge worker object to append page to */
  3833         -  Pgno iFPtr                      /* Pointer value for footer of new page */
         3833  +  LsmPgno iFPtr                   /* Pointer value for footer of new page */
  3834   3834   ){
  3835   3835     int rc = LSM_OK;                /* Return code */
  3836   3836     Page *pNext = 0;                /* New page appended to run */
  3837   3837     lsm_db *pDb = pMW->pDb;         /* Database handle */
  3838   3838   
  3839   3839     rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext);
  3840   3840     assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress );
................................................................................
  3995   3995       nHdr = 1 + lsmVarintLen32(iRPtr) + lsmVarintLen32(nKey);
  3996   3996       if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal);
  3997   3997   
  3998   3998       /* If the entire header will not fit on page pPg, or if page pPg is 
  3999   3999       ** marked read-only, advance to the next page of the output run. */
  4000   4000       iOff = pMerge->iOutputOff;
  4001   4001       if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){
         4002  +      if( iOff>=0 && pPg ){
         4003  +        /* Zero any free space on the page */
         4004  +        assert( aData );
         4005  +        memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff);
         4006  +      }
  4002   4007         iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
  4003   4008         iRPtr = iPtr - iFPtr;
  4004   4009         iOff = 0;
  4005   4010         nRec = 0;
  4006   4011         rc = mergeWorkerNextPage(pMW, iFPtr);
  4007   4012         pPg = pMW->pPage;
  4008   4013       }
................................................................................
  4065   4070     int i;                          /* Iterator variable */
  4066   4071     int rc = *pRc;
  4067   4072     MultiCursor *pCsr = pMW->pCsr;
  4068   4073   
  4069   4074     /* Unless the merge has finished, save the cursor position in the
  4070   4075     ** Merge.aInput[] array. See function mergeWorkerInit() for the 
  4071   4076     ** code to restore a cursor position based on aInput[].  */
  4072         -  if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){
         4077  +  if( rc==LSM_OK && pCsr ){
  4073   4078       Merge *pMerge = pMW->pLevel->pMerge;
  4074         -    int bBtree = (pCsr->pBtCsr!=0);
  4075         -    int iPtr;
         4079  +    if( lsmMCursorValid(pCsr) ){
         4080  +      int bBtree = (pCsr->pBtCsr!=0);
         4081  +      int iPtr;
  4076   4082   
  4077         -    /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
  4078         -    assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
  4079         -    assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
         4083  +      /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
         4084  +      assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
         4085  +      assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
  4080   4086   
  4081         -    for(i=0; i<(pMerge->nInput-bBtree); i++){
  4082         -      SegmentPtr *pPtr = &pCsr->aPtr[i];
  4083         -      if( pPtr->pPg ){
  4084         -        pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
  4085         -        pMerge->aInput[i].iCell = pPtr->iCell;
         4087  +      for(i=0; i<(pMerge->nInput-bBtree); i++){
         4088  +        SegmentPtr *pPtr = &pCsr->aPtr[i];
         4089  +        if( pPtr->pPg ){
         4090  +          pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
         4091  +          pMerge->aInput[i].iCell = pPtr->iCell;
         4092  +        }else{
         4093  +          pMerge->aInput[i].iPg = 0;
         4094  +          pMerge->aInput[i].iCell = 0;
         4095  +        }
         4096  +      }
         4097  +      if( bBtree && pMerge->nInput ){
         4098  +        assert( i==pCsr->nPtr );
         4099  +        btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
         4100  +      }
         4101  +
         4102  +      /* Store the location of the split-key */
         4103  +      iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
         4104  +      if( iPtr<pCsr->nPtr ){
         4105  +        pMerge->splitkey = pMerge->aInput[iPtr];
  4086   4106         }else{
  4087         -        pMerge->aInput[i].iPg = 0;
  4088         -        pMerge->aInput[i].iCell = 0;
         4107  +        btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
  4089   4108         }
  4090   4109       }
  4091         -    if( bBtree && pMerge->nInput ){
  4092         -      assert( i==pCsr->nPtr );
  4093         -      btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
  4094         -    }
  4095   4110   
  4096         -    /* Store the location of the split-key */
  4097         -    iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
  4098         -    if( iPtr<pCsr->nPtr ){
  4099         -      pMerge->splitkey = pMerge->aInput[iPtr];
  4100         -    }else{
  4101         -      btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
         4111  +    /* Zero any free space left on the final page. This helps with
         4112  +    ** compression if using a compression hook. And prevents valgrind
         4113  +    ** from complaining about uninitialized byte passed to write(). */
         4114  +    if( pMW->pPage ){
         4115  +      int nData;
         4116  +      u8 *aData = fsPageData(pMW->pPage, &nData);
         4117  +      int iOff = pMerge->iOutputOff;
         4118  +      int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData));
         4119  +      memset(&aData[iOff], 0, iEof - iOff);
  4102   4120       }
  4103   4121       
  4104   4122       pMerge->iOutputOff = -1;
  4105   4123     }
  4106   4124   
  4107   4125     lsmMCursorClose(pCsr, 0);
  4108   4126   
................................................................................
  4196   4214   
  4197   4215   static int mergeWorkerStep(MergeWorker *pMW){
  4198   4216     lsm_db *pDb = pMW->pDb;       /* Database handle */
  4199   4217     MultiCursor *pCsr;            /* Cursor to read input data from */
  4200   4218     int rc = LSM_OK;              /* Return code */
  4201   4219     int eType;                    /* SORTED_SEPARATOR, WRITE or DELETE */
  4202   4220     void *pKey; int nKey;         /* Key */
  4203         -  Pgno iPtr;
         4221  +  LsmPgno iPtr;
  4204   4222     int iVal;
  4205   4223   
  4206   4224     pCsr = pMW->pCsr;
  4207   4225   
  4208   4226     /* Pull the next record out of the source cursor. */
  4209   4227     lsmMCursorKey(pCsr, &pKey, &nKey);
  4210   4228     eType = pCsr->eType;
................................................................................
  4349   4367         multiCursorIgnoreDelete(pCsr);
  4350   4368       }
  4351   4369     }
  4352   4370   
  4353   4371     if( rc!=LSM_OK ){
  4354   4372       lsmMCursorClose(pCsr, 0);
  4355   4373     }else{
  4356         -    Pgno iLeftPtr = 0;
         4374  +    LsmPgno iLeftPtr = 0;
  4357   4375       Merge merge;                  /* Merge object used to create new level */
  4358   4376       MergeWorker mergeworker;      /* MergeWorker object for the same purpose */
  4359   4377   
  4360   4378       memset(&merge, 0, sizeof(Merge));
  4361   4379       memset(&mergeworker, 0, sizeof(MergeWorker));
  4362   4380   
  4363   4381       pNew->pMerge = &merge;
................................................................................
  4526   4544     assert( pDb->pWorker );
  4527   4545     assert( pLevel->pMerge );
  4528   4546     assert( pLevel->nRight>0 );
  4529   4547   
  4530   4548     memset(pMW, 0, sizeof(MergeWorker));
  4531   4549     pMW->pDb = pDb;
  4532   4550     pMW->pLevel = pLevel;
  4533         -  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno) * pLevel->nRight, &rc);
         4551  +  pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc);
  4534   4552   
  4535   4553     /* Create a multi-cursor to read the data to write to the new
  4536   4554     ** segment. The new segment contains:
  4537   4555     **
  4538   4556     **   1. Records from LHS of each of the nMerge levels being merged.
  4539   4557     **   2. Separators from either the last level being merged, or the
  4540   4558     **      separators attached to the LHS of the following level, or neither.
................................................................................
  4608   4626     lsm_db *pDb,                    /* Worker connection */
  4609   4627     MultiCursor *pCsr,              /* Multi-cursor being used for a merge */
  4610   4628     int iGobble                     /* pCsr->aPtr[] entry to operate on */
  4611   4629   ){
  4612   4630     int rc = LSM_OK;
  4613   4631     if( rtTopic(pCsr->eType)==0 ){
  4614   4632       Segment *pSeg = pCsr->aPtr[iGobble].pSeg;
  4615         -    Pgno *aPg;
         4633  +    LsmPgno *aPg;
  4616   4634       int nPg;
  4617   4635   
  4618   4636       /* Seek from the root of the b-tree to the segment leaf that may contain
  4619   4637       ** a key equal to the one multi-cursor currently points to. Record the
  4620   4638       ** page number of each b-tree page and the leaf. The segment may be
  4621   4639       ** gobbled up to (but not including) the first of these page numbers.
  4622   4640       */
  4623   4641       assert( pSeg->iRoot>0 );
  4624         -    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc);
         4642  +    aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc);
  4625   4643       if( rc==LSM_OK ){
  4626   4644         rc = seekInBtree(pCsr, pSeg, 
  4627   4645             rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0
  4628   4646         ); 
  4629   4647       }
  4630   4648   
  4631   4649       if( rc==LSM_OK ){
................................................................................
  5230   5248       nRem -= nPg;
  5231   5249       if( nPg ) bDirty = 1;
  5232   5250     }
  5233   5251   
  5234   5252     /* If the in-memory part of the free-list is too large, write a new 
  5235   5253     ** top-level containing just the in-memory free-list entries to disk. */
  5236   5254     if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){
  5237         -    int nPg = 0;
  5238   5255       while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
         5256  +      int nPg = 0;
  5239   5257         rc = sortedWork(pDb, 16, nMerge, 1, &nPg);
  5240   5258         nRem -= nPg;
  5241   5259       }
  5242   5260       if( rc==LSM_OK ){
  5243   5261         rc = sortedNewFreelistOnly(pDb);
  5244   5262       }
  5245         -    nRem -= nPg;
  5246         -    if( nPg ) bDirty = 1;
         5263  +    bDirty = 1;
  5247   5264     }
  5248   5265   
  5249   5266     if( rc==LSM_OK ){
  5250   5267       *pnWrite = (nMax - nRem);
  5251   5268       *pbCkpt = (bCkpt && nRem<=0);
  5252   5269       if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0
  5253   5270        && pWorker->pLevel 
................................................................................
  5444   5461   /*
  5445   5462   ** Return a string representation of the segment passed as the only argument.
  5446   5463   ** Space for the returned string is allocated using lsmMalloc(), and should
  5447   5464   ** be freed by the caller using lsmFree().
  5448   5465   */
  5449   5466   static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){
  5450   5467     int nSize = pSeg->nSize;
  5451         -  Pgno iRoot = pSeg->iRoot;
  5452         -  Pgno iFirst = pSeg->iFirst;
  5453         -  Pgno iLast = pSeg->iLastPg;
         5468  +  LsmPgno iRoot = pSeg->iRoot;
         5469  +  LsmPgno iFirst = pSeg->iFirst;
         5470  +  LsmPgno iLast = pSeg->iLastPg;
  5454   5471     char *z;
  5455   5472   
  5456   5473     char *z1;
  5457   5474     char *z2;
  5458   5475     int nPad;
  5459   5476   
  5460   5477     z1 = lsmMallocPrintf(pEnv, "%d.%d", iFirst, iLast);
................................................................................
  5505   5522       aBuf[0] = '\0';
  5506   5523     }
  5507   5524   
  5508   5525     return i;
  5509   5526   }
  5510   5527   
  5511   5528   void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
  5512         -  Blob blob = {0, 0, 0};         /* Blob used for keys */
         5529  +  LsmBlob blob = {0, 0, 0};       /* LsmBlob used for keys */
  5513   5530     LsmString s;
  5514   5531     int i;
  5515   5532   
  5516   5533     int nRec;
  5517   5534     int iPtr;
  5518   5535     int flags;
  5519   5536     u8 *aData;
................................................................................
  5541   5558   
  5542   5559       aCell = pageGetCell(aData, nData, i);
  5543   5560       eType = *aCell++;
  5544   5561       assert( (flags & SEGMENT_BTREE_FLAG) || eType!=0 );
  5545   5562       aCell += lsmVarintGet32(aCell, &iPgPtr);
  5546   5563   
  5547   5564       if( eType==0 ){
  5548         -      Pgno iRef;                  /* Page number of referenced page */
         5565  +      LsmPgno iRef;               /* Page number of referenced page */
  5549   5566         aCell += lsmVarintGet64(aCell, &iRef);
  5550   5567         lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef);
  5551   5568         aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob);
  5552   5569       }else{
  5553   5570         aCell += lsmVarintGet32(aCell, &nKey);
  5554   5571         if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal);
  5555   5572         sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob);
................................................................................
  5585   5602     int bIndirect,                  /* True to follow indirect refs */
  5586   5603     Page *pPg,
  5587   5604     int iCell,
  5588   5605     int *peType,
  5589   5606     int *piPgPtr,
  5590   5607     u8 **paKey, int *pnKey,
  5591   5608     u8 **paVal, int *pnVal,
  5592         -  Blob *pBlob
         5609  +  LsmBlob *pBlob
  5593   5610   ){
  5594   5611     u8 *aData; int nData;           /* Page data */
  5595   5612     u8 *aKey; int nKey = 0;         /* Key */
  5596   5613     u8 *aVal = 0; int nVal = 0;     /* Value */
  5597   5614     int eType;
  5598   5615     int iPgPtr;
  5599   5616     Page *pRef = 0;                 /* Pointer to page iRef */
................................................................................
  5603   5620   
  5604   5621     aCell = pageGetCell(aData, nData, iCell);
  5605   5622     eType = *aCell++;
  5606   5623     aCell += lsmVarintGet32(aCell, &iPgPtr);
  5607   5624   
  5608   5625     if( eType==0 ){
  5609   5626       int dummy;
  5610         -    Pgno iRef;                  /* Page number of referenced page */
         5627  +    LsmPgno iRef;                 /* Page number of referenced page */
  5611   5628       aCell += lsmVarintGet64(aCell, &iRef);
  5612   5629       if( bIndirect ){
  5613   5630         lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef);
  5614   5631         pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob);
  5615   5632         aKey = (u8 *)pBlob->pData;
  5616   5633         nKey = pBlob->nData;
  5617   5634         lsmFsPageRelease(pRef);
................................................................................
  5649   5666   #define INFO_PAGE_DUMP_DATA     0x01
  5650   5667   #define INFO_PAGE_DUMP_VALUES   0x02
  5651   5668   #define INFO_PAGE_DUMP_HEX      0x04
  5652   5669   #define INFO_PAGE_DUMP_INDIRECT 0x08
  5653   5670   
  5654   5671   static int infoPageDump(
  5655   5672     lsm_db *pDb,                    /* Database handle */
  5656         -  Pgno iPg,                       /* Page number of page to dump */
         5673  +  LsmPgno iPg,                    /* Page number of page to dump */
  5657   5674     int flags,
  5658   5675     char **pzOut                    /* OUT: lsmMalloc'd string */
  5659   5676   ){
  5660   5677     int rc = LSM_OK;                /* Return code */
  5661   5678     Page *pPg = 0;                  /* Handle for page iPg */
  5662   5679     int i, j;                       /* Loop counters */
  5663   5680     const int perLine = 16;         /* Bytes per line in the raw hex dump */
................................................................................
  5690   5707     ** to pass a NULL in place of the segment pointer as the second argument
  5691   5708     ** to lsmFsDbPageGet() here.  */
  5692   5709     if( rc==LSM_OK ){
  5693   5710       rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg);
  5694   5711     }
  5695   5712   
  5696   5713     if( rc==LSM_OK ){
  5697         -    Blob blob = {0, 0, 0, 0};
         5714  +    LsmBlob blob = {0, 0, 0, 0};
  5698   5715       int nKeyWidth = 0;
  5699   5716       LsmString str;
  5700   5717       int nRec;
  5701   5718       int iPtr;
  5702   5719       int flags2;
  5703   5720       int iCell;
  5704   5721       u8 *aData; int nData;         /* Page data and size thereof */
................................................................................
  5725   5742       if( bHex ) nKeyWidth = nKeyWidth * 2;
  5726   5743   
  5727   5744       for(iCell=0; iCell<nRec; iCell++){
  5728   5745         u8 *aKey; int nKey = 0;       /* Key */
  5729   5746         u8 *aVal; int nVal = 0;       /* Value */
  5730   5747         int iPgPtr;
  5731   5748         int eType;
  5732         -      Pgno iAbsPtr;
         5749  +      LsmPgno iAbsPtr;
  5733   5750         char zFlags[8];
  5734   5751   
  5735   5752         infoCellDump(pDb, pSeg, bIndirect, pPg, iCell, &eType, &iPgPtr,
  5736   5753             &aKey, &nKey, &aVal, &nVal, &blob
  5737   5754         );
  5738   5755         iAbsPtr = iPgPtr + ((flags2 & SEGMENT_BTREE_FLAG) ? 0 : iPtr);
  5739   5756   
................................................................................
  5791   5808     }
  5792   5809   
  5793   5810     return rc;
  5794   5811   }
  5795   5812   
  5796   5813   int lsmInfoPageDump(
  5797   5814     lsm_db *pDb,                    /* Database handle */
  5798         -  Pgno iPg,                       /* Page number of page to dump */
         5815  +  LsmPgno iPg,                    /* Page number of page to dump */
  5799   5816     int bHex,                       /* True to output key/value in hex form */
  5800   5817     char **pzOut                    /* OUT: lsmMalloc'd string */
  5801   5818   ){
  5802   5819     int flags = INFO_PAGE_DUMP_DATA | INFO_PAGE_DUMP_VALUES;
  5803   5820     if( bHex ) flags |= INFO_PAGE_DUMP_HEX;
  5804   5821     return infoPageDump(pDb, iPg, flags, pzOut);
  5805   5822   }
................................................................................
  5967   5984     iHdr = SEGMENT_EOF(nOrig, nEntry);
  5968   5985     memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr);
  5969   5986   }
  5970   5987   
  5971   5988   #ifdef LSM_DEBUG_EXPENSIVE
  5972   5989   static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){
  5973   5990     Page *pPg = 0;
  5974         -  Blob blob1 = {0, 0, 0, 0};
  5975         -  Blob blob2 = {0, 0, 0, 0};
         5991  +  LsmBlob blob1 = {0, 0, 0, 0};
         5992  +  LsmBlob blob2 = {0, 0, 0, 0};
  5976   5993   
  5977   5994     lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg);
  5978   5995     while( pPg ){
  5979   5996       u8 *aData; int nData;
  5980   5997       Page *pNext;
  5981   5998   
  5982   5999       aData = lsmFsPageData(pPg, &nData);
................................................................................
  6030   6047     Segment *pOne,                  /* Segment containing pointers */
  6031   6048     Segment *pTwo,                  /* Segment containing pointer targets */
  6032   6049     int bRhs                        /* True if pTwo may have been Gobble()d */
  6033   6050   ){
  6034   6051     int rc = LSM_OK;                /* Error code */
  6035   6052     SegmentPtr ptr1;                /* Iterates through pOne */
  6036   6053     SegmentPtr ptr2;                /* Iterates through pTwo */
  6037         -  Pgno iPrev;
         6054  +  LsmPgno iPrev;
  6038   6055   
  6039   6056     assert( pOne && pTwo );
  6040   6057   
  6041   6058     memset(&ptr1, 0, sizeof(ptr1));
  6042   6059     memset(&ptr2, 0, sizeof(ptr1));
  6043   6060     ptr1.pSeg = pOne;
  6044   6061     ptr2.pSeg = pTwo;
................................................................................
  6053   6070     }
  6054   6071   
  6055   6072     if( rc==LSM_OK && ptr1.nCell>0 ){
  6056   6073       rc = segmentPtrLoadCell(&ptr1, 0);
  6057   6074     }
  6058   6075         
  6059   6076     while( rc==LSM_OK && ptr2.pPg ){
  6060         -    Pgno iThis;
         6077  +    LsmPgno iThis;
  6061   6078   
  6062   6079       /* Advance to the next page of segment pTwo that contains at least
  6063   6080       ** one cell. Break out of the loop if the iterator reaches EOF.  */
  6064   6081       do{
  6065   6082         rc = segmentPtrNextPage(&ptr2, 1);
  6066   6083         assert( rc==LSM_OK );
  6067   6084       }while( rc==LSM_OK && ptr2.pPg && ptr2.nCell==0 );
................................................................................
  6115   6132   */
  6116   6133   static int assertBtreeOk(
  6117   6134     lsm_db *pDb,
  6118   6135     Segment *pSeg
  6119   6136   ){
  6120   6137     int rc = LSM_OK;                /* Return code */
  6121   6138     if( pSeg->iRoot ){
  6122         -    Blob blob = {0, 0, 0};        /* Buffer used to cache overflow keys */
         6139  +    LsmBlob blob = {0, 0, 0};     /* Buffer used to cache overflow keys */
  6123   6140       FileSystem *pFS = pDb->pFS;   /* File system to read from */
  6124   6141       Page *pPg = 0;                /* Main run page */
  6125   6142       BtreeCursor *pCsr = 0;        /* Btree cursor */
  6126   6143   
  6127   6144       rc = btreeCursorNew(pDb, pSeg, &pCsr);
  6128   6145       if( rc==LSM_OK ){
  6129   6146         rc = btreeCursorFirst(pCsr);

Added ext/lsm1/tool/mklsm1c.tcl.

            1  +#!/bin/sh
            2  +# restart with tclsh \
            3  +exec tclsh "$0" "$@"
            4  +
            5  +set srcdir [file dirname [file dirname [info script]]]
            6  +set G(src) [string map [list %dir% $srcdir] {
            7  +  %dir%/lsm.h
            8  +  %dir%/lsmInt.h
            9  +  %dir%/lsm_vtab.c
           10  +  %dir%/lsm_ckpt.c
           11  +  %dir%/lsm_file.c
           12  +  %dir%/lsm_log.c
           13  +  %dir%/lsm_main.c
           14  +  %dir%/lsm_mem.c
           15  +  %dir%/lsm_mutex.c
           16  +  %dir%/lsm_shared.c
           17  +  %dir%/lsm_sorted.c
           18  +  %dir%/lsm_str.c
           19  +  %dir%/lsm_tree.c
           20  +  %dir%/lsm_unix.c
           21  +  %dir%/lsm_varint.c
           22  +  %dir%/lsm_win32.c
           23  +}]
           24  +
           25  +set G(hdr) {
           26  +
           27  +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) 
           28  +
           29  +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
           30  +# define NDEBUG 1
           31  +#endif
           32  +#if defined(NDEBUG) && defined(SQLITE_DEBUG)
           33  +# undef NDEBUG
           34  +#endif
           35  +
           36  +}
           37  +
           38  +set G(footer) {
           39  +    
           40  +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */
           41  +}
           42  +
           43  +#-------------------------------------------------------------------------
           44  +# Read and return the entire contents of text file $zFile from disk.
           45  +#
           46  +proc readfile {zFile} {
           47  +  set fd [open $zFile]
           48  +  set data [read $fd]
           49  +  close $fd
           50  +  return $data
           51  +}
           52  +
           53  +proc lsm1c_init {zOut} {
           54  +  global G
           55  +  set G(fd) stdout
           56  +  set G(fd) [open $zOut w]
           57  +
           58  +  puts -nonewline $G(fd) $G(hdr)
           59  +}
           60  +
           61  +proc lsm1c_printfile {zIn} {
           62  +  global G
           63  +  set data [readfile $zIn]
           64  +  set zTail [file tail $zIn]
           65  +  puts $G(fd) "#line 1 \"$zTail\""
           66  +
           67  +  foreach line [split $data "\n"] {
           68  +    if {[regexp {^# *include.*lsm} $line]} {
           69  +      set line "/* $line */"
           70  +    } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } {
           71  +      set line "static $line"
           72  +    }
           73  +    puts $G(fd) $line
           74  +  }
           75  +}
           76  +
           77  +proc lsm1c_close {} {
           78  +  global G
           79  +  puts -nonewline $G(fd) $G(footer)
           80  +  if {$G(fd)!="stdout"} {
           81  +    close $G(fd)
           82  +  }
           83  +}
           84  +
           85  +
           86  +lsm1c_init lsm1.c
           87  +foreach f $G(src) { lsm1c_printfile $f }
           88  +lsm1c_close

Changes to ext/misc/README.md.

    10     10   as follows:
    11     11   
    12     12     *  **carray.c** &mdash;  This module implements the
    13     13        [carray](https://www.sqlite.org/carray.html) table-valued function.
    14     14        It is a good example of how to go about implementing a custom
    15     15        [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
    16     16   
           17  +  *  **csv.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
           18  +     for reading 
           19  +     [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values).
           20  +
    17     21     *  **dbdump.c** &mdash;  This is not actually a loadable extension, but
    18     22        rather a library that implements an approximate equivalent to the
    19     23        ".dump" command of the
    20     24        [command-line shell](https://www.sqlite.org/cli.html).
           25  +
           26  +  *  **json1.c** &mdash;  Various SQL functions and table-valued functions
           27  +     for processing JSON.  This extension is already built into the
           28  +     [SQLite amalgamation](https://sqlite.org/amalgamation.html).  See
           29  +     <https://sqlite.org/json1.html> for additional information.
    21     30   
    22     31     *  **memvfs.c** &mdash;  This file implements a custom
    23     32        [VFS](https://www.sqlite.org/vfs.html) that stores an entire database
    24     33        file in a single block of RAM.  It serves as a good example of how
    25     34        to implement a simple custom VFS.
    26     35   
    27     36     *  **rot13.c** &mdash;  This file implements the very simple rot13()
................................................................................
    34     43   
    35     44     *  **shathree.c** &mdash;  An implementation of the sha3() and
    36     45        sha3_query() SQL functions.  The file is named "shathree.c" instead
    37     46        of "sha3.c" because the default entry point names in SQLite are based
    38     47        on the source filename with digits removed, so if we used the name
    39     48        "sha3.c" then the entry point would conflict with the prior "sha1.c"
    40     49        extension.
           50  +
           51  +  *  **unionvtab.c** &mdash; Implementation of the unionvtab and
           52  +     [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables.
           53  +     These virtual tables allow a single
           54  +     large table to be spread out across multiple database files.  In the
           55  +     case of swarmvtab, the individual database files can be attached on
           56  +     demand.
           57  +
           58  +  *  **zipfile.c** &mdash;  A [virtual table](https://sqlite.org/vtab.html)
           59  +     that can read and write a 
           60  +     [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29).

Added ext/misc/appendvfs.c.

            1  +/*
            2  +** 2017-10-20
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file implements a VFS shim that allows an SQLite database to be
           14  +** appended onto the end of some other file, such as an executable.
           15  +**
           16  +** A special record must appear at the end of the file that identifies the
           17  +** file as an appended database and provides an offset to page 1.  For
           18  +** best performance page 1 should be located at a disk page boundary, though
           19  +** that is not required.
           20  +**
           21  +** When opening a database using this VFS, the connection might treat
           22  +** the file as an ordinary SQLite database, or it might treat is as a
           23  +** database appended onto some other file.  Here are the rules:
           24  +**
           25  +**  (1)  When opening a new empty file, that file is treated as an ordinary
           26  +**       database.
           27  +**
           28  +**  (2)  When opening a file that begins with the standard SQLite prefix
           29  +**       string "SQLite format 3", that file is treated as an ordinary
           30  +**       database.
           31  +**
           32  +**  (3)  When opening a file that ends with the appendvfs trailer string
           33  +**       "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
           34  +**       database.
           35  +**
           36  +**  (4)  If none of the above apply and the SQLITE_OPEN_CREATE flag is
           37  +**       set, then a new database is appended to the already existing file.
           38  +**
           39  +**  (5)  Otherwise, SQLITE_CANTOPEN is returned.
           40  +**
           41  +** To avoid unnecessary complications with the PENDING_BYTE, the size of
           42  +** the file containing the database is limited to 1GB.  This VFS will refuse
           43  +** to read or write past the 1GB mark.  This restriction might be lifted in
           44  +** future versions.  For now, if you need a large database, then keep the
           45  +** database in a separate file.
           46  +**
           47  +** If the file being opened is not an appended database, then this shim is
           48  +** a pass-through into the default underlying VFS.
           49  +**/
           50  +#include "sqlite3ext.h"
           51  +SQLITE_EXTENSION_INIT1
           52  +#include <string.h>
           53  +#include <assert.h>
           54  +
           55  +/* The append mark at the end of the database is:
           56  +**
           57  +**     Start-Of-SQLite3-NNNNNNNN
           58  +**     123456789 123456789 12345
           59  +**
           60  +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
           61  +** the offset to page 1.
           62  +*/
           63  +#define APND_MARK_PREFIX     "Start-Of-SQLite3-"
           64  +#define APND_MARK_PREFIX_SZ  17
           65  +#define APND_MARK_SIZE       25
           66  +
           67  +/*
           68  +** Maximum size of the combined prefix + database + append-mark.  This
           69  +** must be less than 0x40000000 to avoid locking issues on Windows.
           70  +*/
           71  +#define APND_MAX_SIZE  (65536*15259)
           72  +
           73  +/*
           74  +** Forward declaration of objects used by this utility
           75  +*/
           76  +typedef struct sqlite3_vfs ApndVfs;
           77  +typedef struct ApndFile ApndFile;
           78  +
           79  +/* Access to a lower-level VFS that (might) implement dynamic loading,
           80  +** access to randomness, etc.
           81  +*/
           82  +#define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
           83  +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
           84  +
           85  +/* An open file */
           86  +struct ApndFile {
           87  +  sqlite3_file base;              /* IO methods */
           88  +  sqlite3_int64 iPgOne;           /* File offset to page 1 */
           89  +  sqlite3_int64 iMark;            /* Start of the append-mark */
           90  +};
           91  +
           92  +/*
           93  +** Methods for ApndFile
           94  +*/
           95  +static int apndClose(sqlite3_file*);
           96  +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
           97  +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
           98  +static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
           99  +static int apndSync(sqlite3_file*, int flags);
          100  +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
          101  +static int apndLock(sqlite3_file*, int);
          102  +static int apndUnlock(sqlite3_file*, int);
          103  +static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
          104  +static int apndFileControl(sqlite3_file*, int op, void *pArg);
          105  +static int apndSectorSize(sqlite3_file*);
          106  +static int apndDeviceCharacteristics(sqlite3_file*);
          107  +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
          108  +static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
          109  +static void apndShmBarrier(sqlite3_file*);
          110  +static int apndShmUnmap(sqlite3_file*, int deleteFlag);
          111  +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
          112  +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
          113  +
          114  +/*
          115  +** Methods for ApndVfs
          116  +*/
          117  +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
          118  +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
          119  +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
          120  +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
          121  +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
          122  +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
          123  +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
          124  +static void apndDlClose(sqlite3_vfs*, void*);
          125  +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
          126  +static int apndSleep(sqlite3_vfs*, int microseconds);
          127  +static int apndCurrentTime(sqlite3_vfs*, double*);
          128  +static int apndGetLastError(sqlite3_vfs*, int, char *);
          129  +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
          130  +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
          131  +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
          132  +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
          133  +
          134  +static sqlite3_vfs apnd_vfs = {
          135  +  3,                            /* iVersion (set when registered) */
          136  +  0,                            /* szOsFile (set when registered) */
          137  +  1024,                         /* mxPathname */
          138  +  0,                            /* pNext */
          139  +  "apndvfs",                    /* zName */
          140  +  0,                            /* pAppData (set when registered) */ 
          141  +  apndOpen,                     /* xOpen */
          142  +  apndDelete,                   /* xDelete */
          143  +  apndAccess,                   /* xAccess */
          144  +  apndFullPathname,             /* xFullPathname */
          145  +  apndDlOpen,                   /* xDlOpen */
          146  +  apndDlError,                  /* xDlError */
          147  +  apndDlSym,                    /* xDlSym */
          148  +  apndDlClose,                  /* xDlClose */
          149  +  apndRandomness,               /* xRandomness */
          150  +  apndSleep,                    /* xSleep */
          151  +  apndCurrentTime,              /* xCurrentTime */
          152  +  apndGetLastError,             /* xGetLastError */
          153  +  apndCurrentTimeInt64,         /* xCurrentTimeInt64 */
          154  +  apndSetSystemCall,            /* xSetSystemCall */
          155  +  apndGetSystemCall,            /* xGetSystemCall */
          156  +  apndNextSystemCall            /* xNextSystemCall */
          157  +};
          158  +
          159  +static const sqlite3_io_methods apnd_io_methods = {
          160  +  3,                              /* iVersion */
          161  +  apndClose,                      /* xClose */
          162  +  apndRead,                       /* xRead */
          163  +  apndWrite,                      /* xWrite */
          164  +  apndTruncate,                   /* xTruncate */
          165  +  apndSync,                       /* xSync */
          166  +  apndFileSize,                   /* xFileSize */
          167  +  apndLock,                       /* xLock */
          168  +  apndUnlock,                     /* xUnlock */
          169  +  apndCheckReservedLock,          /* xCheckReservedLock */
          170  +  apndFileControl,                /* xFileControl */
          171  +  apndSectorSize,                 /* xSectorSize */
          172  +  apndDeviceCharacteristics,      /* xDeviceCharacteristics */
          173  +  apndShmMap,                     /* xShmMap */
          174  +  apndShmLock,                    /* xShmLock */
          175  +  apndShmBarrier,                 /* xShmBarrier */
          176  +  apndShmUnmap,                   /* xShmUnmap */
          177  +  apndFetch,                      /* xFetch */
          178  +  apndUnfetch                     /* xUnfetch */
          179  +};
          180  +
          181  +
          182  +
          183  +/*
          184  +** Close an apnd-file.
          185  +*/
          186  +static int apndClose(sqlite3_file *pFile){
          187  +  pFile = ORIGFILE(pFile);
          188  +  return pFile->pMethods->xClose(pFile);
          189  +}
          190  +
          191  +/*
          192  +** Read data from an apnd-file.
          193  +*/
          194  +static int apndRead(
          195  +  sqlite3_file *pFile, 
          196  +  void *zBuf, 
          197  +  int iAmt, 
          198  +  sqlite_int64 iOfst
          199  +){
          200  +  ApndFile *p = (ApndFile *)pFile;
          201  +  pFile = ORIGFILE(pFile);
          202  +  return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
          203  +}
          204  +
          205  +/*
          206  +** Add the append-mark onto the end of the file.
          207  +*/
          208  +static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
          209  +  int i;
          210  +  unsigned char a[APND_MARK_SIZE];
          211  +  memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
          212  +  for(i=0; i<8; i++){
          213  +    a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
          214  +  }
          215  +  return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
          216  +}
          217  +
          218  +/*
          219  +** Write data to an apnd-file.
          220  +*/
          221  +static int apndWrite(
          222  +  sqlite3_file *pFile,
          223  +  const void *zBuf,
          224  +  int iAmt,
          225  +  sqlite_int64 iOfst
          226  +){
          227  +  int rc;
          228  +  ApndFile *p = (ApndFile *)pFile;
          229  +  pFile = ORIGFILE(pFile);
          230  +  if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
          231  +  rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
          232  +  if( rc==SQLITE_OK &&  iOfst + iAmt + p->iPgOne > p->iMark ){
          233  +    sqlite3_int64 sz = 0;
          234  +    rc = pFile->pMethods->xFileSize(pFile, &sz);
          235  +    if( rc==SQLITE_OK ){
          236  +      p->iMark = sz - APND_MARK_SIZE;
          237  +      if( iOfst + iAmt + p->iPgOne > p->iMark ){
          238  +        p->iMark = p->iPgOne + iOfst + iAmt;
          239  +        rc = apndWriteMark(p, pFile);
          240  +      }
          241  +    }
          242  +  }
          243  +  return rc;
          244  +}
          245  +
          246  +/*
          247  +** Truncate an apnd-file.
          248  +*/
          249  +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
          250  +  int rc;
          251  +  ApndFile *p = (ApndFile *)pFile;
          252  +  pFile = ORIGFILE(pFile);
          253  +  rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
          254  +  if( rc==SQLITE_OK ){
          255  +    p->iMark = p->iPgOne+size;
          256  +    rc = apndWriteMark(p, pFile);
          257  +  }
          258  +  return rc;
          259  +}
          260  +
          261  +/*
          262  +** Sync an apnd-file.
          263  +*/
          264  +static int apndSync(sqlite3_file *pFile, int flags){
          265  +  pFile = ORIGFILE(pFile);
          266  +  return pFile->pMethods->xSync(pFile, flags);
          267  +}
          268  +
          269  +/*
          270  +** Return the current file-size of an apnd-file.
          271  +*/
          272  +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
          273  +  ApndFile *p = (ApndFile *)pFile;
          274  +  int rc;
          275  +  pFile = ORIGFILE(p);
          276  +  rc = pFile->pMethods->xFileSize(pFile, pSize);
          277  +  if( rc==SQLITE_OK && p->iPgOne ){
          278  +    *pSize -= p->iPgOne + APND_MARK_SIZE;
          279  +  }
          280  +  return rc;
          281  +}
          282  +
          283  +/*
          284  +** Lock an apnd-file.
          285  +*/
          286  +static int apndLock(sqlite3_file *pFile, int eLock){
          287  +  pFile = ORIGFILE(pFile);
          288  +  return pFile->pMethods->xLock(pFile, eLock);
          289  +}
          290  +
          291  +/*
          292  +** Unlock an apnd-file.
          293  +*/
          294  +static int apndUnlock(sqlite3_file *pFile, int eLock){
          295  +  pFile = ORIGFILE(pFile);
          296  +  return pFile->pMethods->xUnlock(pFile, eLock);
          297  +}
          298  +
          299  +/*
          300  +** Check if another file-handle holds a RESERVED lock on an apnd-file.
          301  +*/
          302  +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
          303  +  pFile = ORIGFILE(pFile);
          304  +  return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
          305  +}
          306  +
          307  +/*
          308  +** File control method. For custom operations on an apnd-file.
          309  +*/
          310  +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
          311  +  ApndFile *p = (ApndFile *)pFile;
          312  +  int rc;
          313  +  pFile = ORIGFILE(pFile);
          314  +  rc = pFile->pMethods->xFileControl(pFile, op, pArg);
          315  +  if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
          316  +    *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
          317  +  }
          318  +  return rc;
          319  +}
          320  +
          321  +/*
          322  +** Return the sector-size in bytes for an apnd-file.
          323  +*/
          324  +static int apndSectorSize(sqlite3_file *pFile){
          325  +  pFile = ORIGFILE(pFile);
          326  +  return pFile->pMethods->xSectorSize(pFile);
          327  +}
          328  +
          329  +/*
          330  +** Return the device characteristic flags supported by an apnd-file.
          331  +*/
          332  +static int apndDeviceCharacteristics(sqlite3_file *pFile){
          333  +  pFile = ORIGFILE(pFile);
          334  +  return pFile->pMethods->xDeviceCharacteristics(pFile);
          335  +}
          336  +
          337  +/* Create a shared memory file mapping */
          338  +static int apndShmMap(
          339  +  sqlite3_file *pFile,
          340  +  int iPg,
          341  +  int pgsz,
          342  +  int bExtend,
          343  +  void volatile **pp
          344  +){
          345  +  pFile = ORIGFILE(pFile);
          346  +  return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
          347  +}
          348  +
          349  +/* Perform locking on a shared-memory segment */
          350  +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
          351  +  pFile = ORIGFILE(pFile);
          352  +  return pFile->pMethods->xShmLock(pFile,offset,n,flags);
          353  +}
          354  +
          355  +/* Memory barrier operation on shared memory */
          356  +static void apndShmBarrier(sqlite3_file *pFile){
          357  +  pFile = ORIGFILE(pFile);
          358  +  pFile->pMethods->xShmBarrier(pFile);
          359  +}
          360  +
          361  +/* Unmap a shared memory segment */
          362  +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
          363  +  pFile = ORIGFILE(pFile);
          364  +  return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
          365  +}
          366  +
          367  +/* Fetch a page of a memory-mapped file */
          368  +static int apndFetch(
          369  +  sqlite3_file *pFile,
          370  +  sqlite3_int64 iOfst,
          371  +  int iAmt,
          372  +  void **pp
          373  +){
          374  +  ApndFile *p = (ApndFile *)pFile;
          375  +  pFile = ORIGFILE(pFile);
          376  +  return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
          377  +}
          378  +
          379  +/* Release a memory-mapped page */
          380  +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
          381  +  ApndFile *p = (ApndFile *)pFile;
          382  +  pFile = ORIGFILE(pFile);
          383  +  return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
          384  +}
          385  +
          386  +/*
          387  +** Check to see if the file is an ordinary SQLite database file.
          388  +*/
          389  +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
          390  +  int rc;
          391  +  char zHdr[16];
          392  +  static const char aSqliteHdr[] = "SQLite format 3";
          393  +  if( sz<512 ) return 0;
          394  +  rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
          395  +  if( rc ) return 0;
          396  +  return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
          397  +}
          398  +
          399  +/*
          400  +** Try to read the append-mark off the end of a file.  Return the
          401  +** start of the appended database if the append-mark is present.  If
          402  +** there is no append-mark, return -1;
          403  +*/
          404  +static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
          405  +  int rc, i;
          406  +  sqlite3_int64 iMark;
          407  +  unsigned char a[APND_MARK_SIZE];
          408  +
          409  +  if( sz<=APND_MARK_SIZE ) return -1;
          410  +  rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
          411  +  if( rc ) return -1;
          412  +  if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
          413  +  iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
          414  +  for(i=1; i<8; i++){    
          415  +    iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
          416  +  }
          417  +  return iMark;
          418  +}
          419  +
          420  +/*
          421  +** Open an apnd file handle.
          422  +*/
          423  +static int apndOpen(
          424  +  sqlite3_vfs *pVfs,
          425  +  const char *zName,
          426  +  sqlite3_file *pFile,
          427  +  int flags,
          428  +  int *pOutFlags
          429  +){
          430  +  ApndFile *p;
          431  +  sqlite3_file *pSubFile;
          432  +  sqlite3_vfs *pSubVfs;
          433  +  int rc;
          434  +  sqlite3_int64 sz;
          435  +  pSubVfs = ORIGVFS(pVfs);
          436  +  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
          437  +    return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
          438  +  }
          439  +  p = (ApndFile*)pFile;
          440  +  memset(p, 0, sizeof(*p));
          441  +  pSubFile = ORIGFILE(pFile);
          442  +  p->base.pMethods = &apnd_io_methods;
          443  +  rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
          444  +  if( rc ) goto apnd_open_done;
          445  +  rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
          446  +  if( rc ){
          447  +    pSubFile->pMethods->xClose(pSubFile);
          448  +    goto apnd_open_done;
          449  +  }
          450  +  if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
          451  +    memmove(pFile, pSubFile, pSubVfs->szOsFile);
          452  +    return SQLITE_OK;
          453  +  }
          454  +  p->iMark = 0;
          455  +  p->iPgOne = apndReadMark(sz, pFile);
          456  +  if( p->iPgOne>0 ){
          457  +    return SQLITE_OK;
          458  +  }
          459  +  if( (flags & SQLITE_OPEN_CREATE)==0 ){
          460  +    pSubFile->pMethods->xClose(pSubFile);
          461  +    rc = SQLITE_CANTOPEN;
          462  +  }
          463  +  p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
          464  +apnd_open_done:
          465  +  if( rc ) pFile->pMethods = 0;
          466  +  return rc;
          467  +}
          468  +
          469  +/*
          470  +** All other VFS methods are pass-thrus.
          471  +*/
          472  +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
          473  +  return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
          474  +}
          475  +static int apndAccess(
          476  +  sqlite3_vfs *pVfs, 
          477  +  const char *zPath, 
          478  +  int flags, 
          479  +  int *pResOut
          480  +){
          481  +  return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
          482  +}
          483  +static int apndFullPathname(
          484  +  sqlite3_vfs *pVfs, 
          485  +  const char *zPath, 
          486  +  int nOut, 
          487  +  char *zOut
          488  +){
          489  +  return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
          490  +}
          491  +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
          492  +  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
          493  +}
          494  +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
          495  +  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
          496  +}
          497  +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
          498  +  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
          499  +}
          500  +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
          501  +  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
          502  +}
          503  +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
          504  +  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
          505  +}
          506  +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
          507  +  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
          508  +}
          509  +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
          510  +  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
          511  +}
          512  +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
          513  +  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
          514  +}
          515  +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
          516  +  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
          517  +}
          518  +static int apndSetSystemCall(
          519  +  sqlite3_vfs *pVfs,
          520  +  const char *zName,
          521  +  sqlite3_syscall_ptr pCall
          522  +){
          523  +  return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
          524  +}
          525  +static sqlite3_syscall_ptr apndGetSystemCall(
          526  +  sqlite3_vfs *pVfs,
          527  +  const char *zName
          528  +){
          529  +  return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
          530  +}
          531  +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
          532  +  return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
          533  +}
          534  +
          535  +  
          536  +#ifdef _WIN32
          537  +__declspec(dllexport)
          538  +#endif
          539  +/* 
          540  +** This routine is called when the extension is loaded.
          541  +** Register the new VFS.
          542  +*/
          543  +int sqlite3_appendvfs_init(
          544  +  sqlite3 *db, 
          545  +  char **pzErrMsg, 
          546  +  const sqlite3_api_routines *pApi
          547  +){
          548  +  int rc = SQLITE_OK;
          549  +  sqlite3_vfs *pOrig;
          550  +  SQLITE_EXTENSION_INIT2(pApi);
          551  +  (void)pzErrMsg;
          552  +  (void)db;
          553  +  pOrig = sqlite3_vfs_find(0);
          554  +  apnd_vfs.iVersion = pOrig->iVersion;
          555  +  apnd_vfs.pAppData = pOrig;
          556  +  apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
          557  +  rc = sqlite3_vfs_register(&apnd_vfs, 0);
          558  +#ifdef APPENDVFS_TEST
          559  +  if( rc==SQLITE_OK ){
          560  +    rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
          561  +  }
          562  +#endif
          563  +  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
          564  +  return rc;
          565  +}

Changes to ext/misc/btreeinfo.c.

   335    335   ){
   336    336     BinfoCursor *pCsr = (BinfoCursor *)pCursor;
   337    337     if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){
   338    338       int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1);
   339    339       sqlite3 *db = sqlite3_context_db_handle(ctx);
   340    340       int rc = binfoCompute(db, pgno, pCsr);
   341    341       if( rc ){
   342         -      return rc;
          342  +      pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
          343  +      return SQLITE_ERROR;
   343    344       }
   344    345     }
   345    346     switch( i ){
   346    347       case BINFO_COLUMN_NAME:
   347    348       case BINFO_COLUMN_TYPE:
   348    349       case BINFO_COLUMN_TBL_NAME:
   349    350       case BINFO_COLUMN_ROOTPAGE:

Changes to ext/misc/compress.c.

    23     23   **
    24     24   ** The output is a BLOB that begins with a variable-length integer that
    25     25   ** is the input size in bytes (the size of X before compression).  The
    26     26   ** variable-length integer is implemented as 1 to 5 bytes.  There are
    27     27   ** seven bits per integer stored in the lower seven bits of each byte.
    28     28   ** More significant bits occur first.  The most significant bit (0x80)
    29     29   ** is a flag to indicate the end of the integer.
           30  +**
           31  +** This function, SQLAR, and ZIP all use the same "deflate" compression
           32  +** algorithm, but each is subtly different:
           33  +**
           34  +**   *  ZIP uses raw deflate.
           35  +**
           36  +**   *  SQLAR uses the "zlib format" which is raw deflate with a two-byte
           37  +**      algorithm-identification header and a four-byte checksum at the end.
           38  +**
           39  +**   *  This utility uses the "zlib format" like SQLAR, but adds the variable-
           40  +**      length integer uncompressed size value at the beginning.
           41  +**
           42  +** This function might be extended in the future to support compression
           43  +** formats other than deflate, by providing a different algorithm-id
           44  +** mark following the variable-length integer size parameter.
    30     45   */
    31     46   static void compressFunc(
    32     47     sqlite3_context *context,
    33     48     int argc,
    34     49     sqlite3_value **argv
    35     50   ){
    36     51     const unsigned char *pIn;

Changes to ext/misc/fileio.c.

     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   ** This SQLite extension implements SQL functions readfile() and
    14         -** writefile().
           14  +** writefile(), and eponymous virtual type "fsdir".
           15  +**
           16  +** WRITEFILE(FILE, DATA [, MODE [, MTIME]]):
           17  +**
           18  +**   If neither of the optional arguments is present, then this UDF
           19  +**   function writes blob DATA to file FILE. If successful, the number
           20  +**   of bytes written is returned. If an error occurs, NULL is returned.
           21  +**
           22  +**   If the first option argument - MODE - is present, then it must
           23  +**   be passed an integer value that corresponds to a POSIX mode
           24  +**   value (file type + permissions, as returned in the stat.st_mode
           25  +**   field by the stat() system call). Three types of files may
           26  +**   be written/created:
           27  +**
           28  +**     regular files:  (mode & 0170000)==0100000
           29  +**     symbolic links: (mode & 0170000)==0120000
           30  +**     directories:    (mode & 0170000)==0040000
           31  +**
           32  +**   For a directory, the DATA is ignored. For a symbolic link, it is
           33  +**   interpreted as text and used as the target of the link. For a
           34  +**   regular file, it is interpreted as a blob and written into the
           35  +**   named file. Regardless of the type of file, its permissions are
           36  +**   set to (mode & 0777) before returning.
           37  +**
           38  +**   If the optional MTIME argument is present, then it is interpreted
           39  +**   as an integer - the number of seconds since the unix epoch. The
           40  +**   modification-time of the target file is set to this value before
           41  +**   returning.
           42  +**
           43  +**   If three or more arguments are passed to this function and an
           44  +**   error is encountered, an exception is raised.
           45  +**
           46  +** READFILE(FILE):
           47  +**
           48  +**   Read and return the contents of file FILE (type blob) from disk.
           49  +**
           50  +** FSDIR:
           51  +**
           52  +**   Used as follows:
           53  +**
           54  +**     SELECT * FROM fsdir($path [, $dir]);
           55  +**
           56  +**   Parameter $path is an absolute or relative pathname. If the file that it
           57  +**   refers to does not exist, it is an error. If the path refers to a regular
           58  +**   file or symbolic link, it returns a single row. Or, if the path refers
           59  +**   to a directory, it returns one row for the directory, and one row for each
           60  +**   file within the hierarchy rooted at $path.
           61  +**
           62  +**   Each row has the following columns:
           63  +**
           64  +**     name:  Path to file or directory (text value).
           65  +**     mode:  Value of stat.st_mode for directory entry (an integer).
           66  +**     mtime: Value of stat.st_mtime for directory entry (an integer).
           67  +**     data:  For a regular file, a blob containing the file data. For a
           68  +**            symlink, a text value containing the text of the link. For a
           69  +**            directory, NULL.
           70  +**
           71  +**   If a non-NULL value is specified for the optional $dir parameter and
           72  +**   $path is a relative path, then $path is interpreted relative to $dir. 
           73  +**   And the paths returned in the "name" column of the table are also 
           74  +**   relative to directory $dir.
    15     75   */
    16     76   #include "sqlite3ext.h"
    17     77   SQLITE_EXTENSION_INIT1
    18     78   #include <stdio.h>
           79  +#include <string.h>
           80  +#include <assert.h>
           81  +
           82  +#include <sys/types.h>
           83  +#include <sys/stat.h>
           84  +#include <fcntl.h>
           85  +#if !defined(_WIN32) && !defined(WIN32)
           86  +#  include <unistd.h>
           87  +#  include <dirent.h>
           88  +#  include <utime.h>
           89  +#  include <sys/time.h>
           90  +#else
           91  +#  include "windows.h"
           92  +#  include <io.h>
           93  +#  include <direct.h>
           94  +#  include "test_windirent.h"
           95  +#  define dirent DIRENT
           96  +#  ifndef stat
           97  +#    define stat _stat
           98  +#  endif
           99  +#  define mkdir(path,mode) _mkdir(path)
          100  +#  define lstat(path,buf) stat(path,buf)
          101  +#endif
          102  +#include <time.h>
          103  +#include <errno.h>
          104  +
          105  +
          106  +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
          107  +
          108  +/*
          109  +** Set the result stored by context ctx to a blob containing the 
          110  +** contents of file zName.
          111  +*/
          112  +static void readFileContents(sqlite3_context *ctx, const char *zName){
          113  +  FILE *in;
          114  +  long nIn;
          115  +  void *pBuf;
          116  +
          117  +  in = fopen(zName, "rb");
          118  +  if( in==0 ) return;
          119  +  fseek(in, 0, SEEK_END);
          120  +  nIn = ftell(in);
          121  +  rewind(in);
          122  +  pBuf = sqlite3_malloc( nIn );
          123  +  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
          124  +    sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free);
          125  +  }else{
          126  +    sqlite3_free(pBuf);
          127  +  }
          128  +  fclose(in);
          129  +}
    19    130   
    20    131   /*
    21    132   ** Implementation of the "readfile(X)" SQL function.  The entire content
    22    133   ** of the file named X is read and returned as a BLOB.  NULL is returned
    23    134   ** if the file does not exist or is unreadable.
    24    135   */
    25    136   static void readfileFunc(
    26    137     sqlite3_context *context,
    27    138     int argc,
    28    139     sqlite3_value **argv
    29    140   ){
    30    141     const char *zName;
    31         -  FILE *in;
    32         -  long nIn;
    33         -  void *pBuf;
    34         -
    35    142     (void)(argc);  /* Unused parameter */
    36    143     zName = (const char*)sqlite3_value_text(argv[0]);
    37    144     if( zName==0 ) return;
    38         -  in = fopen(zName, "rb");
    39         -  if( in==0 ) return;
    40         -  fseek(in, 0, SEEK_END);
    41         -  nIn = ftell(in);
    42         -  rewind(in);
    43         -  pBuf = sqlite3_malloc( nIn );
    44         -  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
    45         -    sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);
          145  +  readFileContents(context, zName);
          146  +}
          147  +
          148  +/*
          149  +** Set the error message contained in context ctx to the results of
          150  +** vprintf(zFmt, ...).
          151  +*/
          152  +static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
          153  +  char *zMsg = 0;
          154  +  va_list ap;
          155  +  va_start(ap, zFmt);
          156  +  zMsg = sqlite3_vmprintf(zFmt, ap);
          157  +  sqlite3_result_error(ctx, zMsg, -1);
          158  +  sqlite3_free(zMsg);
          159  +  va_end(ap);
          160  +}
          161  +
          162  +/*
          163  +** Argument zFile is the name of a file that will be created and/or written
          164  +** by SQL function writefile(). This function ensures that the directory
          165  +** zFile will be written to exists, creating it if required. The permissions
          166  +** for any path components created by this function are set to (mode&0777).
          167  +**
          168  +** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise,
          169  +** SQLITE_OK is returned if the directory is successfully created, or
          170  +** SQLITE_ERROR otherwise.
          171  +*/
          172  +static int makeDirectory(
          173  +  const char *zFile,
          174  +  mode_t mode
          175  +){
          176  +  char *zCopy = sqlite3_mprintf("%s", zFile);
          177  +  int rc = SQLITE_OK;
          178  +
          179  +  if( zCopy==0 ){
          180  +    rc = SQLITE_NOMEM;
    46    181     }else{
    47         -    sqlite3_free(pBuf);
    48         -  }
    49         -  fclose(in);
    50         -}
    51         -
    52         -/*
    53         -** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
    54         -** is written into file X.  The number of bytes written is returned.  Or
    55         -** NULL is returned if something goes wrong, such as being unable to open
    56         -** file X for writing.
          182  +    int nCopy = (int)strlen(zCopy);
          183  +    int i = 1;
          184  +
          185  +    while( rc==SQLITE_OK ){
          186  +      struct stat sStat;
          187  +      int rc2;
          188  +
          189  +      for(; zCopy[i]!='/' && i<nCopy; i++);
          190  +      if( i==nCopy ) break;
          191  +      zCopy[i] = '\0';
          192  +
          193  +      rc2 = stat(zCopy, &sStat);
          194  +      if( rc2!=0 ){
          195  +        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
          196  +      }else{
          197  +        if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
          198  +      }
          199  +      zCopy[i] = '/';
          200  +      i++;
          201  +    }
          202  +
          203  +    sqlite3_free(zCopy);
          204  +  }
          205  +
          206  +  return rc;
          207  +}
          208  +
          209  +/*
          210  +** This function does the work for the writefile() UDF. Refer to 
          211  +** header comments at the top of this file for details.
          212  +*/
          213  +static int writeFile(
          214  +  sqlite3_context *pCtx,          /* Context to return bytes written in */
          215  +  const char *zFile,              /* File to write */
          216  +  sqlite3_value *pData,           /* Data to write */
          217  +  mode_t mode,                    /* MODE parameter passed to writefile() */
          218  +  sqlite3_int64 mtime             /* MTIME parameter (or -1 to not set time) */
          219  +){
          220  +#if !defined(_WIN32) && !defined(WIN32)
          221  +  if( S_ISLNK(mode) ){
          222  +    const char *zTo = (const char*)sqlite3_value_text(pData);
          223  +    if( symlink(zTo, zFile)<0 ) return 1;
          224  +  }else
          225  +#endif
          226  +  {
          227  +    if( S_ISDIR(mode) ){
          228  +      if( mkdir(zFile, mode) ){
          229  +        /* The mkdir() call to create the directory failed. This might not
          230  +        ** be an error though - if there is already a directory at the same
          231  +        ** path and either the permissions already match or can be changed
          232  +        ** to do so using chmod(), it is not an error.  */
          233  +        struct stat sStat;
          234  +        if( errno!=EEXIST
          235  +         || 0!=stat(zFile, &sStat)
          236  +         || !S_ISDIR(sStat.st_mode)
          237  +         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
          238  +        ){
          239  +          return 1;
          240  +        }
          241  +      }
          242  +    }else{
          243  +      sqlite3_int64 nWrite = 0;
          244  +      const char *z;
          245  +      int rc = 0;
          246  +      FILE *out = fopen(zFile, "wb");
          247  +      if( out==0 ) return 1;
          248  +      z = (const char*)sqlite3_value_blob(pData);
          249  +      if( z ){
          250  +        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(pData), out);
          251  +        nWrite = sqlite3_value_bytes(pData);
          252  +        if( nWrite!=n ){
          253  +          rc = 1;
          254  +        }
          255  +      }
          256  +      fclose(out);
          257  +      if( rc==0 && mode && chmod(zFile, mode & 0777) ){
          258  +        rc = 1;
          259  +      }
          260  +      if( rc ) return 2;
          261  +      sqlite3_result_int64(pCtx, nWrite);
          262  +    }
          263  +  }
          264  +
          265  +  if( mtime>=0 ){
          266  +#if defined(_WIN32)
          267  +    /* Windows */
          268  +    FILETIME lastAccess;
          269  +    FILETIME lastWrite;
          270  +    SYSTEMTIME currentTime;
          271  +    LONGLONG intervals;
          272  +    HANDLE hFile;
          273  +    LPWSTR zUnicodeName;
          274  +    extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
          275  +
          276  +    GetSystemTime(&currentTime);
          277  +    SystemTimeToFileTime(&currentTime, &lastAccess);
          278  +    intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
          279  +    lastWrite.dwLowDateTime = (DWORD)intervals;
          280  +    lastWrite.dwHighDateTime = intervals >> 32;
          281  +    zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
          282  +    hFile = CreateFileW(
          283  +      zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
          284  +      FILE_FLAG_BACKUP_SEMANTICS, NULL
          285  +    );
          286  +    sqlite3_free(zUnicodeName);
          287  +    if( hFile!=INVALID_HANDLE_VALUE ){
          288  +      BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
          289  +      CloseHandle(hFile);
          290  +      return !bResult;
          291  +    }else{
          292  +      return 1;
          293  +    }
          294  +#elif defined(AT_FDCWD) && 0 /* utimensat() is not univerally available */
          295  +    /* Recent unix */
          296  +    struct timespec times[2];
          297  +    times[0].tv_nsec = times[1].tv_nsec = 0;
          298  +    times[0].tv_sec = time(0);
          299  +    times[1].tv_sec = mtime;
          300  +    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
          301  +      return 1;
          302  +    }
          303  +#else
          304  +    /* Legacy unix */
          305  +    struct timeval times[2];
          306  +    times[0].tv_usec = times[1].tv_usec = 0;
          307  +    times[0].tv_sec = time(0);
          308  +    times[1].tv_sec = mtime;
          309  +    if( utimes(zFile, times) ){
          310  +      return 1;
          311  +    }
          312  +#endif
          313  +  }
          314  +
          315  +  return 0;
          316  +}
          317  +
          318  +/*
          319  +** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function.  
          320  +** Refer to header comments at the top of this file for details.
    57    321   */
    58    322   static void writefileFunc(
    59    323     sqlite3_context *context,
    60    324     int argc,
    61    325     sqlite3_value **argv
    62    326   ){
    63         -  FILE *out;
    64         -  const char *z;
    65         -  sqlite3_int64 rc;
    66    327     const char *zFile;
          328  +  mode_t mode = 0;
          329  +  int res;
          330  +  sqlite3_int64 mtime = -1;
    67    331   
    68         -  (void)(argc);  /* Unused parameter */
          332  +  if( argc<2 || argc>4 ){
          333  +    sqlite3_result_error(context, 
          334  +        "wrong number of arguments to function writefile()", -1
          335  +    );
          336  +    return;
          337  +  }
          338  +
    69    339     zFile = (const char*)sqlite3_value_text(argv[0]);
    70    340     if( zFile==0 ) return;
    71         -  out = fopen(zFile, "wb");
    72         -  if( out==0 ) return;
    73         -  z = (const char*)sqlite3_value_blob(argv[1]);
    74         -  if( z==0 ){
    75         -    rc = 0;
          341  +  if( argc>=3 ){
          342  +    mode = (mode_t)sqlite3_value_int(argv[2]);
          343  +  }
          344  +  if( argc==4 ){
          345  +    mtime = sqlite3_value_int64(argv[3]);
          346  +  }
          347  +
          348  +  res = writeFile(context, zFile, argv[1], mode, mtime);
          349  +  if( res==1 && errno==ENOENT ){
          350  +    if( makeDirectory(zFile, mode)==SQLITE_OK ){
          351  +      res = writeFile(context, zFile, argv[1], mode, mtime);
          352  +    }
          353  +  }
          354  +
          355  +  if( argc>2 && res!=0 ){
          356  +    if( S_ISLNK(mode) ){
          357  +      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
          358  +    }else if( S_ISDIR(mode) ){
          359  +      ctxErrorMsg(context, "failed to create directory: %s", zFile);
          360  +    }else{
          361  +      ctxErrorMsg(context, "failed to write file: %s", zFile);
          362  +    }
          363  +  }
          364  +}
          365  +
          366  +/*
          367  +** SQL function:   lsmode(MODE)
          368  +**
          369  +** Given a numberic st_mode from stat(), convert it into a human-readable
          370  +** text string in the style of "ls -l".
          371  +*/
          372  +static void lsModeFunc(
          373  +  sqlite3_context *context,
          374  +  int argc,
          375  +  sqlite3_value **argv
          376  +){
          377  +  int i;
          378  +  int iMode = sqlite3_value_int(argv[0]);
          379  +  char z[16];
          380  +  (void)argc;
          381  +  if( S_ISLNK(iMode) ){
          382  +    z[0] = 'l';
          383  +  }else if( S_ISREG(iMode) ){
          384  +    z[0] = '-';
          385  +  }else if( S_ISDIR(iMode) ){
          386  +    z[0] = 'd';
          387  +  }else{
          388  +    z[0] = '?';
          389  +  }
          390  +  for(i=0; i<3; i++){
          391  +    int m = (iMode >> ((2-i)*3));
          392  +    char *a = &z[1 + i*3];
          393  +    a[0] = (m & 0x4) ? 'r' : '-';
          394  +    a[1] = (m & 0x2) ? 'w' : '-';
          395  +    a[2] = (m & 0x1) ? 'x' : '-';
          396  +  }
          397  +  z[10] = '\0';
          398  +  sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT);
          399  +}
          400  +
          401  +#ifndef SQLITE_OMIT_VIRTUALTABLE
          402  +
          403  +/* 
          404  +** Cursor type for recursively iterating through a directory structure.
          405  +*/
          406  +typedef struct fsdir_cursor fsdir_cursor;
          407  +typedef struct FsdirLevel FsdirLevel;
          408  +
          409  +struct FsdirLevel {
          410  +  DIR *pDir;                 /* From opendir() */
          411  +  char *zDir;                /* Name of directory (nul-terminated) */
          412  +};
          413  +
          414  +struct fsdir_cursor {
          415  +  sqlite3_vtab_cursor base;  /* Base class - must be first */
          416  +
          417  +  int nLvl;                  /* Number of entries in aLvl[] array */
          418  +  int iLvl;                  /* Index of current entry */
          419  +  FsdirLevel *aLvl;          /* Hierarchy of directories being traversed */
          420  +
          421  +  const char *zBase;
          422  +  int nBase;
          423  +
          424  +  struct stat sStat;         /* Current lstat() results */
          425  +  char *zPath;               /* Path to current entry */
          426  +  sqlite3_int64 iRowid;      /* Current rowid */
          427  +};
          428  +
          429  +typedef struct fsdir_tab fsdir_tab;
          430  +struct fsdir_tab {
          431  +  sqlite3_vtab base;         /* Base class - must be first */
          432  +};
          433  +
          434  +/*
          435  +** Construct a new fsdir virtual table object.
          436  +*/
          437  +static int fsdirConnect(
          438  +  sqlite3 *db,
          439  +  void *pAux,
          440  +  int argc, const char *const*argv,
          441  +  sqlite3_vtab **ppVtab,
          442  +  char **pzErr
          443  +){
          444  +  fsdir_tab *pNew = 0;
          445  +  int rc;
          446  +  (void)pAux;
          447  +  (void)argc;
          448  +  (void)argv;
          449  +  (void)pzErr;
          450  +  rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
          451  +  if( rc==SQLITE_OK ){
          452  +    pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
          453  +    if( pNew==0 ) return SQLITE_NOMEM;
          454  +    memset(pNew, 0, sizeof(*pNew));
          455  +  }
          456  +  *ppVtab = (sqlite3_vtab*)pNew;
          457  +  return rc;
          458  +}
          459  +
          460  +/*
          461  +** This method is the destructor for fsdir vtab objects.
          462  +*/
          463  +static int fsdirDisconnect(sqlite3_vtab *pVtab){
          464  +  sqlite3_free(pVtab);
          465  +  return SQLITE_OK;
          466  +}
          467  +
          468  +/*
          469  +** Constructor for a new fsdir_cursor object.
          470  +*/
          471  +static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
          472  +  fsdir_cursor *pCur;
          473  +  (void)p;
          474  +  pCur = sqlite3_malloc( sizeof(*pCur) );
          475  +  if( pCur==0 ) return SQLITE_NOMEM;
          476  +  memset(pCur, 0, sizeof(*pCur));
          477  +  pCur->iLvl = -1;
          478  +  *ppCursor = &pCur->base;
          479  +  return SQLITE_OK;
          480  +}
          481  +
          482  +/*
          483  +** Reset a cursor back to the state it was in when first returned
          484  +** by fsdirOpen().
          485  +*/
          486  +static void fsdirResetCursor(fsdir_cursor *pCur){
          487  +  int i;
          488  +  for(i=0; i<=pCur->iLvl; i++){
          489  +    FsdirLevel *pLvl = &pCur->aLvl[i];
          490  +    if( pLvl->pDir ) closedir(pLvl->pDir);
          491  +    sqlite3_free(pLvl->zDir);
          492  +  }
          493  +  sqlite3_free(pCur->zPath);
          494  +  sqlite3_free(pCur->aLvl);
          495  +  pCur->aLvl = 0;
          496  +  pCur->zPath = 0;
          497  +  pCur->zBase = 0;
          498  +  pCur->nBase = 0;
          499  +  pCur->iLvl = -1;
          500  +  pCur->iRowid = 1;
          501  +}
          502  +
          503  +/*
          504  +** Destructor for an fsdir_cursor.
          505  +*/
          506  +static int fsdirClose(sqlite3_vtab_cursor *cur){
          507  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          508  +
          509  +  fsdirResetCursor(pCur);
          510  +  sqlite3_free(pCur);
          511  +  return SQLITE_OK;
          512  +}
          513  +
          514  +/*
          515  +** Set the error message for the virtual table associated with cursor
          516  +** pCur to the results of vprintf(zFmt, ...).
          517  +*/
          518  +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
          519  +  va_list ap;
          520  +  va_start(ap, zFmt);
          521  +  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
          522  +  va_end(ap);
          523  +}
          524  +
          525  +
          526  +/*
          527  +** Advance an fsdir_cursor to its next row of output.
          528  +*/
          529  +static int fsdirNext(sqlite3_vtab_cursor *cur){
          530  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          531  +  mode_t m = pCur->sStat.st_mode;
          532  +
          533  +  pCur->iRowid++;
          534  +  if( S_ISDIR(m) ){
          535  +    /* Descend into this directory */
          536  +    int iNew = pCur->iLvl + 1;
          537  +    FsdirLevel *pLvl;
          538  +    if( iNew>=pCur->nLvl ){
          539  +      int nNew = iNew+1;
          540  +      int nByte = nNew*sizeof(FsdirLevel);
          541  +      FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
          542  +      if( aNew==0 ) return SQLITE_NOMEM;
          543  +      memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
          544  +      pCur->aLvl = aNew;
          545  +      pCur->nLvl = nNew;
          546  +    }
          547  +    pCur->iLvl = iNew;
          548  +    pLvl = &pCur->aLvl[iNew];
          549  +    
          550  +    pLvl->zDir = pCur->zPath;
          551  +    pCur->zPath = 0;
          552  +    pLvl->pDir = opendir(pLvl->zDir);
          553  +    if( pLvl->pDir==0 ){
          554  +      fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
          555  +      return SQLITE_ERROR;
          556  +    }
          557  +  }
          558  +
          559  +  while( pCur->iLvl>=0 ){
          560  +    FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
          561  +    struct dirent *pEntry = readdir(pLvl->pDir);
          562  +    if( pEntry ){
          563  +      if( pEntry->d_name[0]=='.' ){
          564  +       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
          565  +       if( pEntry->d_name[1]=='\0' ) continue;
          566  +      }
          567  +      sqlite3_free(pCur->zPath);
          568  +      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
          569  +      if( pCur->zPath==0 ) return SQLITE_NOMEM;
          570  +      if( lstat(pCur->zPath, &pCur->sStat) ){
          571  +        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
          572  +        return SQLITE_ERROR;
          573  +      }
          574  +      return SQLITE_OK;
          575  +    }
          576  +    closedir(pLvl->pDir);
          577  +    sqlite3_free(pLvl->zDir);
          578  +    pLvl->pDir = 0;
          579  +    pLvl->zDir = 0;
          580  +    pCur->iLvl--;
          581  +  }
          582  +
          583  +  /* EOF */
          584  +  sqlite3_free(pCur->zPath);
          585  +  pCur->zPath = 0;
          586  +  return SQLITE_OK;
          587  +}
          588  +
          589  +/*
          590  +** Return values of columns for the row at which the series_cursor
          591  +** is currently pointing.
          592  +*/
          593  +static int fsdirColumn(
          594  +  sqlite3_vtab_cursor *cur,   /* The cursor */
          595  +  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
          596  +  int i                       /* Which column to return */
          597  +){
          598  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          599  +  switch( i ){
          600  +    case 0: { /* name */
          601  +      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
          602  +      break;
          603  +    }
          604  +
          605  +    case 1: /* mode */
          606  +      sqlite3_result_int64(ctx, pCur->sStat.st_mode);
          607  +      break;
          608  +
          609  +    case 2: /* mtime */
          610  +      sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
          611  +      break;
          612  +
          613  +    case 3: { /* data */
          614  +      mode_t m = pCur->sStat.st_mode;
          615  +      if( S_ISDIR(m) ){
          616  +        sqlite3_result_null(ctx);
          617  +#if !defined(_WIN32) && !defined(WIN32)
          618  +      }else if( S_ISLNK(m) ){
          619  +        char aStatic[64];
          620  +        char *aBuf = aStatic;
          621  +        int nBuf = 64;
          622  +        int n;
          623  +
          624  +        while( 1 ){
          625  +          n = readlink(pCur->zPath, aBuf, nBuf);
          626  +          if( n<nBuf ) break;
          627  +          if( aBuf!=aStatic ) sqlite3_free(aBuf);
          628  +          nBuf = nBuf*2;
          629  +          aBuf = sqlite3_malloc(nBuf);
          630  +          if( aBuf==0 ){
          631  +            sqlite3_result_error_nomem(ctx);
          632  +            return SQLITE_NOMEM;
          633  +          }
          634  +        }
          635  +
          636  +        sqlite3_result_text(ctx, aBuf, n, SQLITE_TRANSIENT);
          637  +        if( aBuf!=aStatic ) sqlite3_free(aBuf);
          638  +#endif
          639  +      }else{
          640  +        readFileContents(ctx, pCur->zPath);
          641  +      }
          642  +    }
          643  +  }
          644  +  return SQLITE_OK;
          645  +}
          646  +
          647  +/*
          648  +** Return the rowid for the current row. In this implementation, the
          649  +** first row returned is assigned rowid value 1, and each subsequent
          650  +** row a value 1 more than that of the previous.
          651  +*/
          652  +static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          653  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          654  +  *pRowid = pCur->iRowid;
          655  +  return SQLITE_OK;
          656  +}
          657  +
          658  +/*
          659  +** Return TRUE if the cursor has been moved off of the last
          660  +** row of output.
          661  +*/
          662  +static int fsdirEof(sqlite3_vtab_cursor *cur){
          663  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          664  +  return (pCur->zPath==0);
          665  +}
          666  +
          667  +/*
          668  +** xFilter callback.
          669  +*/
          670  +static int fsdirFilter(
          671  +  sqlite3_vtab_cursor *cur, 
          672  +  int idxNum, const char *idxStr,
          673  +  int argc, sqlite3_value **argv
          674  +){
          675  +  const char *zDir = 0;
          676  +  fsdir_cursor *pCur = (fsdir_cursor*)cur;
          677  +  (void)idxStr;
          678  +  fsdirResetCursor(pCur);
          679  +
          680  +  if( idxNum==0 ){
          681  +    fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
          682  +    return SQLITE_ERROR;
          683  +  }
          684  +
          685  +  assert( argc==idxNum && (argc==1 || argc==2) );
          686  +  zDir = (const char*)sqlite3_value_text(argv[0]);
          687  +  if( zDir==0 ){
          688  +    fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
          689  +    return SQLITE_ERROR;
          690  +  }
          691  +  if( argc==2 ){
          692  +    pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
          693  +  }
          694  +  if( pCur->zBase ){
          695  +    pCur->nBase = (int)strlen(pCur->zBase)+1;
          696  +    pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
          697  +  }else{
          698  +    pCur->zPath = sqlite3_mprintf("%s", zDir);
          699  +  }
          700  +
          701  +  if( pCur->zPath==0 ){
          702  +    return SQLITE_NOMEM;
          703  +  }
          704  +  if( lstat(pCur->zPath, &pCur->sStat) ){
          705  +    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
          706  +    return SQLITE_ERROR;
          707  +  }
          708  +
          709  +  return SQLITE_OK;
          710  +}
          711  +
          712  +/*
          713  +** SQLite will invoke this method one or more times while planning a query
          714  +** that uses the generate_series virtual table.  This routine needs to create
          715  +** a query plan for each invocation and compute an estimated cost for that
          716  +** plan.
          717  +**
          718  +** In this implementation idxNum is used to represent the
          719  +** query plan.  idxStr is unused.
          720  +**
          721  +** The query plan is represented by bits in idxNum:
          722  +**
          723  +**  (1)  start = $value  -- constraint exists
          724  +**  (2)  stop = $value   -- constraint exists
          725  +**  (4)  step = $value   -- constraint exists
          726  +**  (8)  output in descending order
          727  +*/
          728  +static int fsdirBestIndex(
          729  +  sqlite3_vtab *tab,
          730  +  sqlite3_index_info *pIdxInfo
          731  +){
          732  +  int i;                 /* Loop over constraints */
          733  +  int idx4 = -1;
          734  +  int idx5 = -1;
          735  +  const struct sqlite3_index_constraint *pConstraint;
          736  +
          737  +  (void)tab;
          738  +  pConstraint = pIdxInfo->aConstraint;
          739  +  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
          740  +    if( pConstraint->usable==0 ) continue;
          741  +    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
          742  +    if( pConstraint->iColumn==4 ) idx4 = i;
          743  +    if( pConstraint->iColumn==5 ) idx5 = i;
          744  +  }
          745  +
          746  +  if( idx4<0 ){
          747  +    pIdxInfo->idxNum = 0;
          748  +    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
    76    749     }else{
    77         -    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
          750  +    pIdxInfo->aConstraintUsage[idx4].omit = 1;
          751  +    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
          752  +    if( idx5>=0 ){
          753  +      pIdxInfo->aConstraintUsage[idx5].omit = 1;
          754  +      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
          755  +      pIdxInfo->idxNum = 2;
          756  +      pIdxInfo->estimatedCost = 10.0;
          757  +    }else{
          758  +      pIdxInfo->idxNum = 1;
          759  +      pIdxInfo->estimatedCost = 100.0;
          760  +    }
    78    761     }
    79         -  fclose(out);
    80         -  sqlite3_result_int64(context, rc);
          762  +
          763  +  return SQLITE_OK;
    81    764   }
    82    765   
          766  +/*
          767  +** Register the "fsdir" virtual table.
          768  +*/
          769  +static int fsdirRegister(sqlite3 *db){
          770  +  static sqlite3_module fsdirModule = {
          771  +    0,                         /* iVersion */
          772  +    0,                         /* xCreate */
          773  +    fsdirConnect,              /* xConnect */
          774  +    fsdirBestIndex,            /* xBestIndex */
          775  +    fsdirDisconnect,           /* xDisconnect */
          776  +    0,                         /* xDestroy */
          777  +    fsdirOpen,                 /* xOpen - open a cursor */
          778  +    fsdirClose,                /* xClose - close a cursor */
          779  +    fsdirFilter,               /* xFilter - configure scan constraints */
          780  +    fsdirNext,                 /* xNext - advance a cursor */
          781  +    fsdirEof,                  /* xEof - check for end of scan */
          782  +    fsdirColumn,               /* xColumn - read data */
          783  +    fsdirRowid,                /* xRowid - read data */
          784  +    0,                         /* xUpdate */
          785  +    0,                         /* xBegin */
          786  +    0,                         /* xSync */
          787  +    0,                         /* xCommit */
          788  +    0,                         /* xRollback */
          789  +    0,                         /* xFindMethod */
          790  +    0,                         /* xRename */
          791  +    0,                         /* xSavepoint */
          792  +    0,                         /* xRelease */
          793  +    0                          /* xRollbackTo */
          794  +  };
          795  +
................................................................................
          796  +  int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
          797  +  return rc;
          798  +}
          799  +#else         /* SQLITE_OMIT_VIRTUALTABLE */
          800  +# define fsdirRegister(x) SQLITE_OK
          801  +#endif
    83    802   
    84    803   #ifdef _WIN32
    85    804   __declspec(dllexport)
    86    805   #endif
    87    806   int sqlite3_fileio_init(
    88    807     sqlite3 *db, 
    89    808     char **pzErrMsg, 
................................................................................
    91    810   ){
    92    811     int rc = SQLITE_OK;
    93    812     SQLITE_EXTENSION_INIT2(pApi);
    94    813     (void)pzErrMsg;  /* Unused parameter */
    95    814     rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
    96    815                                  readfileFunc, 0, 0);
    97    816     if( rc==SQLITE_OK ){
    98         -    rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
          817  +    rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0,
    99    818                                    writefileFunc, 0, 0);
   100    819     }
          820  +  if( rc==SQLITE_OK ){
          821  +    rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0,
          822  +                                 lsModeFunc, 0, 0);
          823  +  }
          824  +  if( rc==SQLITE_OK ){
          825  +    rc = fsdirRegister(db);
          826  +  }
   101    827     return rc;
   102    828   }

Changes to ext/misc/memvfs.c.

     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13         -** This is an in-memory read-only VFS implementation.  The application
    14         -** supplies a block of memory which is the database file, and this VFS
    15         -** uses that block of memory.
           13  +** This is an in-memory VFS implementation.  The application supplies
           14  +** a chunk of memory to hold the database file.
    16     15   **
    17         -** Because there is no place to store journals and no good way to lock
    18         -** the "file", this VFS is read-only.
           16  +** Because there is place to store a rollback or wal journal, the database
           17  +** must use one of journal_mode=MEMORY or journal_mode=NONE.
    19     18   **
    20     19   ** USAGE:
    21     20   **
    22         -**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
    23         -**                    SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
           21  +**    sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db,
           22  +**                    SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI,
    24     23   **                    "memvfs");
    25     24   **
    26         -** The ptr= and sz= query parameters are required or the open will fail.
    27         -** The ptr= parameter gives the memory address of the buffer holding the
    28         -** read-only database and sz= gives the size of the database.  The parameter
    29         -** values may be in hexadecimal or decimal.  The filename is ignored.
           25  +** These are the query parameters:
           26  +**
           27  +**    ptr=          The address of the memory buffer that holds the database.
           28  +**
           29  +**    sz=           The current size the database file
           30  +**
           31  +**    maxsz=        The maximum size of the database.  In other words, the
           32  +**                  amount of space allocated for the ptr= buffer.
           33  +**
           34  +**    freeonclose=  If true, then sqlite3_free() is called on the ptr=
           35  +**                  value when the connection closes.
           36  +**
           37  +** The ptr= and sz= query parameters are required.  If maxsz= is omitted,
           38  +** then it defaults to the sz= value.  Parameter values can be in either
           39  +** decimal or hexadecimal.  The filename in the URI is ignored.
    30     40   */
    31     41   #include <sqlite3ext.h>
    32     42   SQLITE_EXTENSION_INIT1
    33     43   #include <string.h>
    34     44   #include <assert.h>
    35     45   
    36     46   
................................................................................
    45     55   */
    46     56   #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
    47     57   
    48     58   /* An open file */
    49     59   struct MemFile {
    50     60     sqlite3_file base;              /* IO methods */
    51     61     sqlite3_int64 sz;               /* Size of the file */
           62  +  sqlite3_int64 szMax;            /* Space allocated to aData */
    52     63     unsigned char *aData;           /* content of the file */
           64  +  int bFreeOnClose;               /* Invoke sqlite3_free() on aData at close */
    53     65   };
    54     66   
    55     67   /*
    56     68   ** Methods for MemFile
    57     69   */
    58     70   static int memClose(sqlite3_file*);
    59     71   static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
................................................................................
   140    152   /*
   141    153   ** Close an mem-file.
   142    154   **
   143    155   ** The pData pointer is owned by the application, so there is nothing
   144    156   ** to free.
   145    157   */
   146    158   static int memClose(sqlite3_file *pFile){
          159  +  MemFile *p = (MemFile *)pFile;
          160  +  if( p->bFreeOnClose ) sqlite3_free(p->aData);
   147    161     return SQLITE_OK;
   148    162   }
   149    163   
   150    164   /*
   151    165   ** Read data from an mem-file.
   152    166   */
   153    167   static int memRead(
................................................................................
   166    180   */
   167    181   static int memWrite(
   168    182     sqlite3_file *pFile,
   169    183     const void *z,
   170    184     int iAmt,
   171    185     sqlite_int64 iOfst
   172    186   ){
   173         -  return SQLITE_READONLY;
          187  +  MemFile *p = (MemFile *)pFile;
          188  +  if( iOfst+iAmt>p->sz ){
          189  +    if( iOfst+iAmt>p->szMax ) return SQLITE_FULL;
          190  +    if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
          191  +    p->sz = iOfst+iAmt;
          192  +  }
          193  +  memcpy(p->aData+iOfst, z, iAmt);
          194  +  return SQLITE_OK;
   174    195   }
   175    196   
   176    197   /*
   177    198   ** Truncate an mem-file.
   178    199   */
   179    200   static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
   180         -  return SQLITE_READONLY;
          201  +  MemFile *p = (MemFile *)pFile;
          202  +  if( size>p->sz ){
          203  +    if( size>p->szMax ) return SQLITE_FULL;
          204  +    memset(p->aData+p->sz, 0, size-p->sz);
          205  +  }
          206  +  p->sz = size; 
          207  +  return SQLITE_OK;
   181    208   }
   182    209   
   183    210   /*
   184    211   ** Sync an mem-file.
   185    212   */
   186    213   static int memSync(sqlite3_file *pFile, int flags){
   187         -  return SQLITE_READONLY;
          214  +  return SQLITE_OK;
   188    215   }
   189    216   
   190    217   /*
   191    218   ** Return the current file-size of an mem-file.
   192    219   */
   193    220   static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
   194    221     MemFile *p = (MemFile *)pFile;
................................................................................
   196    223     return SQLITE_OK;
   197    224   }
   198    225   
   199    226   /*
   200    227   ** Lock an mem-file.
   201    228   */
   202    229   static int memLock(sqlite3_file *pFile, int eLock){
   203         -  return SQLITE_READONLY;
          230  +  return SQLITE_OK;
   204    231   }
   205    232   
   206    233   /*
   207    234   ** Unlock an mem-file.
   208    235   */
   209    236   static int memUnlock(sqlite3_file *pFile, int eLock){
   210    237     return SQLITE_OK;
................................................................................
   238    265     return 1024;
   239    266   }
   240    267   
   241    268   /*
   242    269   ** Return the device characteristic flags supported by an mem-file.
   243    270   */
   244    271   static int memDeviceCharacteristics(sqlite3_file *pFile){
   245         -  return SQLITE_IOCAP_IMMUTABLE;
          272  +  return SQLITE_IOCAP_ATOMIC | 
          273  +         SQLITE_IOCAP_POWERSAFE_OVERWRITE |
          274  +         SQLITE_IOCAP_SAFE_APPEND |
          275  +         SQLITE_IOCAP_SEQUENTIAL;
   246    276   }
   247    277   
   248    278   /* Create a shared memory file mapping */
   249    279   static int memShmMap(
   250    280     sqlite3_file *pFile,
   251    281     int iPg,
   252    282     int pgsz,
   253    283     int bExtend,
   254    284     void volatile **pp
   255    285   ){
   256         -  return SQLITE_READONLY;
          286  +  return SQLITE_IOERR_SHMMAP;
   257    287   }
   258    288   
   259    289   /* Perform locking on a shared-memory segment */
   260    290   static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
   261         -  return SQLITE_READONLY;
          291  +  return SQLITE_IOERR_SHMLOCK;
   262    292   }
   263    293   
   264    294   /* Memory barrier operation on shared memory */
   265    295   static void memShmBarrier(sqlite3_file *pFile){
   266    296     return;
   267    297   }
   268    298   
................................................................................
   301    331     MemFile *p = (MemFile*)pFile;
   302    332     memset(p, 0, sizeof(*p));
   303    333     if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN;
   304    334     p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0);
   305    335     if( p->aData==0 ) return SQLITE_CANTOPEN;
   306    336     p->sz = sqlite3_uri_int64(zName,"sz",0);
   307    337     if( p->sz<0 ) return SQLITE_CANTOPEN;
          338  +  p->szMax = sqlite3_uri_int64(zName,"max",p->sz);
          339  +  if( p->szMax<p->sz ) return SQLITE_CANTOPEN;
          340  +  p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0);
   308    341     pFile->pMethods = &mem_io_methods;
   309    342     return SQLITE_OK;
   310    343   }
   311    344   
   312    345   /*
   313    346   ** Delete the file located at zPath. If the dirSync argument is true,
   314    347   ** ensure the file-system modifications are synced to disk before
   315    348   ** returning.
   316    349   */
   317    350   static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
   318         -  return SQLITE_READONLY;
          351  +  return SQLITE_IOERR_DELETE;
   319    352   }
   320    353   
   321    354   /*
   322    355   ** Test for access permissions. Return true if the requested permission
   323    356   ** is available, or false otherwise.
   324    357   */
   325    358   static int memAccess(
   326    359     sqlite3_vfs *pVfs, 
   327    360     const char *zPath, 
   328    361     int flags, 
   329    362     int *pResOut
   330    363   ){
   331         -  /* The spec says there are three possible values for flags.  But only
   332         -  ** two of them are actually used */
   333         -  assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
   334         -  if( flags==SQLITE_ACCESS_READWRITE ){
   335         -    *pResOut = 0;
   336         -  }else{
   337         -    *pResOut = 1;
   338         -  }
          364  +  *pResOut = 0;
   339    365     return SQLITE_OK;
   340    366   }
   341    367   
   342    368   /*
   343    369   ** Populate buffer zOut with the full canonical pathname corresponding
   344    370   ** to the pathname in zPath. zOut is guaranteed to point to a buffer
   345    371   ** of at least (INST_MAX_PATHNAME+1) bytes.
................................................................................
   412    438   }
   413    439   static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
   414    440     return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
   415    441   }
   416    442   
   417    443   #ifdef MEMVFS_TEST
   418    444   /*
   419         -**       memload(FILENAME)
          445  +**       memvfs_from_file(FILENAME, MAXSIZE)
   420    446   **
   421    447   ** This an SQL function used to help in testing the memvfs VFS.  The
   422    448   ** function reads the content of a file into memory and then returns
   423         -** a string that gives the locate and size of the in-memory buffer.
          449  +** a URI that can be handed to ATTACH to attach the memory buffer as
          450  +** a database.  Example:
          451  +**
          452  +**       ATTACH memvfs_from_file('test.db',1048576) AS inmem;
          453  +**
          454  +** The optional MAXSIZE argument gives the size of the memory allocation
          455  +** used to hold the database.  If omitted, it defaults to the size of the
          456  +** file on disk.
   424    457   */
   425    458   #include <stdio.h>
   426         -static void memvfsMemloadFunc(
          459  +static void memvfsFromFileFunc(
   427    460     sqlite3_context *context,
   428    461     int argc,
   429    462     sqlite3_value **argv
   430    463   ){
   431    464     unsigned char *p;
   432    465     sqlite3_int64 sz;
          466  +  sqlite3_int64 szMax;
   433    467     FILE *in;
   434    468     const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
   435         -  char zReturn[100];
          469  +  char *zUri;
   436    470   
   437    471     if( zFilename==0 ) return;
   438    472     in = fopen(zFilename, "rb");
   439    473     if( in==0 ) return;
   440    474     fseek(in, 0, SEEK_END);
   441         -  sz = ftell(in);
          475  +  szMax = sz = ftell(in);
   442    476     rewind(in);
   443         -  p = sqlite3_malloc( sz );
          477  +  if( argc>=2 ){
          478  +    szMax = sqlite3_value_int64(argv[1]);
          479  +    if( szMax<sz ) szMax = sz;
          480  +  }
          481  +  p = sqlite3_malloc64( szMax );
   444    482     if( p==0 ){
   445    483       fclose(in);
   446    484       sqlite3_result_error_nomem(context);
   447    485       return;
   448    486     }
   449    487     fread(p, sz, 1, in);
   450    488     fclose(in);
   451         -  sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld",
   452         -                   (sqlite3_int64)p, sz);
   453         -  sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT);
          489  +  zUri = sqlite3_mprintf(
          490  +           "file:/mem?vfs=memvfs&ptr=%lld&sz=%lld&max=%lld&freeonclose=1",
          491  +                         (sqlite3_int64)p, sz, szMax);
          492  +  sqlite3_result_text(context, zUri, -1, sqlite3_free);
          493  +}
          494  +#endif /* MEMVFS_TEST */
          495  +
          496  +#ifdef MEMVFS_TEST
          497  +/*
          498  +**       memvfs_to_file(SCHEMA, FILENAME)
          499  +**
          500  +** The schema identified by SCHEMA must be a memvfs database.  Write
          501  +** the content of this database into FILENAME.
          502  +*/
          503  +static void memvfsToFileFunc(
          504  +  sqlite3_context *context,
          505  +  int argc,
          506  +  sqlite3_value **argv
          507  +){
          508  +  MemFile *p = 0;
          509  +  FILE *out;
          510  +  int rc;
          511  +  sqlite3 *db = sqlite3_context_db_handle(context);
          512  +  sqlite3_vfs *pVfs = 0;
          513  +  const char *zSchema = (const char*)sqlite3_value_text(argv[0]);
          514  +  const char *zFilename = (const char*)sqlite3_value_text(argv[1]);
          515  +
          516  +  if( zFilename==0 ) return;
          517  +  out = fopen(zFilename, "wb");
          518  +  if( out==0 ) return;
          519  +  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_VFS_POINTER, &pVfs);
          520  +  if( rc || pVfs==0 ) return;
          521  +  if( strcmp(pVfs->zName,"memvfs")!=0 ) return;
          522  +  rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p);
          523  +  if( rc ) return;
          524  +  fwrite(p->aData, 1, (size_t)p->sz, out);
          525  +  fclose(out);
   454    526   }
          527  +#endif /* MEMVFS_TEST */
          528  +
          529  +#ifdef MEMVFS_TEST
   455    530   /* Called for each new database connection */
   456    531   static int memvfsRegister(
   457    532     sqlite3 *db,
   458         -  const char **pzErrMsg,
          533  +  char **pzErrMsg,
   459    534     const struct sqlite3_api_routines *pThunk
   460    535   ){
   461         -  return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
   462         -                                 memvfsMemloadFunc, 0, 0);
          536  +  sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0,
          537  +                          memvfsFromFileFunc, 0, 0);
          538  +  sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0,
          539  +                          memvfsFromFileFunc, 0, 0);
          540  +  sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0,
          541  +                          memvfsToFileFunc, 0, 0);
          542  +  return SQLITE_OK;
   463    543   }
   464    544   #endif /* MEMVFS_TEST */
   465    545   
   466    546     
   467    547   #ifdef _WIN32
   468    548   __declspec(dllexport)
   469    549   #endif
................................................................................
   481    561     mem_vfs.pAppData = sqlite3_vfs_find(0);
   482    562     mem_vfs.szOsFile = sizeof(MemFile);
   483    563     rc = sqlite3_vfs_register(&mem_vfs, 1);
   484    564   #ifdef MEMVFS_TEST
   485    565     if( rc==SQLITE_OK ){
   486    566       rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
   487    567     }
          568  +  if( rc==SQLITE_OK ){
          569  +    rc = memvfsRegister(db, pzErrMsg, pApi);
          570  +  }
   488    571   #endif
   489    572     if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
   490    573     return rc;
   491    574   }

Added ext/misc/normalize.c.

            1  +/*
            2  +** 2018-01-08
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file contains code to implement the sqlite3_normalize() function.
           14  +**
           15  +**    char *sqlite3_normalize(const char *zSql);
           16  +**
           17  +** This function takes an SQL string as input and returns a "normalized"
           18  +** version of that string in memory obtained from sqlite3_malloc64().  The
           19  +** caller is responsible for ensuring that the returned memory is freed.
           20  +**
           21  +** If a memory allocation error occurs, this routine returns NULL.
           22  +**
           23  +** The normalization consists of the following transformations:
           24  +**
           25  +**   (1)  Convert every literal (string, blob literal, numeric constant,
           26  +**        or "NULL" constant) into a ?
           27  +**
           28  +**   (2)  Remove all superfluous whitespace, including comments.  Change
           29  +**        all required whitespace to a single space character.
           30  +**
           31  +**   (3)  Lowercase all ASCII characters.
           32  +**
           33  +**   (4)  If an IN or NOT IN operator is followed by a list of 1 or more
           34  +**        values, convert that list into "(?,?,?)".
           35  +**
           36  +** The purpose of normalization is two-fold:
           37  +**
           38  +**   (1)  Sanitize queries by removing potentially private or sensitive
           39  +**        information contained in literals.
           40  +**
           41  +**   (2)  Identify structurally identical queries by comparing their
           42  +**        normalized forms.
           43  +**
           44  +** Command-Line Utility
           45  +** --------------------
           46  +**
           47  +** This file also contains code for a command-line utility that converts
           48  +** SQL queries in text files into their normalized forms.  To build the
           49  +** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI
           50  +** and link it against the SQLite library.
           51  +*/
           52  +#include <sqlite3.h>
           53  +#include <string.h>
           54  +
           55  +/*
           56  +** Implementation note:
           57  +**
           58  +** Much of the tokenizer logic is copied out of the tokenize.c source file
           59  +** of SQLite.  That logic could be simplified for this particular application,
           60  +** but that would impose a risk of introducing subtle errors.  It is best to
           61  +** keep the code as close to the original as possible.
           62  +**
           63  +** The tokenize code is in sync with the SQLite core as of 2018-01-08.
           64  +** Any future changes to the core tokenizer might require corresponding
           65  +** adjustments to the tokenizer logic in this module.
           66  +*/
           67  +
           68  +
           69  +/* Character classes for tokenizing
           70  +**
           71  +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
           72  +** using a lookup table, whereas a switch() directly on c uses a binary search.
           73  +** The lookup table is much faster.  To maximize speed, and to ensure that
           74  +** a lookup table is used, all of the classes need to be small integers and
           75  +** all of them need to be used within the switch.
           76  +*/
           77  +#define CC_X          0    /* The letter 'x', or start of BLOB literal */
           78  +#define CC_KYWD       1    /* Alphabetics or '_'.  Usable in a keyword */
           79  +#define CC_ID         2    /* unicode characters usable in IDs */
           80  +#define CC_DIGIT      3    /* Digits */
           81  +#define CC_DOLLAR     4    /* '$' */
           82  +#define CC_VARALPHA   5    /* '@', '#', ':'.  Alphabetic SQL variables */
           83  +#define CC_VARNUM     6    /* '?'.  Numeric SQL variables */
           84  +#define CC_SPACE      7    /* Space characters */
           85  +#define CC_QUOTE      8    /* '"', '\'', or '`'.  String literals, quoted ids */
           86  +#define CC_QUOTE2     9    /* '['.   [...] style quoted ids */
           87  +#define CC_PIPE      10    /* '|'.   Bitwise OR or concatenate */
           88  +#define CC_MINUS     11    /* '-'.  Minus or SQL-style comment */
           89  +#define CC_LT        12    /* '<'.  Part of < or <= or <> */
           90  +#define CC_GT        13    /* '>'.  Part of > or >= */
           91  +#define CC_EQ        14    /* '='.  Part of = or == */
           92  +#define CC_BANG      15    /* '!'.  Part of != */
           93  +#define CC_SLASH     16    /* '/'.  / or c-style comment */
           94  +#define CC_LP        17    /* '(' */
           95  +#define CC_RP        18    /* ')' */
           96  +#define CC_SEMI      19    /* ';' */
           97  +#define CC_PLUS      20    /* '+' */
           98  +#define CC_STAR      21    /* '*' */
           99  +#define CC_PERCENT   22    /* '%' */
          100  +#define CC_COMMA     23    /* ',' */
          101  +#define CC_AND       24    /* '&' */
          102  +#define CC_TILDA     25    /* '~' */
          103  +#define CC_DOT       26    /* '.' */
          104  +#define CC_ILLEGAL   27    /* Illegal character */
          105  +
          106  +static const unsigned char aiClass[] = {
          107  +/*         x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf */
          108  +/* 0x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  7,  7, 27,  7,  7, 27, 27,
          109  +/* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
          110  +/* 2x */    7, 15,  8,  5,  4, 22, 24,  8, 17, 18, 21, 20, 23, 11, 26, 16,
          111  +/* 3x */    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  5, 19, 12, 14, 13,  6,
          112  +/* 4x */    5,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
          113  +/* 5x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1,  9, 27, 27, 27,  1,
          114  +/* 6x */    8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
          115  +/* 7x */    1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 10, 27, 25, 27,
          116  +/* 8x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
          117  +/* 9x */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
          118  +/* Ax */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
          119  +/* Bx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
          120  +/* Cx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
          121  +/* Dx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
          122  +/* Ex */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
          123  +/* Fx */    2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2
          124  +};
          125  +
          126  +/* An array to map all upper-case characters into their corresponding
          127  +** lower-case character. 
          128  +**
          129  +** SQLite only considers US-ASCII (or EBCDIC) characters.  We do not
          130  +** handle case conversions for the UTF character set since the tables
          131  +** involved are nearly as big or bigger than SQLite itself.
          132  +*/
          133  +static const unsigned char sqlite3UpperToLower[] = {
          134  +      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
          135  +     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
          136  +     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
          137  +     54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
          138  +    104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
          139  +    122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
          140  +    108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
          141  +    126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
          142  +    144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
          143  +    162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
          144  +    180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
          145  +    198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
          146  +    216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
          147  +    234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
          148  +    252,253,254,255
          149  +};
          150  +
          151  +/*
          152  +** The following 256 byte lookup table is used to support SQLites built-in
          153  +** equivalents to the following standard library functions:
          154  +**
          155  +**   isspace()                        0x01
          156  +**   isalpha()                        0x02
          157  +**   isdigit()                        0x04
          158  +**   isalnum()                        0x06
          159  +**   isxdigit()                       0x08
          160  +**   toupper()                        0x20
          161  +**   SQLite identifier character      0x40
          162  +**   Quote character                  0x80
          163  +**
          164  +** Bit 0x20 is set if the mapped character requires translation to upper
          165  +** case. i.e. if the character is a lower-case ASCII character.
          166  +** If x is a lower-case ASCII character, then its upper-case equivalent
          167  +** is (x - 0x20). Therefore toupper() can be implemented as:
          168  +**
          169  +**   (x & ~(map[x]&0x20))
          170  +**
          171  +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[]
          172  +** array. tolower() is used more often than toupper() by SQLite.
          173  +**
          174  +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an 
          175  +** SQLite identifier.  Identifiers are alphanumerics, "_", "$", and any
          176  +** non-ASCII UTF character. Hence the test for whether or not a character is
          177  +** part of an identifier is 0x46.
          178  +*/
          179  +static const unsigned char sqlite3CtypeMap[256] = {
          180  +  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 00..07    ........ */
          181  +  0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,  /* 08..0f    ........ */
          182  +  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 10..17    ........ */
          183  +  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 18..1f    ........ */
          184  +  0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80,  /* 20..27     !"#$%&' */
          185  +  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 28..2f    ()*+,-./ */
          186  +  0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c,  /* 30..37    01234567 */
          187  +  0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 38..3f    89:;<=>? */
          188  +
          189  +  0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02,  /* 40..47    @ABCDEFG */
          190  +  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 48..4f    HIJKLMNO */
          191  +  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,  /* 50..57    PQRSTUVW */
          192  +  0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40,  /* 58..5f    XYZ[\]^_ */
          193  +  0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22,  /* 60..67    `abcdefg */
          194  +  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 68..6f    hijklmno */
          195  +  0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,  /* 70..77    pqrstuvw */
          196  +  0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00,  /* 78..7f    xyz{|}~. */
          197  +
          198  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 80..87    ........ */
          199  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 88..8f    ........ */
          200  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 90..97    ........ */
          201  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* 98..9f    ........ */
          202  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a0..a7    ........ */
          203  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* a8..af    ........ */
          204  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b0..b7    ........ */
          205  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* b8..bf    ........ */
          206  +
          207  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c0..c7    ........ */
          208  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* c8..cf    ........ */
          209  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d0..d7    ........ */
          210  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* d8..df    ........ */
          211  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e0..e7    ........ */
          212  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* e8..ef    ........ */
          213  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,  /* f0..f7    ........ */
          214  +  0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40   /* f8..ff    ........ */
          215  +};
          216  +#define sqlite3Toupper(x)   ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20))
          217  +#define sqlite3Isspace(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x01)
          218  +#define sqlite3Isalnum(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x06)
          219  +#define sqlite3Isalpha(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x02)
          220  +#define sqlite3Isdigit(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
          221  +#define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
          222  +#define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
          223  +#define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
          224  +
          225  +
          226  +/*
          227  +** If X is a character that can be used in an identifier then
          228  +** IdChar(X) will be true.  Otherwise it is false.
          229  +**
          230  +** For ASCII, any character with the high-order bit set is
          231  +** allowed in an identifier.  For 7-bit characters, 
          232  +** sqlite3IsIdChar[X] must be 1.
          233  +**
          234  +** For EBCDIC, the rules are more complex but have the same
          235  +** end result.
          236  +**
          237  +** Ticket #1066.  the SQL standard does not allow '$' in the
          238  +** middle of identifiers.  But many SQL implementations do. 
          239  +** SQLite will allow '$' in identifiers for compatibility.
          240  +** But the feature is undocumented.
          241  +*/
          242  +#define IdChar(C)  ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
          243  +
          244  +/*
          245  +** Ignore testcase() macros
          246  +*/
          247  +#define testcase(X)
          248  +
          249  +/*
          250  +** Token values
          251  +*/
          252  +#define TK_SPACE    0
          253  +#define TK_NAME     1
          254  +#define TK_LITERAL  2
          255  +#define TK_PUNCT    3
          256  +#define TK_ERROR    4
          257  +
          258  +#define TK_MINUS    TK_PUNCT
          259  +#define TK_LP       TK_PUNCT
          260  +#define TK_RP       TK_PUNCT
          261  +#define TK_SEMI     TK_PUNCT
          262  +#define TK_PLUS     TK_PUNCT
          263  +#define TK_STAR     TK_PUNCT
          264  +#define TK_SLASH    TK_PUNCT
          265  +#define TK_REM      TK_PUNCT
          266  +#define TK_EQ       TK_PUNCT
          267  +#define TK_LE       TK_PUNCT
          268  +#define TK_NE       TK_PUNCT
          269  +#define TK_LSHIFT   TK_PUNCT
          270  +#define TK_LT       TK_PUNCT
          271  +#define TK_GE       TK_PUNCT
          272  +#define TK_RSHIFT   TK_PUNCT
          273  +#define TK_GT       TK_PUNCT
          274  +#define TK_GE       TK_PUNCT
          275  +#define TK_BITOR    TK_PUNCT
          276  +#define TK_CONCAT   TK_PUNCT
          277  +#define TK_COMMA    TK_PUNCT
          278  +#define TK_BITAND   TK_PUNCT
          279  +#define TK_BITNOT   TK_PUNCT
          280  +#define TK_STRING   TK_LITERAL
          281  +#define TK_ID       TK_NAME
          282  +#define TK_ILLEGAL  TK_ERROR
          283  +#define TK_DOT      TK_PUNCT
          284  +#define TK_INTEGER  TK_LITERAL
          285  +#define TK_FLOAT    TK_LITERAL
          286  +#define TK_VARIABLE TK_LITERAL
          287  +#define TK_BLOB     TK_LITERAL
          288  +
          289  +/*
          290  +** Return the length (in bytes) of the token that begins at z[0]. 
          291  +** Store the token type in *tokenType before returning.
          292  +*/
          293  +static int sqlite3GetToken(const unsigned char *z, int *tokenType){
          294  +  int i, c;
          295  +  switch( aiClass[*z] ){  /* Switch on the character-class of the first byte
          296  +                          ** of the token. See the comment on the CC_ defines
          297  +                          ** above. */
          298  +    case CC_SPACE: {
          299  +      for(i=1; sqlite3Isspace(z[i]); i++){}
          300  +      *tokenType = TK_SPACE;
          301  +      return i;
          302  +    }
          303  +    case CC_MINUS: {
          304  +      if( z[1]=='-' ){
          305  +        for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
          306  +        *tokenType = TK_SPACE;
          307  +        return i;
          308  +      }
          309  +      *tokenType = TK_MINUS;
          310  +      return 1;
          311  +    }
          312  +    case CC_LP: {
          313  +      *tokenType = TK_LP;
          314  +      return 1;
          315  +    }
          316  +    case CC_RP: {
          317  +      *tokenType = TK_RP;
          318  +      return 1;
          319  +    }
          320  +    case CC_SEMI: {
          321  +      *tokenType = TK_SEMI;
          322  +      return 1;
          323  +    }
          324  +    case CC_PLUS: {
          325  +      *tokenType = TK_PLUS;
          326  +      return 1;
          327  +    }
          328  +    case CC_STAR: {
          329  +      *tokenType = TK_STAR;
          330  +      return 1;
          331  +    }
          332  +    case CC_SLASH: {
          333  +      if( z[1]!='*' || z[2]==0 ){
          334  +        *tokenType = TK_SLASH;
          335  +        return 1;
          336  +      }
          337  +      for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
          338  +      if( c ) i++;
          339  +      *tokenType = TK_SPACE;
          340  +      return i;
          341  +    }
          342  +    case CC_PERCENT: {
          343  +      *tokenType = TK_REM;
          344  +      return 1;
          345  +    }
          346  +    case CC_EQ: {
          347  +      *tokenType = TK_EQ;
          348  +      return 1 + (z[1]=='=');
          349  +    }
          350  +    case CC_LT: {
          351  +      if( (c=z[1])=='=' ){
          352  +        *tokenType = TK_LE;
          353  +        return 2;
          354  +      }else if( c=='>' ){
          355  +        *tokenType = TK_NE;
          356  +        return 2;
          357  +      }else if( c=='<' ){
          358  +        *tokenType = TK_LSHIFT;
          359  +        return 2;
          360  +      }else{
          361  +        *tokenType = TK_LT;
          362  +        return 1;
          363  +      }
          364  +    }
          365  +    case CC_GT: {
          366  +      if( (c=z[1])=='=' ){
          367  +        *tokenType = TK_GE;
          368  +        return 2;
          369  +      }else if( c=='>' ){
          370  +        *tokenType = TK_RSHIFT;
          371  +        return 2;
          372  +      }else{
          373  +        *tokenType = TK_GT;
          374  +        return 1;
          375  +      }
          376  +    }
          377  +    case CC_BANG: {
          378  +      if( z[1]!='=' ){
          379  +        *tokenType = TK_ILLEGAL;
          380  +        return 1;
          381  +      }else{
          382  +        *tokenType = TK_NE;
          383  +        return 2;
          384  +      }
          385  +    }
          386  +    case CC_PIPE: {
          387  +      if( z[1]!='|' ){
          388  +        *tokenType = TK_BITOR;
          389  +        return 1;
          390  +      }else{
          391  +        *tokenType = TK_CONCAT;
          392  +        return 2;
          393  +      }
          394  +    }
          395  +    case CC_COMMA: {
          396  +      *tokenType = TK_COMMA;
          397  +      return 1;
          398  +    }
          399  +    case CC_AND: {
          400  +      *tokenType = TK_BITAND;
          401  +      return 1;
          402  +    }
          403  +    case CC_TILDA: {
          404  +      *tokenType = TK_BITNOT;
          405  +      return 1;
          406  +    }
          407  +    case CC_QUOTE: {
          408  +      int delim = z[0];
          409  +      testcase( delim=='`' );
          410  +      testcase( delim=='\'' );
          411  +      testcase( delim=='"' );
          412  +      for(i=1; (c=z[i])!=0; i++){
          413  +        if( c==delim ){
          414  +          if( z[i+1]==delim ){
          415  +            i++;
          416  +          }else{
          417  +            break;
          418  +          }
          419  +        }
          420  +      }
          421  +      if( c=='\'' ){
          422  +        *tokenType = TK_STRING;
          423  +        return i+1;
          424  +      }else if( c!=0 ){
          425  +        *tokenType = TK_ID;
          426  +        return i+1;
          427  +      }else{
          428  +        *tokenType = TK_ILLEGAL;
          429  +        return i;
          430  +      }
          431  +    }
          432  +    case CC_DOT: {
          433  +      if( !sqlite3Isdigit(z[1]) ){
          434  +        *tokenType = TK_DOT;
          435  +        return 1;
          436  +      }
          437  +      /* If the next character is a digit, this is a floating point
          438  +      ** number that begins with ".".  Fall thru into the next case */
          439  +    }
          440  +    case CC_DIGIT: {
          441  +      *tokenType = TK_INTEGER;
          442  +      if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
          443  +        for(i=3; sqlite3Isxdigit(z[i]); i++){}
          444  +        return i;
          445  +      }
          446  +      for(i=0; sqlite3Isdigit(z[i]); i++){}
          447  +      if( z[i]=='.' ){
          448  +        i++;
          449  +        while( sqlite3Isdigit(z[i]) ){ i++; }
          450  +        *tokenType = TK_FLOAT;
          451  +      }
          452  +      if( (z[i]=='e' || z[i]=='E') &&
          453  +           ( sqlite3Isdigit(z[i+1]) 
          454  +            || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
          455  +           )
          456  +      ){
          457  +        i += 2;
          458  +        while( sqlite3Isdigit(z[i]) ){ i++; }
          459  +        *tokenType = TK_FLOAT;
          460  +      }
          461  +      while( IdChar(z[i]) ){
          462  +        *tokenType = TK_ILLEGAL;
          463  +        i++;
          464  +      }
          465  +      return i;
          466  +    }
          467  +    case CC_QUOTE2: {
          468  +      for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
          469  +      *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
          470  +      return i;
          471  +    }
          472  +    case CC_VARNUM: {
          473  +      *tokenType = TK_VARIABLE;
          474  +      for(i=1; sqlite3Isdigit(z[i]); i++){}
          475  +      return i;
          476  +    }
          477  +    case CC_DOLLAR:
          478  +    case CC_VARALPHA: {
          479  +      int n = 0;
          480  +      testcase( z[0]=='$' );  testcase( z[0]=='@' );
          481  +      testcase( z[0]==':' );  testcase( z[0]=='#' );
          482  +      *tokenType = TK_VARIABLE;
          483  +      for(i=1; (c=z[i])!=0; i++){
          484  +        if( IdChar(c) ){
          485  +          n++;
          486  +        }else if( c=='(' && n>0 ){
          487  +          do{
          488  +            i++;
          489  +          }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
          490  +          if( c==')' ){
          491  +            i++;
          492  +          }else{
          493  +            *tokenType = TK_ILLEGAL;
          494  +          }
          495  +          break;
          496  +        }else if( c==':' && z[i+1]==':' ){
          497  +          i++;
          498  +        }else{
          499  +          break;
          500  +        }
          501  +      }
          502  +      if( n==0 ) *tokenType = TK_ILLEGAL;
          503  +      return i;
          504  +    }
          505  +    case CC_KYWD: {
          506  +      for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
          507  +      if( IdChar(z[i]) ){
          508  +        /* This token started out using characters that can appear in keywords,
          509  +        ** but z[i] is a character not allowed within keywords, so this must
          510  +        ** be an identifier instead */
          511  +        i++;
          512  +        break;
          513  +      }
          514  +      *tokenType = TK_ID;
          515  +      return i;
          516  +    }
          517  +    case CC_X: {
          518  +      testcase( z[0]=='x' ); testcase( z[0]=='X' );
          519  +      if( z[1]=='\'' ){
          520  +        *tokenType = TK_BLOB;
          521  +        for(i=2; sqlite3Isxdigit(z[i]); i++){}
          522  +        if( z[i]!='\'' || i%2 ){
          523  +          *tokenType = TK_ILLEGAL;
          524  +          while( z[i] && z[i]!='\'' ){ i++; }
          525  +        }
          526  +        if( z[i] ) i++;
          527  +        return i;
          528  +      }
          529  +      /* If it is not a BLOB literal, then it must be an ID, since no
          530  +      ** SQL keywords start with the letter 'x'.  Fall through */
          531  +    }
          532  +    case CC_ID: {
          533  +      i = 1;
          534  +      break;
          535  +    }
          536  +    default: {
          537  +      *tokenType = TK_ILLEGAL;
          538  +      return 1;
          539  +    }
          540  +  }
          541  +  while( IdChar(z[i]) ){ i++; }
          542  +  *tokenType = TK_ID;
          543  +  return i;
          544  +}
          545  +
          546  +char *sqlite3_normalize(const char *zSql){
          547  +  char *z;              /* The output string */
          548  +  sqlite3_int64 nZ;     /* Size of the output string in bytes */
          549  +  sqlite3_int64 nSql;   /* Size of the input string in bytes */
          550  +  int i;                /* Next character to read from zSql[] */
          551  +  int j;                /* Next slot to fill in on z[] */
          552  +  int tokenType;        /* Type of the next token */
          553  +  int n;                /* Size of the next token */
          554  +  int k;                /* Loop counter */
          555  +
          556  +  nSql = strlen(zSql);
          557  +  nZ = nSql;
          558  +  z = sqlite3_malloc64( nZ+2 );
          559  +  if( z==0 ) return 0;
          560  +  for(i=j=0; zSql[i]; i += n){
          561  +    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
          562  +    switch( tokenType ){
          563  +      case TK_SPACE: {
          564  +        break;
          565  +      }
          566  +      case TK_ERROR: {
          567  +        sqlite3_free(z);
          568  +        return 0;
          569  +      }
          570  +      case TK_LITERAL: {
          571  +        z[j++] = '?';
          572  +        break;
          573  +      }
          574  +      case TK_PUNCT:
          575  +      case TK_NAME: {
          576  +        if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){
          577  +          if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3]))
          578  +           || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4]))
          579  +          ){
          580  +            /* NULL is a keyword in this case, not a literal value */
          581  +          }else{
          582  +            /* Here the NULL is a literal value */
          583  +            z[j++] = '?';
          584  +            break;
          585  +          }
          586  +        }
          587  +        if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' ';
          588  +        for(k=0; k<n; k++){
          589  +          z[j++] = sqlite3Tolower(zSql[i+k]);
          590  +        }
          591  +        break;
          592  +      }
          593  +    }
          594  +  }
          595  +  while( j>0 && z[j-1]==' ' ){ j--; }
          596  +  if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; }
          597  +  z[j] = 0;
          598  +
          599  +  /* Make a second pass converting "in(...)" where the "..." is not a
          600  +  ** SELECT statement into "in(?,?,?)" */
          601  +  for(i=0; i<j; i=n){
          602  +    char *zIn = strstr(z+i, "in(");
          603  +    int nParen;
          604  +    if( zIn==0 ) break;
          605  +    n = (int)(zIn-z)+3;  /* Index of first char past "in(" */
          606  +    if( n && IdChar(zIn[-1]) ) continue;
          607  +    if( strncmp(zIn, "in(select",9)==0 && !IdChar(zIn[9]) ) continue;
          608  +    if( strncmp(zIn, "in(with",7)==0 && !IdChar(zIn[7]) ) continue;
          609  +    for(nParen=1, k=0; z[n+k]; k++){
          610  +      if( z[n+k]=='(' ) nParen++;
          611  +      if( z[n+k]==')' ){
          612  +        nParen--;
          613  +        if( nParen==0 ) break;
          614  +      }
          615  +    }
          616  +    /* k is the number of bytes in the "..." within "in(...)" */
          617  +    if( k<5 ){
          618  +      z = sqlite3_realloc64(z, j+(5-k)+1);
          619  +      if( z==0 ) return 0;
          620  +      memmove(z+n+5, z+n+k, j-(n+k));
          621  +    }else if( k>5 ){
          622  +      memmove(z+n+5, z+n+k, j-(n+k));
          623  +    }
          624  +    j = j-k+5;
          625  +    z[j] = 0;
          626  +    memcpy(z+n, "?,?,?", 5);
          627  +  }
          628  +  return z;
          629  +}
          630  +
          631  +/*
          632  +** For testing purposes, or to build a stand-alone SQL normalizer program,
          633  +** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link
          634  +** it against any SQLite library.  The resulting command-line program will
          635  +** run sqlite3_normalize() over the text of all files named on the command-
          636  +** line and show the result on standard output.
          637  +*/
          638  +#ifdef SQLITE_NORMALIZE_CLI
          639  +#include <stdio.h>
          640  +#include <stdlib.h>
          641  +
          642  +/*
          643  +** Break zIn up into separate SQL statements and run sqlite3_normalize()
          644  +** on each one.  Print the result of each run.
          645  +*/
          646  +static void normalizeFile(char *zIn){
          647  +  int i;
          648  +  if( zIn==0 ) return;
          649  +  for(i=0; zIn[i]; i++){
          650  +    char cSaved;
          651  +    if( zIn[i]!=';' ) continue;
          652  +    cSaved = zIn[i+1];
          653  +    zIn[i+1] = 0;
          654  +    if( sqlite3_complete(zIn) ){
          655  +      char *zOut = sqlite3_normalize(zIn);
          656  +      if( zOut ){
          657  +        printf("%s\n", zOut);
          658  +        sqlite3_free(zOut);
          659  +      }else{
          660  +        fprintf(stderr, "ERROR: %s\n", zIn);
          661  +      }
          662  +      zIn[i+1] = cSaved;
          663  +      zIn += i+1;
          664  +      i = -1;
          665  +    }else{
          666  +      zIn[i+1] = cSaved;
          667  +    }
          668  +  }
          669  +}
          670  +
          671  +/*
          672  +** The main routine for "sql_normalize".  Read files named on the
          673  +** command-line and run the text of each through sqlite3_normalize().
          674  +*/
          675  +int main(int argc, char **argv){
          676  +  int i;
          677  +  FILE *in;
          678  +  char *zBuf = 0;
          679  +  sqlite3_int64 sz, got;
          680  +
          681  +  for(i=1; i<argc; i++){
          682  +    in = fopen(argv[i], "rb");
          683  +    if( in==0 ){
          684  +      fprintf(stderr, "cannot open \"%s\"\n", argv[i]);
          685  +      continue;
          686  +    }
          687  +    fseek(in, 0, SEEK_END);
          688  +    sz = ftell(in);
          689  +    rewind(in);
          690  +    zBuf = sqlite3_realloc64(zBuf, sz+1);
          691  +    if( zBuf==0 ){
          692  +      fprintf(stderr, "failed to malloc for %lld bytes\n", sz);
          693  +      exit(1);
          694  +    }
          695  +    got = fread(zBuf, 1, sz, in);
          696  +    fclose(in);
          697  +    if( got!=sz ){
          698  +      fprintf(stderr, "only able to read %lld of %lld bytes from \"%s\"\n",
          699  +              got, sz, argv[i]);
          700  +    }else{
          701  +      zBuf[got] = 0;
          702  +      normalizeFile(zBuf);
          703  +    }
          704  +  }
          705  +  sqlite3_free(zBuf);
          706  +}
          707  +#endif /* SQLITE_NORMALIZE_CLI */

Changes to ext/misc/rot13.c.

    43     43     sqlite3_context *context,
    44     44     int argc,
    45     45     sqlite3_value **argv
    46     46   ){
    47     47     const unsigned char *zIn;
    48     48     int nIn;
    49     49     unsigned char *zOut;
    50         -  char *zToFree = 0;
           50  +  unsigned char *zToFree = 0;
    51     51     int i;
    52         -  char zTemp[100];
           52  +  unsigned char zTemp[100];
    53     53     assert( argc==1 );
    54     54     if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
    55     55     zIn = (const unsigned char*)sqlite3_value_text(argv[0]);
    56     56     nIn = sqlite3_value_bytes(argv[0]);
    57     57     if( nIn<sizeof(zTemp)-1 ){
    58     58       zOut = zTemp;
    59     59     }else{
    60         -    zOut = zToFree = sqlite3_malloc( nIn+1 );
           60  +    zOut = zToFree = (unsigned char*)sqlite3_malloc64( nIn+1 );
    61     61       if( zOut==0 ){
    62     62         sqlite3_result_error_nomem(context);
    63     63         return;
    64     64       }
    65     65     }
    66     66     for(i=0; i<nIn; i++) zOut[i] = rot13(zIn[i]);
    67     67     zOut[i] = 0;

Changes to ext/misc/shathree.c.

    74     74   };
    75     75   
    76     76   /*
    77     77   ** A single step of the Keccak mixing function for a 1600-bit state
    78     78   */
    79     79   static void KeccakF1600Step(SHA3Context *p){
    80     80     int i;
    81         -  u64 B0, B1, B2, B3, B4;
    82         -  u64 C0, C1, C2, C3, C4;
    83         -  u64 D0, D1, D2, D3, D4;
           81  +  u64 b0, b1, b2, b3, b4;
           82  +  u64 c0, c1, c2, c3, c4;
           83  +  u64 d0, d1, d2, d3, d4;
    84     84     static const u64 RC[] = {
    85     85       0x0000000000000001ULL,  0x0000000000008082ULL,
    86     86       0x800000000000808aULL,  0x8000000080008000ULL,
    87     87       0x000000000000808bULL,  0x0000000080000001ULL,
    88     88       0x8000000080008081ULL,  0x8000000000008009ULL,
    89     89       0x000000000000008aULL,  0x0000000000000088ULL,
    90     90       0x0000000080008009ULL,  0x000000008000000aULL,
................................................................................
    91     91       0x000000008000808bULL,  0x800000000000008bULL,
    92     92       0x8000000000008089ULL,  0x8000000000008003ULL,
    93     93       0x8000000000008002ULL,  0x8000000000000080ULL,
    94     94       0x000000000000800aULL,  0x800000008000000aULL,
    95     95       0x8000000080008081ULL,  0x8000000000008080ULL,
    96     96       0x0000000080000001ULL,  0x8000000080008008ULL
    97     97     };
    98         -# define A00 (p->u.s[0])
    99         -# define A01 (p->u.s[1])
   100         -# define A02 (p->u.s[2])
   101         -# define A03 (p->u.s[3])
   102         -# define A04 (p->u.s[4])
   103         -# define A10 (p->u.s[5])
   104         -# define A11 (p->u.s[6])
   105         -# define A12 (p->u.s[7])
   106         -# define A13 (p->u.s[8])
   107         -# define A14 (p->u.s[9])
   108         -# define A20 (p->u.s[10])
   109         -# define A21 (p->u.s[11])
   110         -# define A22 (p->u.s[12])
   111         -# define A23 (p->u.s[13])
   112         -# define A24 (p->u.s[14])
   113         -# define A30 (p->u.s[15])
   114         -# define A31 (p->u.s[16])
   115         -# define A32 (p->u.s[17])
   116         -# define A33 (p->u.s[18])
   117         -# define A34 (p->u.s[19])
   118         -# define A40 (p->u.s[20])
   119         -# define A41 (p->u.s[21])
   120         -# define A42 (p->u.s[22])
   121         -# define A43 (p->u.s[23])
   122         -# define A44 (p->u.s[24])
           98  +# define a00 (p->u.s[0])
           99  +# define a01 (p->u.s[1])
          100  +# define a02 (p->u.s[2])
          101  +# define a03 (p->u.s[3])
          102  +# define a04 (p->u.s[4])
          103  +# define a10 (p->u.s[5])
          104  +# define a11 (p->u.s[6])
          105  +# define a12 (p->u.s[7])
          106  +# define a13 (p->u.s[8])
          107  +# define a14 (p->u.s[9])
          108  +# define a20 (p->u.s[10])
          109  +# define a21 (p->u.s[11])
          110  +# define a22 (p->u.s[12])
          111  +# define a23 (p->u.s[13])
          112  +# define a24 (p->u.s[14])
          113  +# define a30 (p->u.s[15])
          114  +# define a31 (p->u.s[16])
          115  +# define a32 (p->u.s[17])
          116  +# define a33 (p->u.s[18])
          117  +# define a34 (p->u.s[19])
          118  +# define a40 (p->u.s[20])
          119  +# define a41 (p->u.s[21])
          120  +# define a42 (p->u.s[22])
          121  +# define a43 (p->u.s[23])
          122  +# define a44 (p->u.s[24])
   123    123   # define ROL64(a,x) ((a<<x)|(a>>(64-x)))
   124    124   
   125    125     for(i=0; i<24; i+=4){
   126         -    C0 = A00^A10^A20^A30^A40;
   127         -    C1 = A01^A11^A21^A31^A41;
   128         -    C2 = A02^A12^A22^A32^A42;
   129         -    C3 = A03^A13^A23^A33^A43;
   130         -    C4 = A04^A14^A24^A34^A44;
   131         -    D0 = C4^ROL64(C1, 1);
   132         -    D1 = C0^ROL64(C2, 1);
   133         -    D2 = C1^ROL64(C3, 1);
   134         -    D3 = C2^ROL64(C4, 1);
   135         -    D4 = C3^ROL64(C0, 1);
   136         -
   137         -    B0 = (A00^D0);
   138         -    B1 = ROL64((A11^D1), 44);
   139         -    B2 = ROL64((A22^D2), 43);
   140         -    B3 = ROL64((A33^D3), 21);
   141         -    B4 = ROL64((A44^D4), 14);
   142         -    A00 =   B0 ^((~B1)&  B2 );
   143         -    A00 ^= RC[i];
   144         -    A11 =   B1 ^((~B2)&  B3 );
   145         -    A22 =   B2 ^((~B3)&  B4 );
   146         -    A33 =   B3 ^((~B4)&  B0 );
   147         -    A44 =   B4 ^((~B0)&  B1 );
   148         -
   149         -    B2 = ROL64((A20^D0), 3);
   150         -    B3 = ROL64((A31^D1), 45);
   151         -    B4 = ROL64((A42^D2), 61);
   152         -    B0 = ROL64((A03^D3), 28);
   153         -    B1 = ROL64((A14^D4), 20);
   154         -    A20 =   B0 ^((~B1)&  B2 );
   155         -    A31 =   B1 ^((~B2)&  B3 );
   156         -    A42 =   B2 ^((~B3)&  B4 );
   157         -    A03 =   B3 ^((~B4)&  B0 );
   158         -    A14 =   B4 ^((~B0)&  B1 );
   159         -
   160         -    B4 = ROL64((A40^D0), 18);
   161         -    B0 = ROL64((A01^D1), 1);
   162         -    B1 = ROL64((A12^D2), 6);
   163         -    B2 = ROL64((A23^D3), 25);
   164         -    B3 = ROL64((A34^D4), 8);
   165         -    A40 =   B0 ^((~B1)&  B2 );
   166         -    A01 =   B1 ^((~B2)&  B3 );
   167         -    A12 =   B2 ^((~B3)&  B4 );
   168         -    A23 =   B3 ^((~B4)&  B0 );
   169         -    A34 =   B4 ^((~B0)&  B1 );
   170         -
   171         -    B1 = ROL64((A10^D0), 36);
   172         -    B2 = ROL64((A21^D1), 10);
   173         -    B3 = ROL64((A32^D2), 15);
   174         -    B4 = ROL64((A43^D3), 56);
   175         -    B0 = ROL64((A04^D4), 27);
   176         -    A10 =   B0 ^((~B1)&  B2 );
   177         -    A21 =   B1 ^((~B2)&  B3 );
   178         -    A32 =   B2 ^((~B3)&  B4 );
   179         -    A43 =   B3 ^((~B4)&  B0 );
   180         -    A04 =   B4 ^((~B0)&  B1 );
   181         -
   182         -    B3 = ROL64((A30^D0), 41);
   183         -    B4 = ROL64((A41^D1), 2);
   184         -    B0 = ROL64((A02^D2), 62);
   185         -    B1 = ROL64((A13^D3), 55);
   186         -    B2 = ROL64((A24^D4), 39);
   187         -    A30 =   B0 ^((~B1)&  B2 );
   188         -    A41 =   B1 ^((~B2)&  B3 );
   189         -    A02 =   B2 ^((~B3)&  B4 );
   190         -    A13 =   B3 ^((~B4)&  B0 );
   191         -    A24 =   B4 ^((~B0)&  B1 );
   192         -
   193         -    C0 = A00^A20^A40^A10^A30;
   194         -    C1 = A11^A31^A01^A21^A41;
   195         -    C2 = A22^A42^A12^A32^A02;
   196         -    C3 = A33^A03^A23^A43^A13;
   197         -    C4 = A44^A14^A34^A04^A24;
   198         -    D0 = C4^ROL64(C1, 1);
   199         -    D1 = C0^ROL64(C2, 1);
   200         -    D2 = C1^ROL64(C3, 1);
   201         -    D3 = C2^ROL64(C4, 1);
   202         -    D4 = C3^ROL64(C0, 1);
   203         -
   204         -    B0 = (A00^D0);
   205         -    B1 = ROL64((A31^D1), 44);
   206         -    B2 = ROL64((A12^D2), 43);
   207         -    B3 = ROL64((A43^D3), 21);
   208         -    B4 = ROL64((A24^D4), 14);
   209         -    A00 =   B0 ^((~B1)&  B2 );
   210         -    A00 ^= RC[i+1];
   211         -    A31 =   B1 ^((~B2)&  B3 );
   212         -    A12 =   B2 ^((~B3)&  B4 );
   213         -    A43 =   B3 ^((~B4)&  B0 );
   214         -    A24 =   B4 ^((~B0)&  B1 );
   215         -
   216         -    B2 = ROL64((A40^D0), 3);
   217         -    B3 = ROL64((A21^D1), 45);
   218         -    B4 = ROL64((A02^D2), 61);
   219         -    B0 = ROL64((A33^D3), 28);
   220         -    B1 = ROL64((A14^D4), 20);
   221         -    A40 =   B0 ^((~B1)&  B2 );
   222         -    A21 =   B1 ^((~B2)&  B3 );
   223         -    A02 =   B2 ^((~B3)&  B4 );
   224         -    A33 =   B3 ^((~B4)&  B0 );
   225         -    A14 =   B4 ^((~B0)&  B1 );
   226         -
   227         -    B4 = ROL64((A30^D0), 18);
   228         -    B0 = ROL64((A11^D1), 1);
   229         -    B1 = ROL64((A42^D2), 6);
   230         -    B2 = ROL64((A23^D3), 25);
   231         -    B3 = ROL64((A04^D4), 8);
   232         -    A30 =   B0 ^((~B1)&  B2 );
   233         -    A11 =   B1 ^((~B2)&  B3 );
   234         -    A42 =   B2 ^((~B3)&  B4 );
   235         -    A23 =   B3 ^((~B4)&  B0 );
   236         -    A04 =   B4 ^((~B0)&  B1 );
   237         -
   238         -    B1 = ROL64((A20^D0), 36);
   239         -    B2 = ROL64((A01^D1), 10);
   240         -    B3 = ROL64((A32^D2), 15);
   241         -    B4 = ROL64((A13^D3), 56);
   242         -    B0 = ROL64((A44^D4), 27);
   243         -    A20 =   B0 ^((~B1)&  B2 );
   244         -    A01 =   B1 ^((~B2)&  B3 );
   245         -    A32 =   B2 ^((~B3)&  B4 );
   246         -    A13 =   B3 ^((~B4)&  B0 );
   247         -    A44 =   B4 ^((~B0)&  B1 );
   248         -
   249         -    B3 = ROL64((A10^D0), 41);
   250         -    B4 = ROL64((A41^D1), 2);
   251         -    B0 = ROL64((A22^D2), 62);
   252         -    B1 = ROL64((A03^D3), 55);
   253         -    B2 = ROL64((A34^D4), 39);
   254         -    A10 =   B0 ^((~B1)&  B2 );
   255         -    A41 =   B1 ^((~B2)&  B3 );
   256         -    A22 =   B2 ^((~B3)&  B4 );
   257         -    A03 =   B3 ^((~B4)&  B0 );
   258         -    A34 =   B4 ^((~B0)&  B1 );
          126  +    c0 = a00^a10^a20^a30^a40;
          127  +    c1 = a01^a11^a21^a31^a41;
          128  +    c2 = a02^a12^a22^a32^a42;
          129  +    c3 = a03^a13^a23^a33^a43;
          130  +    c4 = a04^a14^a24^a34^a44;
          131  +    d0 = c4^ROL64(c1, 1);
          132  +    d1 = c0^ROL64(c2, 1);
          133  +    d2 = c1^ROL64(c3, 1);
          134  +    d3 = c2^ROL64(c4, 1);
          135  +    d4 = c3^ROL64(c0, 1);
          136  +
          137  +    b0 = (a00^d0);
          138  +    b1 = ROL64((a11^d1), 44);
          139  +    b2 = ROL64((a22^d2), 43);
          140  +    b3 = ROL64((a33^d3), 21);
          141  +    b4 = ROL64((a44^d4), 14);
          142  +    a00 =   b0 ^((~b1)&  b2 );
          143  +    a00 ^= RC[i];
          144  +    a11 =   b1 ^((~b2)&  b3 );
          145  +    a22 =   b2 ^((~b3)&  b4 );
          146  +    a33 =   b3 ^((~b4)&  b0 );
          147  +    a44 =   b4 ^((~b0)&  b1 );
          148  +
          149  +    b2 = ROL64((a20^d0), 3);
          150  +    b3 = ROL64((a31^d1), 45);
          151  +    b4 = ROL64((a42^d2), 61);
          152  +    b0 = ROL64((a03^d3), 28);
          153  +    b1 = ROL64((a14^d4), 20);
          154  +    a20 =   b0 ^((~b1)&  b2 );
          155  +    a31 =   b1 ^((~b2)&  b3 );
          156  +    a42 =   b2 ^((~b3)&  b4 );
          157  +    a03 =   b3 ^((~b4)&  b0 );
          158  +    a14 =   b4 ^((~b0)&  b1 );
          159  +
          160  +    b4 = ROL64((a40^d0), 18);
          161  +    b0 = ROL64((a01^d1), 1);
          162  +    b1 = ROL64((a12^d2), 6);
          163  +    b2 = ROL64((a23^d3), 25);
          164  +    b3 = ROL64((a34^d4), 8);
          165  +    a40 =   b0 ^((~b1)&  b2 );
          166  +    a01 =   b1 ^((~b2)&  b3 );
          167  +    a12 =   b2 ^((~b3)&  b4 );
          168  +    a23 =   b3 ^((~b4)&  b0 );
          169  +    a34 =   b4 ^((~b0)&  b1 );
          170  +
          171  +    b1 = ROL64((a10^d0), 36);
          172  +    b2 = ROL64((a21^d1), 10);
          173  +    b3 = ROL64((a32^d2), 15);
          174  +    b4 = ROL64((a43^d3), 56);
          175  +    b0 = ROL64((a04^d4), 27);
          176  +    a10 =   b0 ^((~b1)&  b2 );
          177  +    a21 =   b1 ^((~b2)&  b3 );
          178  +    a32 =   b2 ^((~b3)&  b4 );
          179  +    a43 =   b3 ^((~b4)&  b0 );
          180  +    a04 =   b4 ^((~b0)&  b1 );
          181  +
          182  +    b3 = ROL64((a30^d0), 41);
          183  +    b4 = ROL64((a41^d1), 2);
          184  +    b0 = ROL64((a02^d2), 62);
          185  +    b1 = ROL64((a13^d3), 55);
          186  +    b2 = ROL64((a24^d4), 39);
          187  +    a30 =   b0 ^((~b1)&  b2 );
          188  +    a41 =   b1 ^((~b2)&  b3 );
          189  +    a02 =   b2 ^((~b3)&  b4 );
          190  +    a13 =   b3 ^((~b4)&  b0 );
          191  +    a24 =   b4 ^((~b0)&  b1 );
          192  +
          193  +    c0 = a00^a20^a40^a10^a30;
          194  +    c1 = a11^a31^a01^a21^a41;
          195  +    c2 = a22^a42^a12^a32^a02;
          196  +    c3 = a33^a03^a23^a43^a13;
          197  +    c4 = a44^a14^a34^a04^a24;
          198  +    d0 = c4^ROL64(c1, 1);
          199  +    d1 = c0^ROL64(c2, 1);
          200  +    d2 = c1^ROL64(c3, 1);
          201  +    d3 = c2^ROL64(c4, 1);
          202  +    d4 = c3^ROL64(c0, 1);
          203  +
          204  +    b0 = (a00^d0);
          205  +    b1 = ROL64((a31^d1), 44);
          206  +    b2 = ROL64((a12^d2), 43);
          207  +    b3 = ROL64((a43^d3), 21);
          208  +    b4 = ROL64((a24^d4), 14);
          209  +    a00 =   b0 ^((~b1)&  b2 );
          210  +    a00 ^= RC[i+1];
          211  +    a31 =   b1 ^((~b2)&  b3 );
          212  +    a12 =   b2 ^((~b3)&  b4 );
          213  +    a43 =   b3 ^((~b4)&  b0 );
          214  +    a24 =   b4 ^((~b0)&  b1 );
          215  +
          216  +    b2 = ROL64((a40^d0), 3);
          217  +    b3 = ROL64((a21^d1), 45);
          218  +    b4 = ROL64((a02^d2), 61);
          219  +    b0 = ROL64((a33^d3), 28);
          220  +    b1 = ROL64((a14^d4), 20);
          221  +    a40 =   b0 ^((~b1)&  b2 );
          222  +    a21 =   b1 ^((~b2)&  b3 );
          223  +    a02 =   b2 ^((~b3)&  b4 );
          224  +    a33 =   b3 ^((~b4)&  b0 );
          225  +    a14 =   b4 ^((~b0)&  b1 );
          226  +
          227  +    b4 = ROL64((a30^d0), 18);
          228  +    b0 = ROL64((a11^d1), 1);
          229  +    b1 = ROL64((a42^d2), 6);
          230  +    b2 = ROL64((a23^d3), 25);
          231  +    b3 = ROL64((a04^d4), 8);
          232  +    a30 =   b0 ^((~b1)&  b2 );
          233  +    a11 =   b1 ^((~b2)&  b3 );
          234  +    a42 =   b2 ^((~b3)&  b4 );
          235  +    a23 =   b3 ^((~b4)&  b0 );
          236  +    a04 =   b4 ^((~b0)&  b1 );
          237  +
          238  +    b1 = ROL64((a20^d0), 36);
          239  +    b2 = ROL64((a01^d1), 10);
          240  +    b3 = ROL64((a32^d2), 15);
          241  +    b4 = ROL64((a13^d3), 56);
          242  +    b0 = ROL64((a44^d4), 27);
          243  +    a20 =   b0 ^((~b1)&  b2 );
          244  +    a01 =   b1 ^((~b2)&  b3 );
          245  +    a32 =   b2 ^((~b3)&  b4 );
          246  +    a13 =   b3 ^((~b4)&  b0 );
          247  +    a44 =   b4 ^((~b0)&  b1 );
          248  +
          249  +    b3 = ROL64((a10^d0), 41);
          250  +    b4 = ROL64((a41^d1), 2);
          251  +    b0 = ROL64((a22^d2), 62);
          252  +    b1 = ROL64((a03^d3), 55);
          253  +    b2 = ROL64((a34^d4), 39);
          254  +    a10 =   b0 ^((~b1)&  b2 );
          255  +    a41 =   b1 ^((~b2)&  b3 );
          256  +    a22 =   b2 ^((~b3)&  b4 );
          257  +    a03 =   b3 ^((~b4)&  b0 );
          258  +    a34 =   b4 ^((~b0)&  b1 );
          259  +
          260  +    c0 = a00^a40^a30^a20^a10;
          261  +    c1 = a31^a21^a11^a01^a41;
          262  +    c2 = a12^a02^a42^a32^a22;
          263  +    c3 = a43^a33^a23^a13^a03;
          264  +    c4 = a24^a14^a04^a44^a34;
          265  +    d0 = c4^ROL64(c1, 1);
          266  +    d1 = c0^ROL64(c2, 1);
          267  +    d2 = c1^ROL64(c3, 1);
          268  +    d3 = c2^ROL64(c4, 1);
          269  +    d4 = c3^ROL64(c0, 1);
          270  +
          271  +    b0 = (a00^d0);
          272  +    b1 = ROL64((a21^d1), 44);
          273  +    b2 = ROL64((a42^d2), 43);
          274  +    b3 = ROL64((a13^d3), 21);
          275  +    b4 = ROL64((a34^d4), 14);
          276  +    a00 =   b0 ^((~b1)&  b2 );
          277  +    a00 ^= RC[i+2];
          278  +    a21 =   b1 ^((~b2)&  b3 );
          279  +    a42 =   b2 ^((~b3)&  b4 );
          280  +    a13 =   b3 ^((~b4)&  b0 );
          281  +    a34 =   b4 ^((~b0)&  b1 );
          282  +
          283  +    b2 = ROL64((a30^d0), 3);
          284  +    b3 = ROL64((a01^d1), 45);
          285  +    b4 = ROL64((a22^d2), 61);
          286  +    b0 = ROL64((a43^d3), 28);
          287  +    b1 = ROL64((a14^d4), 20);
          288  +    a30 =   b0 ^((~b1)&  b2 );
          289  +    a01 =   b1 ^((~b2)&  b3 );
          290  +    a22 =   b2 ^((~b3)&  b4 );
          291  +    a43 =   b3 ^((~b4)&  b0 );
          292  +    a14 =   b4 ^((~b0)&  b1 );
          293  +
          294  +    b4 = ROL64((a10^d0), 18);
          295  +    b0 = ROL64((a31^d1), 1);
          296  +    b1 = ROL64((a02^d2), 6);
          297  +    b2 = ROL64((a23^d3), 25);
          298  +    b3 = ROL64((a44^d4), 8);
          299  +    a10 =   b0 ^((~b1)&  b2 );
          300  +    a31 =   b1 ^((~b2)&  b3 );
          301  +    a02 =   b2 ^((~b3)&  b4 );
          302  +    a23 =   b3 ^((~b4)&  b0 );
          303  +    a44 =   b4 ^((~b0)&  b1 );
          304  +
          305  +    b1 = ROL64((a40^d0), 36);
          306  +    b2 = ROL64((a11^d1), 10);
          307  +    b3 = ROL64((a32^d2), 15);
          308  +    b4 = ROL64((a03^d3), 56);
          309  +    b0 = ROL64((a24^d4), 27);
          310  +    a40 =   b0 ^((~b1)&  b2 );
          311  +    a11 =   b1 ^((~b2)&  b3 );
          312  +    a32 =   b2 ^((~b3)&  b4 );
          313  +    a03 =   b3 ^((~b4)&  b0 );
          314  +    a24 =   b4 ^((~b0)&  b1 );
          315  +
          316  +    b3 = ROL64((a20^d0), 41);
          317  +    b4 = ROL64((a41^d1), 2);
          318  +    b0 = ROL64((a12^d2), 62);
          319  +    b1 = ROL64((a33^d3), 55);
          320  +    b2 = ROL64((a04^d4), 39);
          321  +    a20 =   b0 ^((~b1)&  b2 );
          322  +    a41 =   b1 ^((~b2)&  b3 );
          323  +    a12 =   b2 ^((~b3)&  b4 );
          324  +    a33 =   b3 ^((~b4)&  b0 );
          325  +    a04 =   b4 ^((~b0)&  b1 );
          326  +
          327  +    c0 = a00^a30^a10^a40^a20;
          328  +    c1 = a21^a01^a31^a11^a41;
          329  +