/ Check-in [e17bca2c]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge recent enhancements from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA3-256: e17bca2cdb5f31a802e1a1870deb6bf17e332117ab272537f26351f1145de690
User & Date: drh 2018-05-04 19:33:34
Context
2018-05-04
20:00
Bring the code that changes the owner of WAL and rollback files when running as root into alignment with trunk. check-in: 180516bf user: drh tags: apple-osx
19:33
Merge recent enhancements from trunk. check-in: e17bca2c user: drh tags: apple-osx
18:32
Fix requirements marks. No code changes. check-in: 7fdad122 user: drh tags: trunk
2018-04-10
18:05
Merge all version 3.23.1 changes and enhancements from trunk. check-in: e20fcb51 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   183    183            memdb.lo memjournal.lo \
   184    184            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   185    185            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   186    186            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   187    187            random.lo resolve.lo rowset.lo rtree.lo \
   188    188            sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
   189    189            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   190         -         update.lo util.lo vacuum.lo \
          190  +         update.lo upsert.lo util.lo vacuum.lo \
   191    191            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   192    192            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   193    193            utf.lo vtab.lo
   194    194   
   195    195   # Object files for the amalgamation.
   196    196   #
   197    197   LIBOBJS1 = sqlite3.lo
................................................................................
   280    280     $(TOP)/src/tclsqlite.c \
   281    281     $(TOP)/src/threads.c \
   282    282     $(TOP)/src/tokenize.c \
   283    283     $(TOP)/src/treeview.c \
   284    284     $(TOP)/src/trigger.c \
   285    285     $(TOP)/src/utf.c \
   286    286     $(TOP)/src/update.c \
          287  +  $(TOP)/src/upsert.c \
   287    288     $(TOP)/src/util.c \
   288    289     $(TOP)/src/vacuum.c \
   289    290     $(TOP)/src/vdbe.c \
   290    291     $(TOP)/src/vdbe.h \
   291    292     $(TOP)/src/vdbeapi.c \
   292    293     $(TOP)/src/vdbeaux.c \
   293    294     $(TOP)/src/vdbeblob.c \
................................................................................
   923    924   
   924    925   trigger.lo:	$(TOP)/src/trigger.c $(HDR)
   925    926   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/trigger.c
   926    927   
   927    928   update.lo:	$(TOP)/src/update.c $(HDR)
   928    929   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c
   929    930   
          931  +upsert.lo:	$(TOP)/src/upsert.c $(HDR)
          932  +	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
          933  +
   930    934   utf.lo:	$(TOP)/src/utf.c $(HDR)
   931    935   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c
   932    936   
   933    937   util.lo:	$(TOP)/src/util.c $(HDR)
   934    938   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/util.c
   935    939   
   936    940   vacuum.lo:	$(TOP)/src/vacuum.c $(HDR)

Changes to Makefile.msc.

  1189   1189            memdb.lo memjournal.lo \
  1190   1190            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
  1191   1191            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
  1192   1192            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
  1193   1193            random.lo resolve.lo rowset.lo rtree.lo \
  1194   1194            sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
  1195   1195            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
  1196         -         update.lo util.lo vacuum.lo \
         1196  +         update.lo upsert.lo util.lo vacuum.lo \
  1197   1197            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
  1198   1198            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
  1199   1199            utf.lo vtab.lo
  1200   1200   # <</mark>>
  1201   1201   
  1202   1202   # Object files for the amalgamation.
  1203   1203   #
................................................................................
  1288   1288     $(TOP)\src\threads.c \
  1289   1289     $(TOP)\src\tclsqlite.c \
  1290   1290     $(TOP)\src\tokenize.c \
  1291   1291     $(TOP)\src\treeview.c \
  1292   1292     $(TOP)\src\trigger.c \
  1293   1293     $(TOP)\src\utf.c \
  1294   1294     $(TOP)\src\update.c \
         1295  +  $(TOP)\src\upsert.c \
  1295   1296     $(TOP)\src\util.c \
  1296   1297     $(TOP)\src\vacuum.c \
  1297   1298     $(TOP)\src\vdbe.c \
  1298   1299     $(TOP)\src\vdbeapi.c \
  1299   1300     $(TOP)\src\vdbeaux.c \
  1300   1301     $(TOP)\src\vdbeblob.c \
  1301   1302     $(TOP)\src\vdbemem.c \
................................................................................
  1669   1670   $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
  1670   1671   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1671   1672   
  1672   1673   # <<block2>>
  1673   1674   sqlite3.def:	libsqlite3.lib
  1674   1675   	echo EXPORTS > sqlite3.def
  1675   1676   	dumpbin /all libsqlite3.lib \
  1676         -		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
         1677  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@]*)(?:@\d+)?$$" \1 \
  1677   1678   		| sort >> sqlite3.def
  1678   1679   # <</block2>>
  1679   1680   
  1680   1681   $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
  1681   1682   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
  1682   1683   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1683   1684   
................................................................................
  1990   1991   
  1991   1992   trigger.lo:	$(TOP)\src\trigger.c $(HDR)
  1992   1993   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\trigger.c
  1993   1994   
  1994   1995   update.lo:	$(TOP)\src\update.c $(HDR)
  1995   1996   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\update.c
  1996   1997   
         1998  +upsert.lo:	$(TOP)\src\upsert.c $(HDR)
         1999  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\upsert.c
         2000  +
  1997   2001   utf.lo:	$(TOP)\src\utf.c $(HDR)
  1998   2002   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\utf.c
  1999   2003   
  2000   2004   util.lo:	$(TOP)\src\util.c $(HDR)
  2001   2005   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\util.c
  2002   2006   
  2003   2007   vacuum.lo:	$(TOP)\src\vacuum.c $(HDR)

Changes to VERSION.

     1         -3.23.1
            1  +3.24.0

Changes to autoconf/Makefile.am.

     1      1   
     2         -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @ZLIB_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
            2  +AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @ZLIB_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE @DEBUG_FLAGS@
     3      3   
     4      4   lib_LTLIBRARIES = libsqlite3.la
     5      5   libsqlite3_la_SOURCES = sqlite3.c
     6      6   libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
     7      7   
     8      8   bin_PROGRAMS = sqlite3
     9      9   sqlite3_SOURCES = shell.c sqlite3.h

Changes to autoconf/Makefile.msc.

   962    962   
   963    963   Replace.exe:
   964    964   	$(CSC) /target:exe $(TOP)\Replace.cs
   965    965   
   966    966   sqlite3.def:	Replace.exe $(LIBOBJ)
   967    967   	echo EXPORTS > sqlite3.def
   968    968   	dumpbin /all $(LIBOBJ) \
   969         -		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
          969  +		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
   970    970   		| sort >> sqlite3.def
   971    971   
   972    972   $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
   973    973   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
   974    974   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
   975    975   
   976    976   

Changes to autoconf/configure.ac.

   111    111   AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
   112    112   #-----------------------------------------------------------------------
   113    113   
   114    114   #-----------------------------------------------------------------------
   115    115   #   --enable-fts5
   116    116   #
   117    117   AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
   118         -  [--enable-fts5], [include fts5 support [default=no]])], 
   119         -  [], [enable_fts5=no])
          118  +  [--enable-fts5], [include fts5 support [default=yes]])], 
          119  +  [], [enable_fts5=yes])
   120    120   if test x"$enable_fts5" = "xyes"; then
   121    121     AC_SEARCH_LIBS(log, m)
   122    122     FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
   123    123   fi
   124    124   AC_SUBST(FTS5_FLAGS)
   125    125   #-----------------------------------------------------------------------
   126    126   
   127    127   #-----------------------------------------------------------------------
   128    128   #   --enable-json1
   129    129   #
   130    130   AC_ARG_ENABLE(json1, [AS_HELP_STRING(
   131         -  [--enable-json1], [include json1 support [default=no]])], 
   132         -  [], [enable_json1=no])
          131  +  [--enable-json1], [include json1 support [default=yes]])], 
          132  +  [], [enable_json1=yes])
   133    133   if test x"$enable_json1" = "xyes"; then
   134    134     JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
   135    135   fi
   136    136   AC_SUBST(JSON1_FLAGS)
   137    137   #-----------------------------------------------------------------------
   138    138   
   139    139   #-----------------------------------------------------------------------
................................................................................
   143    143     [--enable-session], [enable the session extension [default=no]])], 
   144    144     [], [enable_session=no])
   145    145   if test x"$enable_session" = "xyes"; then
   146    146     SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK"
   147    147   fi
   148    148   AC_SUBST(SESSION_FLAGS)
   149    149   #-----------------------------------------------------------------------
          150  +
          151  +#-----------------------------------------------------------------------
          152  +#   --enable-debug
          153  +#
          154  +AC_ARG_ENABLE(debug, [AS_HELP_STRING(
          155  +  [--enable-debug], [build with debugging features enabled [default=no]])], 
          156  +  [], [enable_session=no])
          157  +if test x"$enable_debug" = "xyes"; then
          158  +  DEBUG_FLAGS="-DSQLITE_DEBUG -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE"
          159  +fi
          160  +AC_SUBST(DEBUG_FLAGS)
          161  +#-----------------------------------------------------------------------
   150    162   
   151    163   #-----------------------------------------------------------------------
   152    164   #   --enable-static-shell
   153    165   #
   154    166   AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
   155    167     [--enable-static-shell], 
   156    168     [statically link libsqlite3 into shell tool [default=yes]])], 

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.23.1.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.24.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.23.1'
   730         -PACKAGE_STRING='sqlite 3.23.1'
          729  +PACKAGE_VERSION='3.24.0'
          730  +PACKAGE_STRING='sqlite 3.24.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1461   1461   #
  1462   1462   # Report the --help message.
  1463   1463   #
  1464   1464   if test "$ac_init_help" = "long"; then
  1465   1465     # Omit some internal or obsolete options to make the list less imposing.
  1466   1466     # This message is too long to be a string in the A/UX 3.1 sh.
  1467   1467     cat <<_ACEOF
  1468         -\`configure' configures sqlite 3.23.1 to adapt to many kinds of systems.
         1468  +\`configure' configures sqlite 3.24.0 to adapt to many kinds of systems.
  1469   1469   
  1470   1470   Usage: $0 [OPTION]... [VAR=VALUE]...
  1471   1471   
  1472   1472   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1473   1473   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1474   1474   
  1475   1475   Defaults for the options are specified in brackets.
................................................................................
  1526   1526     --build=BUILD     configure for building on BUILD [guessed]
  1527   1527     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1528   1528   _ACEOF
  1529   1529   fi
  1530   1530   
  1531   1531   if test -n "$ac_init_help"; then
  1532   1532     case $ac_init_help in
  1533         -     short | recursive ) echo "Configuration of sqlite 3.23.1:";;
         1533  +     short | recursive ) echo "Configuration of sqlite 3.24.0:";;
  1534   1534      esac
  1535   1535     cat <<\_ACEOF
  1536   1536   
  1537   1537   Optional Features:
  1538   1538     --disable-option-checking  ignore unrecognized --enable/--with options
  1539   1539     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1540   1540     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1651   1651       cd "$ac_pwd" || { ac_status=$?; break; }
  1652   1652     done
  1653   1653   fi
  1654   1654   
  1655   1655   test -n "$ac_init_help" && exit $ac_status
  1656   1656   if $ac_init_version; then
  1657   1657     cat <<\_ACEOF
  1658         -sqlite configure 3.23.1
         1658  +sqlite configure 3.24.0
  1659   1659   generated by GNU Autoconf 2.69
  1660   1660   
  1661   1661   Copyright (C) 2012 Free Software Foundation, Inc.
  1662   1662   This configure script is free software; the Free Software Foundation
  1663   1663   gives unlimited permission to copy, distribute and modify it.
  1664   1664   _ACEOF
  1665   1665     exit
................................................................................
  2070   2070     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2071   2071   
  2072   2072   } # ac_fn_c_check_header_mongrel
  2073   2073   cat >config.log <<_ACEOF
  2074   2074   This file contains any messages produced by compilers while
  2075   2075   running configure, to aid debugging if configure makes a mistake.
  2076   2076   
  2077         -It was created by sqlite $as_me 3.23.1, which was
         2077  +It was created by sqlite $as_me 3.24.0, which was
  2078   2078   generated by GNU Autoconf 2.69.  Invocation command line was
  2079   2079   
  2080   2080     $ $0 $@
  2081   2081   
  2082   2082   _ACEOF
  2083   2083   exec 5>>config.log
  2084   2084   {
................................................................................
 12238  12238   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12239  12239   
 12240  12240   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12241  12241   # Save the log message, to keep $0 and so on meaningful, and to
 12242  12242   # report actual input values of CONFIG_FILES etc. instead of their
 12243  12243   # values after options handling.
 12244  12244   ac_log="
 12245         -This file was extended by sqlite $as_me 3.23.1, which was
        12245  +This file was extended by sqlite $as_me 3.24.0, which was
 12246  12246   generated by GNU Autoconf 2.69.  Invocation command line was
 12247  12247   
 12248  12248     CONFIG_FILES    = $CONFIG_FILES
 12249  12249     CONFIG_HEADERS  = $CONFIG_HEADERS
 12250  12250     CONFIG_LINKS    = $CONFIG_LINKS
 12251  12251     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12252  12252     $ $0 $@
................................................................................
 12304  12304   
 12305  12305   Report bugs to the package provider."
 12306  12306   
 12307  12307   _ACEOF
 12308  12308   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12309  12309   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12310  12310   ac_cs_version="\\
 12311         -sqlite config.status 3.23.1
        12311  +sqlite config.status 3.24.0
 12312  12312   configured by $0, generated by GNU Autoconf 2.69,
 12313  12313     with options \\"\$ac_cs_config\\"
 12314  12314   
 12315  12315   Copyright (C) 2012 Free Software Foundation, Inc.
 12316  12316   This config.status script is free software; the Free Software Foundation
 12317  12317   gives unlimited permission to copy, distribute and modify it."
 12318  12318   

Changes to doc/lemon.html.

    95     95   As of this writing, the following command-line options are supported:
    96     96   <ul>
    97     97   <li><b>-b</b>
    98     98   Show only the basis for each parser state in the report file.
    99     99   <li><b>-c</b>
   100    100   Do not compress the generated action tables.  The parser will be a
   101    101   little larger and slower, but it will detect syntax errors sooner.
          102  +<li><b>-d</b><i>directory</i>
          103  +Write all output files into <i>directory</i>.  Normally, output files
          104  +are written into the directory that contains the input grammar file.
   102    105   <li><b>-D<i>name</i></b>
   103    106   Define C preprocessor macro <i>name</i>.  This macro is usable by
   104    107   "<tt><a href='#pifdef'>%ifdef</a></tt>" and
   105    108   "<tt><a href='#pifdef'>%ifndef</a></tt>" lines
   106    109   in the grammar file.
   107    110   <li><b>-g</b>
   108    111   Do not generate a parser.  Instead write the input grammar to standard
................................................................................
   675    678   </pre></p>
   676    679   
   677    680   <p>Then the Parse() function generated will have an 4th parameter
   678    681   of type "MyStruct*" and all action routines will have access to
   679    682   a variable named "pAbc" that is the value of the 4th parameter
   680    683   in the most recent call to Parse().</p>
   681    684   
          685  +<p>The <tt>%extra_context</tt> directive works the same except that it
          686  +is passed in on the ParseAlloc() or ParseInit() routines instead of
          687  +on Parse().
          688  +
          689  +<a name='extractx'></a>
          690  +<h4>The <tt>%extra_context</tt> directive</h4>
          691  +
          692  +The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
          693  +to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
          694  +doesn't do anything itself with these extra argument, but it does
          695  +store the value make it available to C-code action routines, destructors,
          696  +and so forth.  For example, if the grammar file contains:</p>
          697  +
          698  +<p><pre>
          699  +    %extra_context { MyStruct *pAbc }
          700  +</pre></p>
          701  +
          702  +<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
          703  +of type "MyStruct*" and all action routines will have access to
          704  +a variable named "pAbc" that is the value of that 2th parameter.</p>
          705  +
          706  +<p>The <tt>%extra_argument</tt> directive works the same except that it
          707  +is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
          708  +
   682    709   <a name='pfallback'></a>
   683    710   <h4>The <tt>%fallback</tt> directive</h4>
   684    711   
   685    712   <p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
   686    713   or more tokens.  The alternative meaning is tried if the original token
   687    714   would have generated a syntax error.</p>
   688    715   

Changes to ext/expert/expert1.test.

    91     91     eval $setup
    92     92   
    93     93   
    94     94   do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
    95     95     SELECT * FROM t1
    96     96   } {
    97     97     (no new indexes)
    98         -  0|0|0|SCAN TABLE t1
           98  +  SCAN TABLE t1
    99     99   }
   100    100   
   101    101   do_setup_rec_test $tn.2 {
   102    102     CREATE TABLE t1(a, b, c);
   103    103   } {
   104    104     SELECT * FROM t1 WHERE b>?;
   105    105   } {
   106    106     CREATE INDEX t1_idx_00000062 ON t1(b);
   107         -  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
          107  +  SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
   108    108   }
   109    109   
   110    110   do_setup_rec_test $tn.3 {
   111    111     CREATE TABLE t1(a, b, c);
   112    112   } {
   113    113     SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
   114    114   } {
   115    115     CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
   116         -  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
          116  +  SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b<?)
   117    117   }
   118    118   
   119    119   do_setup_rec_test $tn.4 {
   120    120     CREATE TABLE t1(a, b, c);
   121    121   } {
   122    122     SELECT a FROM t1 ORDER BY b;
   123    123   } {
   124    124     CREATE INDEX t1_idx_00000062 ON t1(b);
   125         -  0|0|0|SCAN TABLE t1 USING INDEX t1_idx_00000062
          125  +  SCAN TABLE t1 USING INDEX t1_idx_00000062
   126    126   }
   127    127   
   128    128   do_setup_rec_test $tn.5 {
   129    129     CREATE TABLE t1(a, b, c);
   130    130   } {
   131    131     SELECT a FROM t1 WHERE a=? ORDER BY b;
   132    132   } {
   133    133     CREATE INDEX t1_idx_000123a7 ON t1(a, b);
   134         -  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
          134  +  SEARCH TABLE t1 USING COVERING INDEX t1_idx_000123a7 (a=?)
   135    135   }
   136    136   
   137    137   do_setup_rec_test $tn.6 {
   138    138     CREATE TABLE t1(a, b, c);
   139    139   } {
   140    140     SELECT min(a) FROM t1
   141    141   } {
   142    142     CREATE INDEX t1_idx_00000061 ON t1(a);
   143         -  0|0|0|SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
          143  +  SEARCH TABLE t1 USING COVERING INDEX t1_idx_00000061
   144    144   }
   145    145   
   146    146   do_setup_rec_test $tn.7 {
   147    147     CREATE TABLE t1(a, b, c);
   148    148   } {
   149    149     SELECT * FROM t1 ORDER BY a, b, c;
   150    150   } {
   151    151     CREATE INDEX t1_idx_033e95fe ON t1(a, b, c);
   152         -  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
          152  +  SCAN TABLE t1 USING COVERING INDEX t1_idx_033e95fe
   153    153   }
   154    154   
   155    155   #do_setup_rec_test $tn.1.8 {
   156    156   #  CREATE TABLE t1(a, b, c);
   157    157   #} {
   158    158   #  SELECT * FROM t1 ORDER BY a ASC, b COLLATE nocase DESC, c ASC;
   159    159   #} {
................................................................................
   163    163   
   164    164   do_setup_rec_test $tn.8.1 {
   165    165     CREATE TABLE t1(a COLLATE NOCase, b, c);
   166    166   } {
   167    167     SELECT * FROM t1 WHERE a=?
   168    168   } {
   169    169     CREATE INDEX t1_idx_00000061 ON t1(a);
   170         -  0|0|0|SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
          170  +  SEARCH TABLE t1 USING INDEX t1_idx_00000061 (a=?)
   171    171   }
   172    172   do_setup_rec_test $tn.8.2 {
   173    173     CREATE TABLE t1(a, b COLLATE nocase, c);
   174    174   } {
   175    175     SELECT * FROM t1 ORDER BY a ASC, b DESC, c ASC;
   176    176   } {
   177    177     CREATE INDEX t1_idx_5cb97285 ON t1(a, b DESC, c);
   178         -  0|0|0|SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
          178  +  SCAN TABLE t1 USING COVERING INDEX t1_idx_5cb97285
   179    179   }
   180    180   
   181    181   
   182    182   # Tables with names that require quotes.
   183    183   #
   184    184   do_setup_rec_test $tn.9.1 {
   185    185     CREATE TABLE "t t"(a, b, c);
   186    186   } {
   187    187     SELECT * FROM "t t" WHERE a=?
   188    188   } {
   189    189     CREATE INDEX 't t_idx_00000061' ON 't t'(a);
   190         -  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
          190  +  SEARCH TABLE t t USING INDEX t t_idx_00000061 (a=?) 
   191    191   }
   192    192   
   193    193   do_setup_rec_test $tn.9.2 {
   194    194     CREATE TABLE "t t"(a, b, c);
   195    195   } {
   196    196     SELECT * FROM "t t" WHERE b BETWEEN ? AND ?
   197    197   } {
   198    198     CREATE INDEX 't t_idx_00000062' ON 't t'(b);
   199         -  0|0|0|SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
          199  +  SEARCH TABLE t t USING INDEX t t_idx_00000062 (b>? AND b<?)
   200    200   }
   201    201   
   202    202   # Columns with names that require quotes.
   203    203   #
   204    204   do_setup_rec_test $tn.10.1 {
   205    205     CREATE TABLE t3(a, "b b", c);
   206    206   } {
   207    207     SELECT * FROM t3 WHERE "b b" = ?
   208    208   } {
   209    209     CREATE INDEX t3_idx_00050c52 ON t3('b b');
   210         -  0|0|0|SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
          210  +  SEARCH TABLE t3 USING INDEX t3_idx_00050c52 (b b=?)
   211    211   }
   212    212   
   213    213   do_setup_rec_test $tn.10.2 {
   214    214     CREATE TABLE t3(a, "b b", c);
   215    215   } {
   216    216     SELECT * FROM t3 ORDER BY "b b"
   217    217   } {
   218    218     CREATE INDEX t3_idx_00050c52 ON t3('b b');
   219         -  0|0|0|SCAN TABLE t3 USING INDEX t3_idx_00050c52
          219  +  SCAN TABLE t3 USING INDEX t3_idx_00050c52
   220    220   }
   221    221   
   222    222   # Transitive constraints
   223    223   #
   224    224   do_setup_rec_test $tn.11.1 {
   225    225     CREATE TABLE t5(a, b);
   226    226     CREATE TABLE t6(c, d);
   227    227   } {
   228    228     SELECT * FROM t5, t6 WHERE a=? AND b=c AND c=?
   229    229   } {
   230    230     CREATE INDEX t5_idx_000123a7 ON t5(a, b);
   231    231     CREATE INDEX t6_idx_00000063 ON t6(c);
   232         -  0|0|1|SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
   233         -  0|1|0|SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
          232  +  SEARCH TABLE t6 USING INDEX t6_idx_00000063 (c=?) 
          233  +  SEARCH TABLE t5 USING COVERING INDEX t5_idx_000123a7 (a=? AND b=?)
   234    234   }
   235    235   
   236    236   # OR terms.
   237    237   #
   238    238   do_setup_rec_test $tn.12.1 {
   239    239     CREATE TABLE t7(a, b);
   240    240   } {
   241    241     SELECT * FROM t7 WHERE a=? OR b=?
   242    242   } {
   243    243     CREATE INDEX t7_idx_00000062 ON t7(b);
   244    244     CREATE INDEX t7_idx_00000061 ON t7(a);
   245         -  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
   246         -  0|0|0|SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
          245  +  MULTI-INDEX OR
          246  +    SEARCH TABLE t7 USING INDEX t7_idx_00000061 (a=?) 
          247  +    SEARCH TABLE t7 USING INDEX t7_idx_00000062 (b=?)
   247    248   }
   248    249   
   249    250   # rowid terms.
   250    251   #
   251    252   do_setup_rec_test $tn.13.1 {
   252    253     CREATE TABLE t8(a, b);
   253    254   } {
   254    255     SELECT * FROM t8 WHERE rowid=?
   255    256   } {
   256    257     (no new indexes)
   257         -  0|0|0|SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
          258  +  SEARCH TABLE t8 USING INTEGER PRIMARY KEY (rowid=?)
   258    259   }
   259    260   do_setup_rec_test $tn.13.2 {
   260    261     CREATE TABLE t8(a, b);
   261    262   } {
   262    263     SELECT * FROM t8 ORDER BY rowid
   263    264   } {
   264    265     (no new indexes)
   265         -  0|0|0|SCAN TABLE t8
          266  +  SCAN TABLE t8
   266    267   }
   267    268   do_setup_rec_test $tn.13.3 {
   268    269     CREATE TABLE t8(a, b);
   269    270   } {
   270    271     SELECT * FROM t8 WHERE a=? ORDER BY rowid
   271    272   } {
   272    273     CREATE INDEX t8_idx_00000061 ON t8(a); 
   273         -  0|0|0|SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
          274  +  SEARCH TABLE t8 USING INDEX t8_idx_00000061 (a=?)
   274    275   }
   275    276   
   276    277   # Triggers
   277    278   #
   278    279   do_setup_rec_test $tn.14 {
   279    280     CREATE TABLE t9(a, b, c);
   280    281     CREATE TABLE t10(a, b, c);
................................................................................
   281    282     CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
   282    283       UPDATE t10 SET a=new.a WHERE b = new.b;
   283    284     END;
   284    285   } {
   285    286     INSERT INTO t9 VALUES(?, ?, ?);
   286    287   } {
   287    288     CREATE INDEX t10_idx_00000062 ON t10(b); 
   288         -  0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
          289  +  -- TRIGGER t9t
          290  +  SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
   289    291   }
   290    292   
   291    293   do_setup_rec_test $tn.15 {
   292    294     CREATE TABLE t1(a, b);
   293    295     CREATE TABLE t2(c, d);
   294    296   
   295    297     WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
................................................................................
   297    299   
   298    300     WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
   299    301     INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
   300    302   } {
   301    303     SELECT * FROM t2, t1 WHERE b=? AND d=? AND t2.rowid=t1.rowid
   302    304   } {
   303    305     CREATE INDEX t2_idx_00000064 ON t2(d);
   304         -  0|0|0|SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
   305         -  0|1|1|SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
          306  +  SEARCH TABLE t2 USING INDEX t2_idx_00000064 (d=?) 
          307  +  SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
   306    308   }
   307    309   
   308    310   do_setup_rec_test $tn.16 {
   309    311     CREATE TABLE t1(a, b);
   310    312   } {
   311    313     SELECT * FROM t1 WHERE b IS NOT NULL;
   312    314   } {
   313    315     (no new indexes)
   314         -  0|0|0|SCAN TABLE t1
          316  +  SCAN TABLE t1
   315    317   }
   316    318   
   317    319   }
   318    320   
   319    321   proc do_candidates_test {tn sql res} {
   320    322     set res [squish [string trim $res]]
   321    323   
................................................................................
   375    377     t2 t2_idx_00000063 {100 20} 
   376    378     t2 t2_idx_00000064 {100 5} 
   377    379     t2 t2_idx_0001295b {100 20 5}
   378    380   }
   379    381   
   380    382   
   381    383   finish_test
   382         -

Changes to ext/expert/sqlite3expert.c.

  1119   1119       IdxHashEntry *pEntry;
  1120   1120       sqlite3_stmt *pExplain = 0;
  1121   1121       idxHashClear(&hIdx);
  1122   1122       rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
  1123   1123           "EXPLAIN QUERY PLAN %s", pStmt->zSql
  1124   1124       );
  1125   1125       while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
  1126         -      int iSelectid = sqlite3_column_int(pExplain, 0);
  1127         -      int iOrder = sqlite3_column_int(pExplain, 1);
  1128         -      int iFrom = sqlite3_column_int(pExplain, 2);
         1126  +      /* int iId = sqlite3_column_int(pExplain, 0); */
         1127  +      /* int iParent = sqlite3_column_int(pExplain, 1); */
         1128  +      /* int iNotUsed = sqlite3_column_int(pExplain, 2); */
  1129   1129         const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
  1130   1130         int nDetail = STRLEN(zDetail);
  1131   1131         int i;
  1132   1132   
  1133   1133         for(i=0; i<nDetail; i++){
  1134   1134           const char *zIdx = 0;
  1135   1135           if( memcmp(&zDetail[i], " USING INDEX ", 13)==0 ){
................................................................................
  1148   1148               idxHashAdd(&rc, &hIdx, zSql, 0);
  1149   1149               if( rc ) goto find_indexes_out;
  1150   1150             }
  1151   1151             break;
  1152   1152           }
  1153   1153         }
  1154   1154   
  1155         -      pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", 
  1156         -          iSelectid, iOrder, iFrom, zDetail
  1157         -      );
         1155  +      pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail);
  1158   1156       }
  1159   1157   
  1160   1158       for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
  1161   1159         pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
  1162   1160       }
  1163   1161   
  1164   1162       idxFinalize(&rc, pExplain);

Changes to ext/fts3/fts3.c.

  3959   3959       ){
  3960   3960         rc = SQLITE_NOMEM;
  3961   3961       }
  3962   3962     }
  3963   3963   
  3964   3964   #ifdef SQLITE_TEST
  3965   3965     if( rc==SQLITE_OK ){
  3966         -    rc = sqlite3Fts3ExprInitTestInterface(db);
         3966  +    rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
  3967   3967     }
  3968   3968   #endif
  3969   3969   
  3970   3970     /* Create the virtual table wrapper around the hash-table and overload 
  3971   3971     ** the four scalar functions. If this is successful, register the
  3972   3972     ** module with sqlite.
  3973   3973     */

Changes to ext/fts3/fts3Int.h.

   580    580   
   581    581   /* fts3_expr.c */
   582    582   int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
   583    583     char **, int, int, int, const char *, int, Fts3Expr **, char **
   584    584   );
   585    585   void sqlite3Fts3ExprFree(Fts3Expr *);
   586    586   #ifdef SQLITE_TEST
   587         -int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
          587  +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
   588    588   int sqlite3Fts3InitTerm(sqlite3 *db);
   589    589   #endif
   590    590   
   591    591   int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
   592    592     sqlite3_tokenizer_cursor **
   593    593   );
   594    594   

Changes to ext/fts3/fts3_expr.c.

  1104   1104   ** Everything after this point is just test code.
  1105   1105   */
  1106   1106   
  1107   1107   #ifdef SQLITE_TEST
  1108   1108   
  1109   1109   #include <stdio.h>
  1110   1110   
  1111         -/*
  1112         -** Function to query the hash-table of tokenizers (see README.tokenizers).
  1113         -*/
  1114         -static int queryTestTokenizer(
  1115         -  sqlite3 *db, 
  1116         -  const char *zName,  
  1117         -  const sqlite3_tokenizer_module **pp
  1118         -){
  1119         -  int rc;
  1120         -  sqlite3_stmt *pStmt;
  1121         -  const char zSql[] = "SELECT fts3_tokenizer(?)";
  1122         -
  1123         -  *pp = 0;
  1124         -  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  1125         -  if( rc!=SQLITE_OK ){
  1126         -    return rc;
  1127         -  }
  1128         -
  1129         -  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  1130         -  if( SQLITE_ROW==sqlite3_step(pStmt) ){
  1131         -    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
  1132         -      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
  1133         -    }
  1134         -  }
  1135         -
  1136         -  return sqlite3_finalize(pStmt);
  1137         -}
  1138         -
  1139   1111   /*
  1140   1112   ** Return a pointer to a buffer containing a text representation of the
  1141   1113   ** expression passed as the first argument. The buffer is obtained from
  1142   1114   ** sqlite3_malloc(). It is the responsibility of the caller to use 
  1143   1115   ** sqlite3_free() to release the memory. If an OOM condition is encountered,
  1144   1116   ** NULL is returned.
  1145   1117   **
................................................................................
  1199   1171   ** to parse the query expression (see README.tokenizers). The second argument
  1200   1172   ** is the query expression to parse. Each subsequent argument is the name
  1201   1173   ** of a column of the fts3 table that the query expression may refer to.
  1202   1174   ** For example:
  1203   1175   **
  1204   1176   **   SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
  1205   1177   */
  1206         -static void fts3ExprTest(
         1178  +static void fts3ExprTestCommon(
         1179  +  int bRebalance,
  1207   1180     sqlite3_context *context,
  1208   1181     int argc,
  1209   1182     sqlite3_value **argv
  1210   1183   ){
  1211         -  sqlite3_tokenizer_module const *pModule = 0;
  1212   1184     sqlite3_tokenizer *pTokenizer = 0;
  1213   1185     int rc;
  1214   1186     char **azCol = 0;
  1215   1187     const char *zExpr;
  1216   1188     int nExpr;
  1217   1189     int nCol;
  1218   1190     int ii;
  1219   1191     Fts3Expr *pExpr;
  1220   1192     char *zBuf = 0;
  1221         -  sqlite3 *db = sqlite3_context_db_handle(context);
         1193  +  Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
         1194  +  const char *zTokenizer = 0;
         1195  +  char *zErr = 0;
  1222   1196   
  1223   1197     if( argc<3 ){
  1224   1198       sqlite3_result_error(context, 
  1225   1199           "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
  1226   1200       );
  1227   1201       return;
  1228   1202     }
  1229   1203   
  1230         -  rc = queryTestTokenizer(db,
  1231         -                          (const char *)sqlite3_value_text(argv[0]), &pModule);
  1232         -  if( rc==SQLITE_NOMEM ){
  1233         -    sqlite3_result_error_nomem(context);
  1234         -    goto exprtest_out;
  1235         -  }else if( !pModule ){
  1236         -    sqlite3_result_error(context, "No such tokenizer module", -1);
  1237         -    goto exprtest_out;
         1204  +  zTokenizer = (const char*)sqlite3_value_text(argv[0]);
         1205  +  rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
         1206  +  if( rc!=SQLITE_OK ){
         1207  +    if( rc==SQLITE_NOMEM ){
         1208  +      sqlite3_result_error_nomem(context);
         1209  +    }else{
         1210  +      sqlite3_result_error(context, zErr, -1);
         1211  +    }
         1212  +    sqlite3_free(zErr);
         1213  +    return;
  1238   1214     }
  1239   1215   
  1240         -  rc = pModule->xCreate(0, 0, &pTokenizer);
  1241         -  assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
  1242         -  if( rc==SQLITE_NOMEM ){
  1243         -    sqlite3_result_error_nomem(context);
  1244         -    goto exprtest_out;
  1245         -  }
  1246         -  pTokenizer->pModule = pModule;
  1247         -
  1248   1216     zExpr = (const char *)sqlite3_value_text(argv[1]);
  1249   1217     nExpr = sqlite3_value_bytes(argv[1]);
  1250   1218     nCol = argc-2;
  1251   1219     azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
  1252   1220     if( !azCol ){
  1253   1221       sqlite3_result_error_nomem(context);
  1254   1222       goto exprtest_out;
  1255   1223     }
  1256   1224     for(ii=0; ii<nCol; ii++){
  1257   1225       azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
  1258   1226     }
  1259   1227   
  1260         -  if( sqlite3_user_data(context) ){
         1228  +  if( bRebalance ){
  1261   1229       char *zDummy = 0;
  1262   1230       rc = sqlite3Fts3ExprParse(
  1263   1231           pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
  1264   1232       );
  1265   1233       assert( rc==SQLITE_OK || pExpr==0 );
  1266   1234       sqlite3_free(zDummy);
  1267   1235     }else{
................................................................................
  1279   1247       sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
  1280   1248       sqlite3_free(zBuf);
  1281   1249     }
  1282   1250   
  1283   1251     sqlite3Fts3ExprFree(pExpr);
  1284   1252   
  1285   1253   exprtest_out:
  1286         -  if( pModule && pTokenizer ){
  1287         -    rc = pModule->xDestroy(pTokenizer);
         1254  +  if( pTokenizer ){
         1255  +    rc = pTokenizer->pModule->xDestroy(pTokenizer);
  1288   1256     }
  1289   1257     sqlite3_free(azCol);
  1290   1258   }
         1259  +
         1260  +static void fts3ExprTest(
         1261  +  sqlite3_context *context,
         1262  +  int argc,
         1263  +  sqlite3_value **argv
         1264  +){
         1265  +  fts3ExprTestCommon(0, context, argc, argv);
         1266  +}
         1267  +static void fts3ExprTestRebalance(
         1268  +  sqlite3_context *context,
         1269  +  int argc,
         1270  +  sqlite3_value **argv
         1271  +){
         1272  +  fts3ExprTestCommon(1, context, argc, argv);
         1273  +}
  1291   1274   
  1292   1275   /*
  1293   1276   ** Register the query expression parser test function fts3_exprtest() 
  1294   1277   ** with database connection db. 
  1295   1278   */
  1296         -int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
         1279  +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
  1297   1280     int rc = sqlite3_create_function(
  1298         -      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
         1281  +      db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
  1299   1282     );
  1300   1283     if( rc==SQLITE_OK ){
  1301   1284       rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", 
  1302         -        -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
         1285  +        -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
  1303   1286       );
  1304   1287     }
  1305   1288     return rc;
  1306   1289   }
  1307   1290   
  1308   1291   #endif
  1309   1292   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

Changes to ext/fts5/test/fts5plan.test.

    25     25     CREATE TABLE t1(x, y);
    26     26     CREATE VIRTUAL TABLE f1 USING fts5(ff);
    27     27   }
    28     28   
    29     29   do_eqp_test 1.1 {
    30     30     SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
    31     31   } {
    32         -  0 0 0 {SCAN TABLE t1} 
    33         -  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
           32  +  QUERY PLAN
           33  +  |--SCAN TABLE t1
           34  +  `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
    34     35   }
    35     36   
    36     37   do_eqp_test 1.2 {
    37     38     SELECT * FROM t1, f1 WHERE f1 > t1.x
    38     39   } {
    39         -  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
    40         -  0 1 0 {SCAN TABLE t1} 
           40  +  QUERY PLAN
           41  +  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
           42  +  `--SCAN TABLE t1
    41     43   }
    42     44   
    43     45   do_eqp_test 1.3 {
    44     46     SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
    45     47   } {
    46         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
    47         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
           48  +  QUERY PLAN
           49  +  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
           50  +  `--USE TEMP B-TREE FOR ORDER BY
    48     51   }
    49     52   
    50     53   do_eqp_test 1.4 {
    51     54     SELECT * FROM f1 ORDER BY rank
    52     55   } {
    53         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
    54         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
           56  +  QUERY PLAN
           57  +  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
           58  +  `--USE TEMP B-TREE FOR ORDER BY
    55     59   }
    56     60   
    57     61   do_eqp_test 1.5 {
    58     62     SELECT * FROM f1 WHERE rank MATCH ?
    59         -} {
    60         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
    61         -}
    62         -
    63         -
    64         -
           63  +} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
    65     64   
    66     65   finish_test

Changes to ext/misc/closure.c.

   822    822   static int closureBestIndex(
   823    823     sqlite3_vtab *pTab,             /* The virtual table */
   824    824     sqlite3_index_info *pIdxInfo    /* Information about the query */
   825    825   ){
   826    826     int iPlan = 0;
   827    827     int i;
   828    828     int idx = 1;
   829         -  int seenMatch = 0;
   830    829     const struct sqlite3_index_constraint *pConstraint;
   831    830     closure_vtab *pVtab = (closure_vtab*)pTab;
   832    831     double rCost = 10000000.0;
   833    832   
   834    833     pConstraint = pIdxInfo->aConstraint;
   835    834     for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
   836         -    if( pConstraint->iColumn==CLOSURE_COL_ROOT
   837         -     && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
   838         -      seenMatch = 1;
   839         -    }
   840    835       if( pConstraint->usable==0 ) continue;
   841    836       if( (iPlan & 1)==0 
   842    837        && pConstraint->iColumn==CLOSURE_COL_ROOT
   843    838        && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ
   844    839       ){
   845    840         iPlan |= 1;
   846    841         pIdxInfo->aConstraintUsage[i].argvIndex = 1;
................................................................................
   888    883      || (pVtab->zIdColumn==0     && (iPlan & 0x00f000)==0)
   889    884      || (pVtab->zParentColumn==0 && (iPlan & 0x0f0000)==0)
   890    885     ){
   891    886       /* All of tablename, idcolumn, and parentcolumn must be specified
   892    887       ** in either the CREATE VIRTUAL TABLE or in the WHERE clause constraints
   893    888       ** or else the result is an empty set. */
   894    889       iPlan = 0;
          890  +  }
          891  +  if( (iPlan&1)==0 ){
          892  +    /* If there is no usable "root=?" term, then set the index-type to 0.
          893  +    ** Also clear any argvIndex variables already set. This is necessary
          894  +    ** to prevent the core from throwing an "xBestIndex malfunction error"
          895  +    ** error (because the argvIndex values are not contiguously assigned
          896  +    ** starting from 1).  */
          897  +    rCost *= 1e30;
          898  +    for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
          899  +      pIdxInfo->aConstraintUsage[i].argvIndex = 0;
          900  +    }
          901  +    iPlan = 0;
   895    902     }
   896    903     pIdxInfo->idxNum = iPlan;
   897    904     if( pIdxInfo->nOrderBy==1
   898    905      && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID
   899    906      && pIdxInfo->aOrderBy[0].desc==0
   900    907     ){
   901    908       pIdxInfo->orderByConsumed = 1;
   902    909     }
   903         -  if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30;
   904    910     pIdxInfo->estimatedCost = rCost;
   905    911      
   906    912     return SQLITE_OK;
   907    913   }
   908    914   
   909    915   /*
   910    916   ** A virtual table module that implements the "transitive_closure".

Changes to ext/misc/completion.c.

    58     58   struct completion_cursor {
    59     59     sqlite3_vtab_cursor base;  /* Base class - must be first */
    60     60     sqlite3 *db;               /* Database connection for this cursor */
    61     61     int nPrefix, nLine;        /* Number of bytes in zPrefix and zLine */
    62     62     char *zPrefix;             /* The prefix for the word we want to complete */
    63     63     char *zLine;               /* The whole that we want to complete */
    64     64     const char *zCurrentRow;   /* Current output row */
           65  +  int szRow;                 /* Length of the zCurrentRow string */
    65     66     sqlite3_stmt *pStmt;       /* Current statement */
    66     67     sqlite3_int64 iRowid;      /* The rowid */
    67     68     int ePhase;                /* Current phase */
    68     69     int j;                     /* inter-phase counter */
    69     70   };
    70     71   
    71     72   /* Values for ePhase:
................................................................................
   170    171   */
   171    172   static int completionClose(sqlite3_vtab_cursor *cur){
   172    173     completionCursorReset((completion_cursor*)cur);
   173    174     sqlite3_free(cur);
   174    175     return SQLITE_OK;
   175    176   }
   176    177   
   177         -/*
   178         -** All SQL keywords understood by SQLite
   179         -*/
   180         -static const char *completionKwrds[] = {
   181         -  "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
   182         -  "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
   183         -  "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
   184         -  "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
   185         -  "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
   186         -  "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
   187         -  "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
   188         -  "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
   189         -  "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
   190         -  "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
   191         -  "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
   192         -  "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
   193         -  "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
   194         -  "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
   195         -  "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
   196         -  "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
   197         -  "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
   198         -  "WITH", "WITHOUT",
   199         -};
   200         -#define completionKwCount \
   201         -   (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0]))
   202         -
   203    178   /*
   204    179   ** Advance a completion_cursor to its next row of output.
   205    180   **
   206    181   ** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object
   207    182   ** record the current state of the scan.  This routine sets ->zCurrentRow
   208    183   ** to the current row of output and then returns.  If no more rows remain,
   209    184   ** then ->ePhase is set to COMPLETION_EOF which will signal the virtual
................................................................................
   218    193     completion_cursor *pCur = (completion_cursor*)cur;
   219    194     int eNextPhase = 0;  /* Next phase to try if current phase reaches end */
   220    195     int iCol = -1;       /* If >=0, step pCur->pStmt and use the i-th column */
   221    196     pCur->iRowid++;
   222    197     while( pCur->ePhase!=COMPLETION_EOF ){
   223    198       switch( pCur->ePhase ){
   224    199         case COMPLETION_KEYWORDS: {
   225         -        if( pCur->j >= completionKwCount ){
          200  +        if( pCur->j >= sqlite3_keyword_count() ){
   226    201             pCur->zCurrentRow = 0;
   227    202             pCur->ePhase = COMPLETION_DATABASES;
   228    203           }else{
   229         -          pCur->zCurrentRow = completionKwrds[pCur->j++];
          204  +          sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow);
   230    205           }
   231    206           iCol = -1;
   232    207           break;
   233    208         }
   234    209         case COMPLETION_DATABASES: {
   235    210           if( pCur->pStmt==0 ){
   236    211             sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1,
................................................................................
   294    269       if( iCol<0 ){
   295    270         /* This case is when the phase presets zCurrentRow */
   296    271         if( pCur->zCurrentRow==0 ) continue;
   297    272       }else{
   298    273         if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){
   299    274           /* Extract the next row of content */
   300    275           pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol);
          276  +        pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol);
   301    277         }else{
   302    278           /* When all rows are finished, advance to the next phase */
   303    279           sqlite3_finalize(pCur->pStmt);
   304    280           pCur->pStmt = 0;
   305    281           pCur->ePhase = eNextPhase;
   306    282           continue;
   307    283         }
   308    284       }
   309    285       if( pCur->nPrefix==0 ) break;
   310         -    if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){
          286  +    if( pCur->nPrefix<=pCur->szRow
          287  +     && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0
          288  +    ){
   311    289         break;
   312    290       }
   313    291     }
   314    292   
   315    293     return SQLITE_OK;
   316    294   }
   317    295   
................................................................................
   323    301     sqlite3_vtab_cursor *cur,   /* The cursor */
   324    302     sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
   325    303     int i                       /* Which column to return */
   326    304   ){
   327    305     completion_cursor *pCur = (completion_cursor*)cur;
   328    306     switch( i ){
   329    307       case COMPLETION_COLUMN_CANDIDATE: {
   330         -      sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT);
          308  +      sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT);
   331    309         break;
   332    310       }
   333    311       case COMPLETION_COLUMN_PREFIX: {
   334    312         sqlite3_result_text(ctx, pCur->zPrefix, -1, SQLITE_TRANSIENT);
   335    313         break;
   336    314       }
   337    315       case COMPLETION_COLUMN_WHOLELINE: {

Changes to ext/misc/csv.c.

   128    128       p->zIn = sqlite3_malloc( CSV_INBUFSZ );
   129    129       if( p->zIn==0 ){
   130    130         csv_errmsg(p, "out of memory");
   131    131         return 1;
   132    132       }
   133    133       p->in = fopen(zFilename, "rb");
   134    134       if( p->in==0 ){
          135  +      sqlite3_free(p->zIn);
   135    136         csv_reader_reset(p);
   136    137         csv_errmsg(p, "cannot open '%s' for reading", zFilename);
   137    138         return 1;
   138    139       }
   139    140     }else{
   140    141       assert( p->in==0 );
   141    142       p->zIn = (char*)zData;

Changes to ext/misc/dbdump.c.

   137    137   ** because it contains non-alphanumeric characters, or because it is an
   138    138   ** SQLite keyword.  Be conservative in this estimate:  When in doubt assume
   139    139   ** that quoting is required.
   140    140   **
   141    141   ** Return '"' if quoting is required.  Return 0 if no quoting is required.
   142    142   */
   143    143   static char quoteChar(const char *zName){
   144         -  /* All SQLite keywords, in alphabetical order */
   145         -  static const char *azKeywords[] = {
   146         -    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
   147         -    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
   148         -    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
   149         -    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
   150         -    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
   151         -    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
   152         -    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
   153         -    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
   154         -    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
   155         -    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
   156         -    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
   157         -    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
   158         -    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
   159         -    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
   160         -    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
   161         -    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
   162         -    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
   163         -    "WITH", "WITHOUT",
   164         -  };
   165         -  int i, lwr, upr, mid, c;
          144  +  int i;
   166    145     if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
   167    146     for(i=0; zName[i]; i++){
   168    147       if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
   169    148     }
   170         -  lwr = 0;
   171         -  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
   172         -  while( lwr<=upr ){
   173         -    mid = (lwr+upr)/2;
   174         -    c = sqlite3_stricmp(azKeywords[mid], zName);
   175         -    if( c==0 ) return '"';
   176         -    if( c<0 ){
   177         -      lwr = mid+1;
   178         -    }else{
   179         -      upr = mid-1;
   180         -    }
   181         -  }
   182         -  return 0;
          149  +  return sqlite3_keyword_check(zName, i) ? '"' : 0;
   183    150   }
   184    151   
   185    152   
   186    153   /*
   187    154   ** Release memory previously allocated by tableColumnList().
   188    155   */
   189    156   static void freeColumnList(char **azCol){

Added ext/misc/templatevtab.c.

            1  +/*
            2  +** 2018-04-19
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file implements a template virtual-table.
           14  +** Developers can make a copy of this file as a baseline for writing
           15  +** new virtual tables and/or table-valued functions.
           16  +**
           17  +** Steps for writing a new virtual table implementation:
           18  +**
           19  +**     (1)  Make a copy of this file.  Perhaps call it "mynewvtab.c"
           20  +**
           21  +**     (2)  Replace this header comment with something appropriate for
           22  +**          the new virtual table
           23  +**
           24  +**     (3)  Change every occurrence of "templatevtab" to some other string
           25  +**          appropriate for the new virtual table.  Ideally, the new string
           26  +**          should be the basename of the source file: "mynewvtab".  Also
           27  +**          globally change "TEMPLATEVTAB" to "MYNEWVTAB".
           28  +**
           29  +**     (4)  Run a test compilation to make sure the unmodified virtual
           30  +**          table works.
           31  +**
           32  +**     (5)  Begin making incremental changes, testing as you go, to evolve
           33  +**          the new virtual table to do what you want it to do.
           34  +**
           35  +** This template is minimal, in the sense that it uses only the required
           36  +** methods on the sqlite3_module object.  As a result, templatevtab is
           37  +** a read-only and eponymous-only table.  Those limitation can be removed
           38  +** by adding new methods.
           39  +**
           40  +** This template implements an eponymous-only virtual table with a rowid and
           41  +** two columns named "a" and "b".  The table as 10 rows with fixed integer
           42  +** values. Usage example:
           43  +**
           44  +**     SELECT rowid, a, b FROM templatevtab;
           45  +*/
           46  +#if !defined(SQLITEINT_H)
           47  +#include "sqlite3ext.h"
           48  +#endif
           49  +SQLITE_EXTENSION_INIT1
           50  +#include <string.h>
           51  +#include <assert.h>
           52  +
           53  +/* templatevtab_vtab is a subclass of sqlite3_vtab which is
           54  +** underlying representation of the virtual table
           55  +*/
           56  +typedef struct templatevtab_vtab templatevtab_vtab;
           57  +struct templatevtab_vtab {
           58  +  sqlite3_vtab base;  /* Base class - must be first */
           59  +  /* Add new fields here, as necessary */
           60  +};
           61  +
           62  +/* templatevtab_cursor is a subclass of sqlite3_vtab_cursor which will
           63  +** serve as the underlying representation of a cursor that scans
           64  +** over rows of the result
           65  +*/
           66  +typedef struct templatevtab_cursor templatevtab_cursor;
           67  +struct templatevtab_cursor {
           68  +  sqlite3_vtab_cursor base;  /* Base class - must be first */
           69  +  /* Insert new fields here.  For this templatevtab we only keep track
           70  +  ** of the rowid */
           71  +  sqlite3_int64 iRowid;      /* The rowid */
           72  +};
           73  +
           74  +/*
           75  +** The templatevtabConnect() method is invoked to create a new
           76  +** template virtual table.
           77  +**
           78  +** Think of this routine as the constructor for templatevtab_vtab objects.
           79  +**
           80  +** All this routine needs to do is:
           81  +**
           82  +**    (1) Allocate the templatevtab_vtab object and initialize all fields.
           83  +**
           84  +**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
           85  +**        result set of queries against the virtual table will look like.
           86  +*/
           87  +static int templatevtabConnect(
           88  +  sqlite3 *db,
           89  +  void *pAux,
           90  +  int argc, const char *const*argv,
           91  +  sqlite3_vtab **ppVtab,
           92  +  char **pzErr
           93  +){
           94  +  templatevtab_vtab *pNew;
           95  +  int rc;
           96  +
           97  +  rc = sqlite3_declare_vtab(db,
           98  +           "CREATE TABLE x(a,b)"
           99  +       );
          100  +  /* For convenience, define symbolic names for the index to each column. */
          101  +#define TEMPLATEVTAB_A  0
          102  +#define TEMPLATEVTAB_B  1
          103  +  if( rc==SQLITE_OK ){
          104  +    pNew = sqlite3_malloc( sizeof(*pNew) );
          105  +    *ppVtab = (sqlite3_vtab*)pNew;
          106  +    if( pNew==0 ) return SQLITE_NOMEM;
          107  +    memset(pNew, 0, sizeof(*pNew));
          108  +  }
          109  +  return rc;
          110  +}
          111  +
          112  +/*
          113  +** This method is the destructor for templatevtab_vtab objects.
          114  +*/
          115  +static int templatevtabDisconnect(sqlite3_vtab *pVtab){
          116  +  templatevtab_vtab *p = (templatevtab_vtab*)pVtab;
          117  +  sqlite3_free(p);
          118  +  return SQLITE_OK;
          119  +}
          120  +
          121  +/*
          122  +** Constructor for a new templatevtab_cursor object.
          123  +*/
          124  +static int templatevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
          125  +  templatevtab_cursor *pCur;
          126  +  pCur = sqlite3_malloc( sizeof(*pCur) );
          127  +  if( pCur==0 ) return SQLITE_NOMEM;
          128  +  memset(pCur, 0, sizeof(*pCur));
          129  +  *ppCursor = &pCur->base;
          130  +  return SQLITE_OK;
          131  +}
          132  +
          133  +/*
          134  +** Destructor for a templatevtab_cursor.
          135  +*/
          136  +static int templatevtabClose(sqlite3_vtab_cursor *cur){
          137  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          138  +  sqlite3_free(pCur);
          139  +  return SQLITE_OK;
          140  +}
          141  +
          142  +
          143  +/*
          144  +** Advance a templatevtab_cursor to its next row of output.
          145  +*/
          146  +static int templatevtabNext(sqlite3_vtab_cursor *cur){
          147  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          148  +  pCur->iRowid++;
          149  +  return SQLITE_OK;
          150  +}
          151  +
          152  +/*
          153  +** Return values of columns for the row at which the templatevtab_cursor
          154  +** is currently pointing.
          155  +*/
          156  +static int templatevtabColumn(
          157  +  sqlite3_vtab_cursor *cur,   /* The cursor */
          158  +  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
          159  +  int i                       /* Which column to return */
          160  +){
          161  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          162  +  switch( i ){
          163  +    case TEMPLATEVTAB_A:
          164  +      sqlite3_result_int(ctx, 1000 + pCur->iRowid);
          165  +      break;
          166  +    default:
          167  +      assert( i==TEMPLATEVTAB_B );
          168  +      sqlite3_result_int(ctx, 2000 + pCur->iRowid);
          169  +      break;
          170  +  }
          171  +  return SQLITE_OK;
          172  +}
          173  +
          174  +/*
          175  +** Return the rowid for the current row.  In this implementation, the
          176  +** rowid is the same as the output value.
          177  +*/
          178  +static int templatevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          179  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          180  +  *pRowid = pCur->iRowid;
          181  +  return SQLITE_OK;
          182  +}
          183  +
          184  +/*
          185  +** Return TRUE if the cursor has been moved off of the last
          186  +** row of output.
          187  +*/
          188  +static int templatevtabEof(sqlite3_vtab_cursor *cur){
          189  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          190  +  return pCur->iRowid>=10;
          191  +}
          192  +
          193  +/*
          194  +** This method is called to "rewind" the templatevtab_cursor object back
          195  +** to the first row of output.  This method is always called at least
          196  +** once prior to any call to templatevtabColumn() or templatevtabRowid() or 
          197  +** templatevtabEof().
          198  +*/
          199  +static int templatevtabFilter(
          200  +  sqlite3_vtab_cursor *pVtabCursor, 
          201  +  int idxNum, const char *idxStr,
          202  +  int argc, sqlite3_value **argv
          203  +){
          204  +  templatevtab_cursor *pCur = (templatevtab_cursor *)pVtabCursor;
          205  +  pCur->iRowid = 1;
          206  +  return SQLITE_OK;
          207  +}
          208  +
          209  +/*
          210  +** SQLite will invoke this method one or more times while planning a query
          211  +** that uses the virtual table.  This routine needs to create
          212  +** a query plan for each invocation and compute an estimated cost for that
          213  +** plan.
          214  +*/
          215  +static int templatevtabBestIndex(
          216  +  sqlite3_vtab *tab,
          217  +  sqlite3_index_info *pIdxInfo
          218  +){
          219  +  pIdxInfo->estimatedCost = (double)10;
          220  +  pIdxInfo->estimatedRows = 10;
          221  +  return SQLITE_OK;
          222  +}
          223  +
          224  +/*
          225  +** This following structure defines all the methods for the 
          226  +** virtual table.
          227  +*/
          228  +static sqlite3_module templatevtabModule = {
          229  +  /* iVersion    */ 0,
          230  +  /* xCreate     */ 0,
          231  +  /* xConnect    */ templatevtabConnect,
          232  +  /* xBestIndex  */ templatevtabBestIndex,
          233  +  /* xDisconnect */ templatevtabDisconnect,
          234  +  /* xDestroy    */ 0,
          235  +  /* xOpen       */ templatevtabOpen,
          236  +  /* xClose      */ templatevtabClose,
          237  +  /* xFilter     */ templatevtabFilter,
          238  +  /* xNext       */ templatevtabNext,
          239  +  /* xEof        */ templatevtabEof,
          240  +  /* xColumn     */ templatevtabColumn,
          241  +  /* xRowid      */ templatevtabRowid,
          242  +  /* xUpdate     */ 0,
          243  +  /* xBegin      */ 0,
          244  +  /* xSync       */ 0,
          245  +  /* xCommit     */ 0,
          246  +  /* xRollback   */ 0,
          247  +  /* xFindMethod */ 0,
          248  +  /* xRename     */ 0,
          249  +  /* xSavepoint  */ 0,
          250  +  /* xRelease    */ 0,
          251  +  /* xRollbackTo */ 0
          252  +};
          253  +
          254  +
          255  +#ifdef _WIN32
          256  +__declspec(dllexport)
          257  +#endif
          258  +int sqlite3_templatevtab_init(
          259  +  sqlite3 *db, 
          260  +  char **pzErrMsg, 
          261  +  const sqlite3_api_routines *pApi
          262  +){
          263  +  int rc = SQLITE_OK;
          264  +  SQLITE_EXTENSION_INIT2(pApi);
          265  +  rc = sqlite3_create_module(db, "templatevtab", &templatevtabModule, 0);
          266  +  return rc;
          267  +}

Changes to ext/rbu/rbu.c.

    25     25   void usage(const char *zArgv0){
    26     26     fprintf(stderr, 
    27     27   "Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n"
    28     28   "\n"
    29     29   "Where options are:\n"
    30     30   "\n"
    31     31   "    -step NSTEP\n"
           32  +"    -statstep NSTATSTEP\n"
    32     33   "    -vacuum\n"
           34  +"    -presql SQL\n"
    33     35   "\n"
    34     36   "  If the -vacuum switch is not present, argument RBU-DB must be an RBU\n"
    35     37   "  database containing an update suitable for target database TARGET-DB.\n"
    36     38   "  Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n"
    37     39   "  RBU, and RBU-DB is used as the state database for the vacuum (refer to\n"
    38     40   "  API documentation for details).\n"
    39     41   "\n"
................................................................................
    75     77     int i;
    76     78     const char *zTarget;            /* Target database to apply RBU to */
    77     79     const char *zRbu;               /* Database containing RBU */
    78     80     char zBuf[200];                 /* Buffer for printf() */
    79     81     char *zErrmsg;                  /* Error message, if any */
    80     82     sqlite3rbu *pRbu;               /* RBU handle */
    81     83     int nStep = 0;                  /* Maximum number of step() calls */
           84  +  int nStatStep = 0;              /* Report stats after this many step calls */
    82     85     int bVacuum = 0;
           86  +  const char *zPreSql = 0;
    83     87     int rc;
    84     88     sqlite3_int64 nProgress = 0;
    85     89     int nArgc = argc-2;
    86     90   
    87     91     if( argc<3 ) usage(argv[0]);
    88     92     for(i=1; i<nArgc; i++){
    89     93       const char *zArg = argv[i];
    90     94       int nArg = strlen(zArg);
    91     95       if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
    92     96         bVacuum = 1;
           97  +    }else if( nArg>1 && nArg<=7 
           98  +           && 0==memcmp(zArg, "-presql", nArg) && i<nArg-1 ){
           99  +      i++;
          100  +      zPreSql = argv[i];
    93    101       }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
    94    102         i++;
    95    103         nStep = atoi(argv[i]);
          104  +    }else if( nArg>1 && nArg<=9 
          105  +           && 0==memcmp(zArg, "-statstep", nArg) && i<nArg-1 
          106  +    ){
          107  +      i++;
          108  +      nStatStep = atoi(argv[i]);
    96    109       }else{
    97    110         usage(argv[0]);
    98    111       }
    99    112     }
   100    113   
   101    114     zTarget = argv[argc-2];
   102    115     zRbu = argv[argc-1];
................................................................................
   107    120     ** regular RBU update handle otherwise.  */
   108    121     if( bVacuum ){
   109    122       pRbu = sqlite3rbu_vacuum(zTarget, zRbu);
   110    123     }else{
   111    124       pRbu = sqlite3rbu_open(zTarget, zRbu, 0);
   112    125     }
   113    126     report_rbu_vfs(pRbu);
          127  +
          128  +  if( zPreSql && pRbu ){
          129  +    sqlite3 *db = sqlite3rbu_db(pRbu, 0);
          130  +    rc = sqlite3_exec(db, zPreSql, 0, 0, 0);
          131  +    if( rc==SQLITE_OK ){
          132  +      sqlite3 *db = sqlite3rbu_db(pRbu, 1);
          133  +      rc = sqlite3_exec(db, zPreSql, 0, 0, 0);
          134  +    }
          135  +  }
   114    136   
   115    137     /* If nStep is less than or equal to zero, call
   116    138     ** sqlite3rbu_step() until either the RBU has been completely applied
   117    139     ** or an error occurs. Or, if nStep is greater than zero, call
   118    140     ** sqlite3rbu_step() a maximum of nStep times.  */
   119         -  for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++);
   120         -  nProgress = sqlite3rbu_progress(pRbu);
   121         -  rc = sqlite3rbu_close(pRbu, &zErrmsg);
          141  +  if( rc==SQLITE_OK ){
          142  +    for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++){
          143  +      if( nStatStep>0 && (i % nStatStep)==0 ){
          144  +        sqlite3_int64 nUsed;
          145  +        sqlite3_int64 nHighwater;
          146  +        sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0);
          147  +        fprintf(stdout, "memory used=%lld highwater=%lld", nUsed, nHighwater);
          148  +        if( bVacuum==0 ){
          149  +          int one;
          150  +          int two;
          151  +          sqlite3rbu_bp_progress(pRbu, &one, &two);
          152  +          fprintf(stdout, "  progress=%d/%d\n", one, two);
          153  +        }else{
          154  +          fprintf(stdout, "\n");
          155  +        }
          156  +        fflush(stdout);
          157  +      }
          158  +    }
          159  +    nProgress = sqlite3rbu_progress(pRbu);
          160  +    rc = sqlite3rbu_close(pRbu, &zErrmsg);
          161  +  }
   122    162   
   123    163     /* Let the user know what happened. */
   124    164     switch( rc ){
   125    165       case SQLITE_OK:
   126    166         sqlite3_snprintf(sizeof(zBuf), zBuf,
   127    167             "SQLITE_OK: rbu update incomplete (%lld operations so far)\n",
   128    168             nProgress

Changes to ext/rbu/rbu1.test.

   135    135   } {
   136    136   
   137    137     eval $create_vfs
   138    138   
   139    139     foreach {tn2 cmd} {
   140    140         1 run_rbu 
   141    141         2 step_rbu 3 step_rbu_uri 4 step_rbu_state
          142  +      5 step_rbu_legacy
   142    143     } {
   143    144       foreach {tn schema} {
   144    145         1 {
   145    146           CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
   146    147         }
   147    148         2 { 
   148    149           CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);

Changes to ext/rbu/rbu_common.tcl.

    65     65       set rc [rbu step]
    66     66       check_poststep_state $rc $target $state
    67     67       rbu close
    68     68       if {$rc != "SQLITE_OK"} break
    69     69     }
    70     70     set rc
    71     71   }
           72  +
           73  +proc step_rbu_legacy {target rbu} {
           74  +  while 1 {
           75  +    sqlite3rbu rbu $target $rbu
           76  +    set state [rbu state]
           77  +    check_prestep_state $target $state
           78  +    set rc [rbu step]
           79  +    check_poststep_state $rc $target $state
           80  +    rbu close
           81  +    if {$rc != "SQLITE_OK"} break
           82  +    sqlite3 tmpdb $rbu
           83  +    tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
           84  +    tmpdb close
           85  +  }
           86  +  set rc
           87  +}
    72     88   
    73     89   proc do_rbu_vacuum_test {tn step} {
    74     90     forcedelete state.db
    75     91     uplevel [list do_test $tn.1 {
    76     92       if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
    77     93       while 1 {
    78     94         if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }

Added ext/rbu/rbusplit.test.

            1  +# 2018 April 28
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +#
           13  +
           14  +source [file join [file dirname [info script]] rbu_common.tcl]
           15  +set ::testprefix rbusplit
           16  +
           17  +db close
           18  +sqlite3_shutdown
           19  +sqlite3_config_uri 1
           20  +
           21  +autoinstall_test_functions
           22  +
           23  +proc build_db {db} {
           24  +  $db eval {
           25  +    CREATE TABLE t1(a PRIMARY KEY, b, c);
           26  +    CREATE TABLE t2(a PRIMARY KEY, b, c);
           27  +
           28  +    CREATE INDEX t1c ON t1(c);
           29  +  }
           30  +}
           31  +
           32  +proc build_rbu {filename} {
           33  +  forcedelete $filename
           34  +  sqlite3 dbRbu $filename
           35  +  dbRbu eval {
           36  +    CREATE TABLE data0_t1(a, b, c, rbu_control);
           37  +    CREATE TABLE data1_t1(a, b, c, rbu_control);
           38  +    CREATE TABLE data2_t1(a, b, c, rbu_control);
           39  +    CREATE TABLE data3_t1(a, b, c, rbu_control);
           40  +
           41  +    CREATE TABLE data_t2(a, b, c, rbu_control);
           42  +
           43  +    INSERT INTO data0_t1 VALUES(1, 1, 1, 0);
           44  +    INSERT INTO data0_t1 VALUES(2, 2, 2, 0);
           45  +    INSERT INTO data0_t1 VALUES(3, 3, 3, 0);
           46  +    INSERT INTO data0_t1 VALUES(4, 4, 4, 0);
           47  +    INSERT INTO data1_t1 VALUES(5, 5, 5, 0);
           48  +    INSERT INTO data1_t1 VALUES(6, 6, 6, 0);
           49  +    INSERT INTO data1_t1 VALUES(7, 7, 7, 0);
           50  +    INSERT INTO data1_t1 VALUES(8, 8, 8, 0);
           51  +    INSERT INTO data3_t1 VALUES(9, 9, 9, 0);
           52  +
           53  +    INSERT INTO data_t2 VALUES(1, 1, 1, 0);
           54  +    INSERT INTO data_t2 VALUES(2, 2, 2, 0);
           55  +    INSERT INTO data_t2 VALUES(3, 3, 3, 0);
           56  +    INSERT INTO data_t2 VALUES(4, 4, 4, 0);
           57  +    INSERT INTO data_t2 VALUES(5, 5, 5, 0);
           58  +    INSERT INTO data_t2 VALUES(6, 6, 6, 0);
           59  +    INSERT INTO data_t2 VALUES(7, 7, 7, 0);
           60  +    INSERT INTO data_t2 VALUES(8, 8, 8, 0);
           61  +    INSERT INTO data_t2 VALUES(9, 9, 9, 0);
           62  +  }
           63  +  
           64  +  dbRbu close
           65  +}
           66  +
           67  +foreach {tn cmd} {
           68  +  1 run_rbu
           69  +  2 step_rbu
           70  +} {
           71  +  reset_db
           72  +  build_db db
           73  +  build_rbu testrbu.db
           74  +
           75  +  do_test 1.$tn.1 {
           76  +    $cmd test.db testrbu.db
           77  +  } {SQLITE_DONE}
           78  +  do_execsql_test 1.$tn.1 {
           79  +    SELECT * FROM t1;
           80  +  } {
           81  +    1 1 1 2 2 2 3 3 3 4 4 4
           82  +    5 5 5 6 6 6 7 7 7 8 8 8
           83  +    9 9 9
           84  +  }
           85  +  do_execsql_test 1.$tn.2 {
           86  +    SELECT * FROM t2;
           87  +  } {
           88  +    1 1 1 2 2 2 3 3 3 4 4 4
           89  +    5 5 5 6 6 6 7 7 7 8 8 8
           90  +    9 9 9
           91  +  }
           92  +}
           93  +
           94  +finish_test
           95  +

Changes to ext/rbu/sqlite3rbu.c.

   149    149   **
   150    150   ** RBU_STATE_COOKIE:
   151    151   **   Valid if STAGE==1. The current change-counter cookie value in the 
   152    152   **   target db file.
   153    153   **
   154    154   ** RBU_STATE_OALSZ:
   155    155   **   Valid if STAGE==1. The size in bytes of the *-oal file.
          156  +**
          157  +** RBU_STATE_DATATBL:
          158  +**   Only valid if STAGE==1. The RBU database name of the table 
          159  +**   currently being read.
   156    160   */
   157    161   #define RBU_STATE_STAGE        1
   158    162   #define RBU_STATE_TBL          2
   159    163   #define RBU_STATE_IDX          3
   160    164   #define RBU_STATE_ROW          4
   161    165   #define RBU_STATE_PROGRESS     5
   162    166   #define RBU_STATE_CKPT         6
   163    167   #define RBU_STATE_COOKIE       7
   164    168   #define RBU_STATE_OALSZ        8
   165    169   #define RBU_STATE_PHASEONESTEP 9
          170  +#define RBU_STATE_DATATBL     10
   166    171   
   167    172   #define RBU_STAGE_OAL         1
   168    173   #define RBU_STAGE_MOVE        2
   169    174   #define RBU_STAGE_CAPTURE     3
   170    175   #define RBU_STAGE_CKPT        4
   171    176   #define RBU_STAGE_DONE        5
   172    177   
................................................................................
   201    206   
   202    207   /*
   203    208   ** A structure to store values read from the rbu_state table in memory.
   204    209   */
   205    210   struct RbuState {
   206    211     int eStage;
   207    212     char *zTbl;
          213  +  char *zDataTbl;
   208    214     char *zIdx;
   209    215     i64 iWalCksum;
   210    216     int nRow;
   211    217     i64 nProgress;
   212    218     u32 iCookie;
   213    219     i64 iOalSz;
   214    220     i64 nPhaseOneStep;
................................................................................
  2264   2270   
  2265   2271   /*
  2266   2272   ** Free an RbuState object allocated by rbuLoadState().
  2267   2273   */
  2268   2274   static void rbuFreeState(RbuState *p){
  2269   2275     if( p ){
  2270   2276       sqlite3_free(p->zTbl);
         2277  +    sqlite3_free(p->zDataTbl);
  2271   2278       sqlite3_free(p->zIdx);
  2272   2279       sqlite3_free(p);
  2273   2280     }
  2274   2281   }
  2275   2282   
  2276   2283   /*
  2277   2284   ** Allocate an RbuState object and load the contents of the rbu_state 
................................................................................
  2333   2340         case RBU_STATE_OALSZ:
  2334   2341           pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
  2335   2342           break;
  2336   2343   
  2337   2344         case RBU_STATE_PHASEONESTEP:
  2338   2345           pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
  2339   2346           break;
         2347  +
         2348  +      case RBU_STATE_DATATBL:
         2349  +        pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
         2350  +        break;
  2340   2351   
  2341   2352         default:
  2342   2353           rc = SQLITE_CORRUPT;
  2343   2354           break;
  2344   2355       }
  2345   2356     }
  2346   2357     rc2 = sqlite3_finalize(pStmt);
................................................................................
  3108   3119             "(%d, %Q), "
  3109   3120             "(%d, %Q), "
  3110   3121             "(%d, %d), "
  3111   3122             "(%d, %d), "
  3112   3123             "(%d, %lld), "
  3113   3124             "(%d, %lld), "
  3114   3125             "(%d, %lld), "
  3115         -          "(%d, %lld) ",
         3126  +          "(%d, %lld), "
         3127  +          "(%d, %Q)  ",
  3116   3128             p->zStateDb,
  3117   3129             RBU_STATE_STAGE, eStage,
  3118   3130             RBU_STATE_TBL, p->objiter.zTbl, 
  3119   3131             RBU_STATE_IDX, p->objiter.zIdx, 
  3120   3132             RBU_STATE_ROW, p->nStep, 
  3121   3133             RBU_STATE_PROGRESS, p->nProgress,
  3122   3134             RBU_STATE_CKPT, p->iWalCksum,
  3123   3135             RBU_STATE_COOKIE, (i64)pFd->iCookie,
  3124   3136             RBU_STATE_OALSZ, p->iOalSz,
  3125         -          RBU_STATE_PHASEONESTEP, p->nPhaseOneStep
         3137  +          RBU_STATE_PHASEONESTEP, p->nPhaseOneStep,
         3138  +          RBU_STATE_DATATBL, p->objiter.zDataTbl
  3126   3139         )
  3127   3140       );
  3128   3141       assert( pInsert==0 || rc==SQLITE_OK );
  3129   3142   
  3130   3143       if( rc==SQLITE_OK ){
  3131   3144         sqlite3_step(pInsert);
  3132   3145         rc = sqlite3_finalize(pInsert);
................................................................................
  3374   3387     assert( p->rc==SQLITE_OK );
  3375   3388     if( pState->zTbl ){
  3376   3389       RbuObjIter *pIter = &p->objiter;
  3377   3390       int rc = SQLITE_OK;
  3378   3391   
  3379   3392       while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup 
  3380   3393          || rbuStrCompare(pIter->zIdx, pState->zIdx)
  3381         -       || rbuStrCompare(pIter->zTbl, pState->zTbl) 
         3394  +       || (pState->zDataTbl==0 && rbuStrCompare(pIter->zTbl, pState->zTbl))
         3395  +       || (pState->zDataTbl && rbuStrCompare(pIter->zDataTbl, pState->zDataTbl))
  3382   3396       )){
  3383   3397         rc = rbuObjIterNext(p, pIter);
  3384   3398       }
  3385   3399   
  3386   3400       if( rc==SQLITE_OK && !pIter->zTbl ){
  3387   3401         rc = SQLITE_ERROR;
  3388   3402         p->zErrmsg = sqlite3_mprintf("rbu_state mismatch error");

Changes to ext/rtree/rtree6.test.

    70     70   do_test rtree6-1.5 {
    71     71     rtree_strategy {SELECT * FROM t1,t2 WHERE k=+ii AND x1<10}
    72     72   } {C0}
    73     73   
    74     74   do_eqp_test rtree6.2.1 {
    75     75     SELECT * FROM t1,t2 WHERE k=+ii AND x1<10
    76     76   } {
    77         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} 
    78         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
           77  +  QUERY PLAN
           78  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0
           79  +  `--SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)
    79     80   }
    80     81   
    81     82   do_eqp_test rtree6.2.2 {
    82     83     SELECT * FROM t1,t2 WHERE k=ii AND x1<10
    83     84   } {
    84         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} 
    85         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
           85  +  QUERY PLAN
           86  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0
           87  +  `--SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)
    86     88   }
    87     89   
    88     90   do_eqp_test rtree6.2.3 {
    89     91     SELECT * FROM t1,t2 WHERE k=ii
    90     92   } {
    91         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} 
    92         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
           93  +  QUERY PLAN
           94  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 2:
           95  +  `--SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)
    93     96   }
    94     97   
    95     98   do_eqp_test rtree6.2.4.1 {
    96     99     SELECT * FROM t1,t2 WHERE v=+ii and x1<10 and x2>10
    97    100   } {
    98         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 
    99         -  0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)}
          101  +  QUERY PLAN
          102  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1
          103  +  `--SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)
   100    104   }
   101    105   do_eqp_test rtree6.2.4.2 {
   102    106     SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10
   103    107   } {
   104         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 
   105         -  0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)}
          108  +  QUERY PLAN
          109  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1
          110  +  `--SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)
   106    111   }
   107    112   
   108    113   do_eqp_test rtree6.2.5 {
   109    114     SELECT * FROM t1,t2 WHERE k=ii AND x1<v
   110    115   } {
   111         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} 
   112         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
          116  +  QUERY PLAN
          117  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 2:
          118  +  `--SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)
   113    119   }
   114    120   
   115    121   do_execsql_test rtree6-3.1 {
   116    122     CREATE VIRTUAL TABLE t3 USING rtree(id, x1, x2, y1, y2);
   117    123     INSERT INTO t3 VALUES(NULL, 1, 1, 2, 2);
   118    124     SELECT * FROM t3 WHERE 
   119    125       x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND 

Changes to ext/rtree/rtreeC.test.

    25     25     CREATE TABLE t(x, y);
    26     26   }
    27     27   
    28     28   do_eqp_test 1.1 {
    29     29     SELECT * FROM r_tree, t 
    30     30     WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
    31     31   } {
    32         -  0 0 1 {SCAN TABLE t}
    33         -  0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0}
           32  +  QUERY PLAN
           33  +  |--SCAN TABLE t
           34  +  `--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0
    34     35   }
    35     36   
    36     37   do_eqp_test 1.2 {
    37     38     SELECT * FROM t, r_tree
    38     39     WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
    39     40   } {
    40         -  0 0 0 {SCAN TABLE t}
    41         -  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0}
           41  +  QUERY PLAN
           42  +  |--SCAN TABLE t
           43  +  `--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0
    42     44   }
    43     45   
    44     46   do_eqp_test 1.3 {
    45     47     SELECT * FROM t, r_tree
    46     48     WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y
    47     49   } {
    48         -  0 0 0 {SCAN TABLE t}
    49         -  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0}
           50  +  QUERY PLAN
           51  +  |--SCAN TABLE t
           52  +  `--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0
    50     53   }
    51     54   
    52     55   do_eqp_test 1.5 {
    53     56     SELECT * FROM t, r_tree
    54     57   } {
    55         -  0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:}
    56         -  0 1 0 {SCAN TABLE t} 
           58  +  QUERY PLAN
           59  +  |--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:
           60  +  `--SCAN TABLE t
    57     61   }
    58     62   
    59     63   do_execsql_test 2.0 {
    60     64     INSERT INTO t VALUES(0, 0);
    61     65     INSERT INTO t VALUES(0, 1);
    62     66     INSERT INTO t VALUES(0, 2);
    63     67     INSERT INTO t VALUES(0, 3);
................................................................................
    78     82   db close
    79     83   sqlite3 db test.db
    80     84   
    81     85   do_eqp_test 2.1 {
    82     86     SELECT * FROM r_tree, t 
    83     87     WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
    84     88   } {
    85         -  0 0 1 {SCAN TABLE t}
    86         -  0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0}
           89  +  QUERY PLAN
           90  +  |--SCAN TABLE t
           91  +  `--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0
    87     92   }
    88     93   
    89     94   do_eqp_test 2.2 {
    90     95     SELECT * FROM t, r_tree
    91     96     WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y
    92     97   } {
    93         -  0 0 0 {SCAN TABLE t}
    94         -  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0}
           98  +  QUERY PLAN
           99  +  |--SCAN TABLE t
          100  +  `--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0
    95    101   }
    96    102   
    97    103   do_eqp_test 2.3 {
    98    104     SELECT * FROM t, r_tree
    99    105     WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y
   100    106   } {
   101         -  0 0 0 {SCAN TABLE t}
   102         -  0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0}
          107  +  QUERY PLAN
          108  +  |--SCAN TABLE t
          109  +  `--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0
   103    110   }
   104    111   
   105    112   do_eqp_test 2.5 {
   106    113     SELECT * FROM t, r_tree
   107    114   } {
   108         -  0 0 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:}
   109         -  0 1 0 {SCAN TABLE t} 
          115  +  QUERY PLAN
          116  +  |--SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:
          117  +  `--SCAN TABLE t
   110    118   }
   111    119   
   112    120   #-------------------------------------------------------------------------
   113    121   # Test that the special CROSS JOIN handling works with rtree tables.
   114    122   #
   115    123   do_execsql_test 3.1 {
   116    124     CREATE TABLE t1(x);
   117    125     CREATE TABLE t2(y);
   118    126     CREATE VIRTUAL TABLE t3 USING rtree(z, x1,x2, y1,y2);
   119    127   }
   120    128   
   121    129   do_eqp_test 3.2.1 { SELECT * FROM t1 CROSS JOIN t2 } {
   122         -  0 0 0 {SCAN TABLE t1} 
   123         -  0 1 1 {SCAN TABLE t2}
          130  +  QUERY PLAN
          131  +  |--SCAN TABLE t1
          132  +  `--SCAN TABLE t2
   124    133   }
   125    134   do_eqp_test 3.2.2 { SELECT * FROM t2 CROSS JOIN t1 } {
   126         -  0 0 0 {SCAN TABLE t2} 0 1 1 {SCAN TABLE t1}
          135  +  QUERY PLAN
          136  +  |--SCAN TABLE t2
          137  +  `--SCAN TABLE t1
   127    138   }
   128    139   
   129    140   do_eqp_test 3.3.1 { SELECT * FROM t1 CROSS JOIN t3 } {
   130         -  0 0 0 {SCAN TABLE t1}
   131         -  0 1 1 {SCAN TABLE t3 VIRTUAL TABLE INDEX 2:} 
          141  +  QUERY PLAN
          142  +  |--SCAN TABLE t1
          143  +  `--SCAN TABLE t3 VIRTUAL TABLE INDEX 2:
   132    144   }
   133    145   do_eqp_test 3.3.2 { SELECT * FROM t3 CROSS JOIN t1 } {
   134         -  0 0 0 {SCAN TABLE t3 VIRTUAL TABLE INDEX 2:} 
   135         -  0 1 1 {SCAN TABLE t1}
          146  +  QUERY PLAN
          147  +  |--SCAN TABLE t3 VIRTUAL TABLE INDEX 2:
          148  +  `--SCAN TABLE t1
   136    149   }
   137    150   
   138    151   #--------------------------------------------------------------------
   139    152   # Test that LEFT JOINs are not reordered if the right-hand-side is
   140    153   # a virtual table.
   141    154   #
   142    155   reset_db
................................................................................
   185    198   
   186    199   # First test a query with no ANALYZE data at all. The outer loop is
   187    200   # real table "t1".
   188    201   #
   189    202   do_eqp_test 5.2 {
   190    203     SELECT * FROM t1, rt WHERE x==id;
   191    204   } {
   192         -  0 0 0 {SCAN TABLE t1} 
   193         -  0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:}
          205  +  QUERY PLAN
          206  +  |--SCAN TABLE t1
          207  +  `--SCAN TABLE rt VIRTUAL TABLE INDEX 1:
   194    208   }
   195    209   
   196    210   # Now create enough ANALYZE data to tell SQLite that virtual table "rt"
   197    211   # contains very few rows. This causes it to move "rt" to the outer loop.
   198    212   #
   199    213   do_execsql_test 5.3 {
   200    214     ANALYZE;
................................................................................
   201    215     DELETE FROM sqlite_stat1 WHERE tbl='t1';
   202    216   }
   203    217   db close
   204    218   sqlite3 db test.db
   205    219   do_eqp_test 5.4 {
   206    220     SELECT * FROM t1, rt WHERE x==id;
   207    221   } {
   208         -  0 0 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:} 
   209         -  0 1 0 {SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (x=?)}
          222  +  QUERY PLAN
          223  +  |--SCAN TABLE rt VIRTUAL TABLE INDEX 2:
          224  +  `--SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (x=?)
   210    225   }
   211    226   
   212    227   # Delete the ANALYZE data. "t1" should be the outer loop again.
   213    228   #
   214    229   do_execsql_test 5.5 { DROP TABLE sqlite_stat1; }
   215    230   db close
   216    231   sqlite3 db test.db
   217    232   do_eqp_test 5.6 {
   218    233     SELECT * FROM t1, rt WHERE x==id;
   219    234   } {
   220         -  0 0 0 {SCAN TABLE t1} 
   221         -  0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:}
          235  +  QUERY PLAN
          236  +  |--SCAN TABLE t1
          237  +  `--SCAN TABLE rt VIRTUAL TABLE INDEX 1:
   222    238   }
   223    239   
   224    240   # This time create and attach a database that contains ANALYZE data for
   225    241   # tables of the same names as those used internally by virtual table
   226    242   # "rt". Check that the rtree module is not fooled into using this data.
   227    243   # Table "t1" should remain the outer loop.
   228    244   #
................................................................................
   237    253     db close
   238    254     sqlite3 db test.db
   239    255     execsql { ATTACH 'test.db2' AS aux; }
   240    256   } {}
   241    257   do_eqp_test 5.8 {
   242    258     SELECT * FROM t1, rt WHERE x==id;
   243    259   } {
   244         -  0 0 0 {SCAN TABLE t1} 
   245         -  0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:}
          260  +  QUERY PLAN
          261  +  |--SCAN TABLE t1
          262  +  `--SCAN TABLE rt VIRTUAL TABLE INDEX 1:
   246    263   }
   247    264   
   248    265   #--------------------------------------------------------------------
   249    266   # Test that having a second connection drop the sqlite_stat1 table
   250    267   # before it is required by rtreeConnect() does not cause problems.
   251    268   #
   252    269   ifcapable rtree {
................................................................................
   295    312     INSERT INTO rt VALUES(1, 2, 7, 12, 14);      -- Not a hit
   296    313     INSERT INTO rt VALUES(2, 2, 7, 8, 12);       -- A hit!
   297    314     INSERT INTO rt VALUES(3, 7, 11, 8, 12);      -- Not a hit!
   298    315     INSERT INTO rt VALUES(4, 5, 5, 10, 10);      -- A hit!
   299    316   
   300    317   }
   301    318   
   302         -proc do_eqp_execsql_test {tn sql res} {
   303         -  set query "EXPLAIN QUERY PLAN $sql ; $sql "
   304         -  uplevel [list do_execsql_test $tn $query $res]
          319  +proc do_eqp_execsql_test {tn sql res1 res2} {
          320  +  do_eqp_test $tn.1 $sql $res1
          321  +  do_execsql_test $tn.2 $sql $res2
   305    322   }
   306    323   
   307    324   do_eqp_execsql_test 7.1 {
   308    325     SELECT id FROM xdir, rt, ydir 
   309    326     ON (y1 BETWEEN ymin AND ymax)
   310    327     WHERE (x1 BETWEEN xmin AND xmax);
   311    328   } {
   312         -  0 0 0 {SCAN TABLE xdir} 
   313         -  0 1 2 {SCAN TABLE ydir} 
   314         -  0 2 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B2D3B0D1}
          329  +  QUERY PLAN
          330  +  |--SCAN TABLE xdir
          331  +  |--SCAN TABLE ydir
          332  +  `--SCAN TABLE rt VIRTUAL TABLE INDEX 2:B2D3B0D1
          333  +} {
   315    334     2 4
   316    335   }
   317    336   
   318    337   do_eqp_execsql_test 7.2 {
   319    338     SELECT * FROM xdir, rt LEFT JOIN ydir 
   320    339     ON (y1 BETWEEN ymin AND ymax)
   321    340     WHERE (x1 BETWEEN xmin AND xmax);
   322    341   } {
   323         -  0 0 0 {SCAN TABLE xdir} 
   324         -  0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1}
   325         -  0 2 2 {SCAN TABLE ydir} 
   326         -
          342  +  QUERY PLAN
          343  +  |--SCAN TABLE xdir
          344  +  |--SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1
          345  +  `--SCAN TABLE ydir
          346  +} {
   327    347     5 1 2 7 12 14 {}
   328    348     5 2 2 7  8 12 10
   329    349     5 4 5 5 10 10 10
   330    350   }
   331    351   
   332    352   do_eqp_execsql_test 7.3 {
   333    353     SELECT id FROM xdir, rt CROSS JOIN ydir 
   334    354     ON (y1 BETWEEN ymin AND ymax)
   335    355     WHERE (x1 BETWEEN xmin AND xmax);
   336    356   } {
   337         -  0 0 0 {SCAN TABLE xdir} 
   338         -  0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1}
   339         -  0 2 2 {SCAN TABLE ydir} 
          357  +  QUERY PLAN
          358  +  |--SCAN TABLE xdir
          359  +  |--SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1
          360  +  `--SCAN TABLE ydir
          361  +} {
   340    362     2 4
   341    363   }
   342    364   
   343    365   do_eqp_execsql_test 7.4 {
   344    366     SELECT id FROM rt, xdir CROSS JOIN ydir 
   345    367     ON (y1 BETWEEN ymin AND ymax)
   346    368     WHERE (x1 BETWEEN xmin AND xmax);
   347    369   } {
   348         -  0 0 1 {SCAN TABLE xdir} 
   349         -  0 1 0 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1}
   350         -  0 2 2 {SCAN TABLE ydir} 
          370  +  QUERY PLAN
          371  +  |--SCAN TABLE xdir
          372  +  |--SCAN TABLE rt VIRTUAL TABLE INDEX 2:B0D1
          373  +  `--SCAN TABLE ydir
          374  +} {
   351    375     2 4
   352    376   }
   353    377   
   354    378   finish_test

Changes to main.mk.

    68     68            memdb.o memjournal.o \
    69     69            mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \
    70     70            notify.o opcodes.o os.o os_unix.o os_win.o \
    71     71            pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \
    72     72            random.o resolve.o rowset.o rtree.o \
    73     73            select.o sqlite3rbu.o status.o stmt.o \
    74     74            table.o threads.o tokenize.o treeview.o trigger.o \
    75         -         update.o userauth.o util.o vacuum.o \
           75  +         update.o upsert.o userauth.o util.o vacuum.o \
    76     76            vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
    77     77   	 vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \
    78     78            utf.o vtab.o
    79     79   
    80     80   LIBOBJ += sqlite3session.o
    81     81   
    82     82   # All of the source code files.
................................................................................
   158    158     $(TOP)/src/tclsqlite.c \
   159    159     $(TOP)/src/threads.c \
   160    160     $(TOP)/src/tokenize.c \
   161    161     $(TOP)/src/treeview.c \
   162    162     $(TOP)/src/trigger.c \
   163    163     $(TOP)/src/utf.c \
   164    164     $(TOP)/src/update.c \
          165  +  $(TOP)/src/upsert.c \
   165    166     $(TOP)/src/util.c \
   166    167     $(TOP)/src/vacuum.c \
   167    168     $(TOP)/src/vdbe.c \
   168    169     $(TOP)/src/vdbe.h \
   169    170     $(TOP)/src/vdbeapi.c \
   170    171     $(TOP)/src/vdbeaux.c \
   171    172     $(TOP)/src/vdbeblob.c \

Changes to src/attach.c.

   154    154       }
   155    155       assert( pVfs );
   156    156       flags |= SQLITE_OPEN_MAIN_DB;
   157    157       rc = sqlite3BtreeOpen(pVfs, zPath, db, &pNew->pBt, 0, flags);
   158    158       sqlite3_free( zPath );
   159    159       db->nDb++;
   160    160     }
   161         -  db->skipBtreeMutex = 0;
          161  +  db->noSharedCache = 0;
   162    162     if( rc==SQLITE_CONSTRAINT ){
   163    163       rc = SQLITE_ERROR;
   164    164       zErrDyn = sqlite3MPrintf(db, "database is already attached");
   165    165     }else if( rc==SQLITE_OK ){
   166    166       Pager *pPager;
   167    167       pNew->pSchema = sqlite3SchemaGet(db, pNew->pBt);
   168    168       if( !pNew->pSchema ){
................................................................................
   226    226     ** If this fails, or if opening the file failed, then close the file and 
   227    227     ** remove the entry from the db->aDb[] array. i.e. put everything back the
   228    228     ** way we found it.
   229    229     */
   230    230     if( rc==SQLITE_OK ){
   231    231       sqlite3BtreeEnterAll(db);
   232    232       db->init.iDb = 0;
          233  +    db->mDbFlags &= ~(DBFLAG_SchemaKnownOk);
   233    234       rc = sqlite3Init(db, &zErrDyn);
   234    235       sqlite3BtreeLeaveAll(db);
   235    236       assert( zErrDyn==0 || rc!=SQLITE_OK );
   236    237     }
   237    238   #ifdef SQLITE_USER_AUTHENTICATION
   238    239     if( rc==SQLITE_OK ){
   239    240       u8 newAuth = 0;
................................................................................
   498    499         pItem->zDatabase = 0;
   499    500         pItem->pSchema = pFix->pSchema;
   500    501       }
   501    502   #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER)
   502    503       if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1;
   503    504       if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1;
   504    505   #endif
          506  +    if( pItem->fg.isTabFunc && sqlite3FixExprList(pFix, pItem->u1.pFuncArg) ){
          507  +      return 1;
          508  +    }
   505    509     }
   506    510     return 0;
   507    511   }
   508    512   #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER)
   509    513   int sqlite3FixSelect(
   510    514     DbFixer *pFix,       /* Context of the fixation */
   511    515     Select *pSelect      /* The SELECT statement to be fixed to one database */
................................................................................
   597    601       }
   598    602       if( sqlite3FixExpr(pFix, pStep->pWhere) ){
   599    603         return 1;
   600    604       }
   601    605       if( sqlite3FixExprList(pFix, pStep->pExprList) ){
   602    606         return 1;
   603    607       }
          608  +#ifndef SQLITE_OMIT_UPSERT
          609  +    if( pStep->pUpsert ){
          610  +      Upsert *pUp = pStep->pUpsert;
          611  +      if( sqlite3FixExprList(pFix, pUp->pUpsertTarget)
          612  +       || sqlite3FixExpr(pFix, pUp->pUpsertTargetWhere)
          613  +       || sqlite3FixExprList(pFix, pUp->pUpsertSet)
          614  +       || sqlite3FixExpr(pFix, pUp->pUpsertWhere)
          615  +      ){
          616  +        return 1;
          617  +      }
          618  +    }
          619  +#endif
   604    620       pStep = pStep->pNext;
   605    621     }
   606    622     return 0;
   607    623   }
   608    624   #endif

Changes to src/btmutex.c.

   191    191     for(i=0; i<db->nDb; i++){
   192    192       p = db->aDb[i].pBt;
   193    193       if( p && p->sharable ){
   194    194         sqlite3BtreeEnter(p);
   195    195         skipOk = 0;
   196    196       }
   197    197     }
   198         -  db->skipBtreeMutex = skipOk;
          198  +  db->noSharedCache = skipOk;
   199    199   }
   200    200   void sqlite3BtreeEnterAll(sqlite3 *db){
   201         -  if( db->skipBtreeMutex==0 ) btreeEnterAll(db);
          201  +  if( db->noSharedCache==0 ) btreeEnterAll(db);
   202    202   }
   203    203   static void SQLITE_NOINLINE btreeLeaveAll(sqlite3 *db){
   204    204     int i;
   205    205     Btree *p;
   206    206     assert( sqlite3_mutex_held(db->mutex) );
   207    207     for(i=0; i<db->nDb; i++){
   208    208       p = db->aDb[i].pBt;
   209    209       if( p ) sqlite3BtreeLeave(p);
   210    210     }
   211    211   }
   212    212   void sqlite3BtreeLeaveAll(sqlite3 *db){
   213         -  if( db->skipBtreeMutex==0 ) btreeLeaveAll(db);
          213  +  if( db->noSharedCache==0 ) btreeLeaveAll(db);
   214    214   }
   215    215   
   216    216   #ifndef NDEBUG
   217    217   /*
   218    218   ** Return true if the current thread holds the database connection
   219    219   ** mutex and all required BtShared mutexes.
   220    220   **

Changes to src/btree.c.

  2973   2973             pDb->safety_level | (db->flags & PAGER_FLAGS_MASK));
  2974   2974       }
  2975   2975     }
  2976   2976   }
  2977   2977   #else
  2978   2978   # define setDefaultSyncFlag(pBt,safety_level)
  2979   2979   #endif
         2980  +
         2981  +/* Forward declaration */
         2982  +static int newDatabase(BtShared*);
         2983  +
  2980   2984   
  2981   2985   /*
  2982   2986   ** Get a reference to pPage1 of the database file.  This will
  2983   2987   ** also acquire a readlock on that file.
  2984   2988   **
  2985   2989   ** SQLITE_OK is returned on success.  If the file is not a
  2986   2990   ** well-formed database file, then SQLITE_CORRUPT is returned.
................................................................................
  3004   3008     /* Do some checking to help insure the file we opened really is
  3005   3009     ** a valid database file. 
  3006   3010     */
  3007   3011     nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData);
  3008   3012     sqlite3PagerPagecount(pBt->pPager, &nPageFile);
  3009   3013     if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){
  3010   3014       nPage = nPageFile;
         3015  +  }
         3016  +  if( (pBt->db->flags & SQLITE_ResetDatabase)!=0 ){
         3017  +    nPage = 0;
  3011   3018     }
  3012   3019     if( nPage>0 ){
  3013   3020       u32 pageSize;
  3014   3021       u32 usableSize;
  3015   3022       u8 *page1 = pPage1->aData;
  3016   3023       rc = SQLITE_NOTADB;
  3017   3024       /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins

Changes to src/build.c.

   339    339   Table *sqlite3LocateTable(
   340    340     Parse *pParse,         /* context in which to report errors */
   341    341     u32 flags,             /* LOCATE_VIEW or LOCATE_NOERR */
   342    342     const char *zName,     /* Name of the table we are looking for */
   343    343     const char *zDbase     /* Name of the database.  Might be NULL */
   344    344   ){
   345    345     Table *p;
          346  +  sqlite3 *db = pParse->db;
   346    347   
   347    348     /* Read the database schema. If an error occurs, leave an error message
   348    349     ** and code in pParse and return NULL. */
   349         -  if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
          350  +  if( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 
          351  +   && SQLITE_OK!=sqlite3ReadSchema(pParse)
          352  +  ){
   350    353       return 0;
   351    354     }
   352    355   
   353         -  p = sqlite3FindTable(pParse->db, zName, zDbase);
          356  +  p = sqlite3FindTable(db, zName, zDbase);
   354    357     if( p==0 ){
   355    358       const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table";
   356    359   #ifndef SQLITE_OMIT_VIRTUALTABLE
   357         -    if( sqlite3FindDbName(pParse->db, zDbase)<1 ){
          360  +    if( sqlite3FindDbName(db, zDbase)<1 ){
   358    361         /* If zName is the not the name of a table in the schema created using
   359    362         ** CREATE, then check to see if it is the name of an virtual table that
   360    363         ** can be an eponymous virtual table. */
   361         -      Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName);
          364  +      Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName);
   362    365         if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){
   363         -        pMod = sqlite3PragmaVtabRegister(pParse->db, zName);
          366  +        pMod = sqlite3PragmaVtabRegister(db, zName);
   364    367         }
   365    368         if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){
   366    369           return pMod->pEpoTab;
   367    370         }
   368    371       }
   369    372   #endif
   370    373       if( (flags & LOCATE_NOERR)==0 ){
................................................................................
   521    524     int i;
   522    525     assert( iDb<db->nDb );
   523    526   
   524    527     if( iDb>=0 ){
   525    528       assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
   526    529       DbSetProperty(db, iDb, DB_ResetWanted);
   527    530       DbSetProperty(db, 1, DB_ResetWanted);
          531  +    db->mDbFlags &= ~DBFLAG_SchemaKnownOk;
   528    532     }
   529    533   
   530    534     if( db->nSchemaLock==0 ){
   531    535       for(i=0; i<db->nDb; i++){
   532    536         if( DbHasProperty(db, i, DB_ResetWanted) ){
   533    537           sqlite3SchemaClear(db->aDb[i].pSchema);
   534    538         }
................................................................................
   546    550     assert( db->nSchemaLock==0 );
   547    551     for(i=0; i<db->nDb; i++){
   548    552       Db *pDb = &db->aDb[i];
   549    553       if( pDb->pSchema ){
   550    554         sqlite3SchemaClear(pDb->pSchema);
   551    555       }
   552    556     }
   553         -  db->mDbFlags &= ~DBFLAG_SchemaChange;
          557  +  db->mDbFlags &= ~(DBFLAG_SchemaChange|DBFLAG_SchemaKnownOk);
   554    558     sqlite3VtabUnlockList(db);
   555    559     sqlite3BtreeLeaveAll(db);
   556    560     sqlite3CollapseDatabaseArray(db);
   557    561   }
   558    562   
   559    563   /*
   560    564   ** This routine is called when a commit occurs.
................................................................................
  1091   1095     pCol = &p->aCol[p->nCol];
  1092   1096     memset(pCol, 0, sizeof(p->aCol[0]));
  1093   1097     pCol->zName = z;
  1094   1098     sqlite3ColumnPropertiesFromName(p, pCol);
  1095   1099    
  1096   1100     if( pType->n==0 ){
  1097   1101       /* If there is no type specified, columns have the default affinity
  1098         -    ** 'BLOB'. */
         1102  +    ** 'BLOB' with a default size of 4 bytes. */
  1099   1103       pCol->affinity = SQLITE_AFF_BLOB;
  1100   1104       pCol->szEst = 1;
         1105  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1106  +    if( 4>=sqlite3GlobalConfig.szSorterRef ){
         1107  +      pCol->colFlags |= COLFLAG_SORTERREF;
         1108  +    }
         1109  +#endif
  1101   1110     }else{
  1102   1111       zType = z + sqlite3Strlen30(z) + 1;
  1103   1112       memcpy(zType, pType->z, pType->n);
  1104   1113       zType[pType->n] = 0;
  1105   1114       sqlite3Dequote(zType);
  1106         -    pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst);
         1115  +    pCol->affinity = sqlite3AffinityType(zType, pCol);
  1107   1116       pCol->colFlags |= COLFLAG_HASTYPE;
  1108   1117     }
  1109   1118     p->nCol++;
  1110   1119     pParse->constraintName.n = 0;
  1111   1120   }
  1112   1121   
  1113   1122   /*
................................................................................
  1159   1168   ** 'REAL'        | SQLITE_AFF_REAL
  1160   1169   ** 'FLOA'        | SQLITE_AFF_REAL
  1161   1170   ** 'DOUB'        | SQLITE_AFF_REAL
  1162   1171   **
  1163   1172   ** If none of the substrings in the above table are found,
  1164   1173   ** SQLITE_AFF_NUMERIC is returned.
  1165   1174   */
  1166         -char sqlite3AffinityType(const char *zIn, u8 *pszEst){
         1175  +char sqlite3AffinityType(const char *zIn, Column *pCol){
  1167   1176     u32 h = 0;
  1168   1177     char aff = SQLITE_AFF_NUMERIC;
  1169   1178     const char *zChar = 0;
  1170   1179   
  1171   1180     assert( zIn!=0 );
  1172   1181     while( zIn[0] ){
  1173   1182       h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff];
................................................................................
  1196   1205   #endif
  1197   1206       }else if( (h&0x00FFFFFF)==(('i'<<16)+('n'<<8)+'t') ){    /* INT */
  1198   1207         aff = SQLITE_AFF_INTEGER;
  1199   1208         break;
  1200   1209       }
  1201   1210     }
  1202   1211   
  1203         -  /* If pszEst is not NULL, store an estimate of the field size.  The
         1212  +  /* If pCol is not NULL, store an estimate of the field size.  The
  1204   1213     ** estimate is scaled so that the size of an integer is 1.  */
  1205         -  if( pszEst ){
  1206         -    *pszEst = 1;   /* default size is approx 4 bytes */
         1214  +  if( pCol ){
         1215  +    int v = 0;   /* default size is approx 4 bytes */
  1207   1216       if( aff<SQLITE_AFF_NUMERIC ){
  1208   1217         if( zChar ){
  1209   1218           while( zChar[0] ){
  1210   1219             if( sqlite3Isdigit(zChar[0]) ){
  1211         -            int v = 0;
         1220  +            /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */
  1212   1221               sqlite3GetInt32(zChar, &v);
  1213         -            v = v/4 + 1;
  1214         -            if( v>255 ) v = 255;
  1215         -            *pszEst = v; /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */
  1216   1222               break;
  1217   1223             }
  1218   1224             zChar++;
  1219   1225           }
  1220   1226         }else{
  1221         -        *pszEst = 5;   /* BLOB, TEXT, CLOB -> r=5  (approx 20 bytes)*/
         1227  +        v = 16;   /* BLOB, TEXT, CLOB -> r=5  (approx 20 bytes)*/
  1222   1228         }
  1223   1229       }
         1230  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1231  +    if( v>=sqlite3GlobalConfig.szSorterRef ){
         1232  +      pCol->colFlags |= COLFLAG_SORTERREF;
         1233  +    }
         1234  +#endif
         1235  +    v = v/4 + 1;
         1236  +    if( v>255 ) v = 255;
         1237  +    pCol->szEst = v;
  1224   1238     }
  1225   1239     return aff;
  1226   1240   }
  1227   1241   
  1228   1242   /*
  1229   1243   ** The expression is the default value for the most recently added column
  1230   1244   ** of the table currently under construction.
................................................................................
  3006   3020     assert( pTab!=0 );
  3007   3021     assert( pParse->nErr==0 );
  3008   3022     if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 
  3009   3023          && db->init.busy==0
  3010   3024   #if SQLITE_USER_AUTHENTICATION
  3011   3025          && sqlite3UserAuthTable(pTab->zName)==0
  3012   3026   #endif
  3013         -       && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){
         3027  +#ifdef SQLITE_ALLOW_SQLITE_MASTER_INDEX
         3028  +       && sqlite3StrICmp(&pTab->zName[7],"master")!=0
         3029  +#endif
         3030  +       && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0
         3031  + ){
  3014   3032       sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName);
  3015   3033       goto exit_create_index;
  3016   3034     }
  3017   3035   #ifndef SQLITE_OMIT_VIEW
  3018   3036     if( pTab->pSelect ){
  3019   3037       sqlite3ErrorMsg(pParse, "views may not be indexed");
  3020   3038       goto exit_create_index;

Changes to src/ctime.c.

   281    281     "ENABLE_SELECTTRACE",
   282    282   #endif
   283    283   #if SQLITE_ENABLE_SESSION
   284    284     "ENABLE_SESSION",
   285    285   #endif
   286    286   #if SQLITE_ENABLE_SNAPSHOT
   287    287     "ENABLE_SNAPSHOT",
          288  +#endif
          289  +#if SQLITE_ENABLE_SORTER_REFERENCES
          290  +  "ENABLE_SORTER_REFERENCES",
   288    291   #endif
   289    292   #if SQLITE_ENABLE_SQLLOG
   290    293     "ENABLE_SQLLOG",
   291    294   #endif
   292    295   #if defined(SQLITE_ENABLE_STAT4)
   293    296     "ENABLE_STAT4",
   294    297   #elif defined(SQLITE_ENABLE_STAT3)

Changes to src/delete.c.

   234    234     int iDataCur = 0;      /* VDBE cursor for the canonical data source */
   235    235     int iIdxCur = 0;       /* Cursor number of the first index */
   236    236     int nIdx;              /* Number of indices */
   237    237     sqlite3 *db;           /* Main database structure */
   238    238     AuthContext sContext;  /* Authorization context */
   239    239     NameContext sNC;       /* Name context to resolve expressions in */
   240    240     int iDb;               /* Database number */
   241         -  int memCnt = -1;       /* Memory cell used for change counting */
          241  +  int memCnt = 0;        /* Memory cell used for change counting */
   242    242     int rcauth;            /* Value returned by authorization callback */
   243    243     int eOnePass;          /* ONEPASS_OFF or _SINGLE or _MULTI */
   244    244     int aiCurOnePass[2];   /* The write cursors opened by WHERE_ONEPASS */
   245    245     u8 *aToOpen = 0;       /* Open cursor iTabCur+j if aToOpen[j] is true */
   246    246     Index *pPk;            /* The PRIMARY KEY index on the table */
   247    247     int iPk = 0;           /* First of nPk registers holding PRIMARY KEY value */
   248    248     i16 nPk = 1;           /* Number of columns in the PRIMARY KEY */
................................................................................
   339    339     /* Begin generating code.
   340    340     */
   341    341     v = sqlite3GetVdbe(pParse);
   342    342     if( v==0 ){
   343    343       goto delete_from_cleanup;
   344    344     }
   345    345     if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
   346         -  sqlite3BeginWriteOperation(pParse, 1, iDb);
          346  +  sqlite3BeginWriteOperation(pParse, bComplex, iDb);
   347    347   
   348    348     /* If we are trying to delete from a view, realize that view into
   349    349     ** an ephemeral table.
   350    350     */
   351    351   #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   352    352     if( isView ){
   353    353       sqlite3MaterializeView(pParse, pTab, 
................................................................................
   367    367     if( sqlite3ResolveExprNames(&sNC, pWhere) ){
   368    368       goto delete_from_cleanup;
   369    369     }
   370    370   
   371    371     /* Initialize the counter of the number of rows deleted, if
   372    372     ** we are counting rows.
   373    373     */
   374         -  if( db->flags & SQLITE_CountRows ){
          374  +  if( (db->flags & SQLITE_CountRows)!=0
          375  +   && !pParse->nested
          376  +   && !pParse->pTriggerTab
          377  +  ){
   375    378       memCnt = ++pParse->nMem;
   376    379       sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt);
   377    380     }
   378    381   
   379    382   #ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION
   380    383     /* Special case: A DELETE without a WHERE clause deletes everything.
   381    384     ** It is easier just to erase the whole table. Prior to version 3.6.5,
................................................................................
   395    398   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   396    399      && db->xPreUpdateCallback==0
   397    400   #endif
   398    401     ){
   399    402       assert( !isView );
   400    403       sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName);
   401    404       if( HasRowid(pTab) ){
   402         -      sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt,
          405  +      sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt ? memCnt : -1,
   403    406                           pTab->zName, P4_STATIC);
   404    407       }
   405    408       for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
   406    409         assert( pIdx->pSchema==pTab->pSchema );
   407    410         sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb);
   408    411       }
   409    412     }else
................................................................................
   440    443       **  ONEPASS_MULTI:  One-pass approach - any number of rows may be deleted.
   441    444       */
   442    445       pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1);
   443    446       if( pWInfo==0 ) goto delete_from_cleanup;
   444    447       eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
   445    448       assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI );
   446    449       assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF );
          450  +    if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse);
   447    451     
   448    452       /* Keep track of the number of rows to be deleted */
   449         -    if( db->flags & SQLITE_CountRows ){
          453  +    if( memCnt ){
   450    454         sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
   451    455       }
   452    456     
   453    457       /* Extract the rowid or primary key for the current row */
   454    458       if( pPk ){
   455    459         for(i=0; i<nPk; i++){
   456    460           assert( pPk->aiColumn[i]>=0 );
................................................................................
   585    589       sqlite3AutoincrementEnd(pParse);
   586    590     }
   587    591   
   588    592     /* Return the number of rows that were deleted. If this routine is 
   589    593     ** generating code because of a call to sqlite3NestedParse(), do not
   590    594     ** invoke the callback function.
   591    595     */
   592         -  if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){
          596  +  if( memCnt ){
   593    597       sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1);
   594    598       sqlite3VdbeSetNumCols(v, 1);
   595    599       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC);
   596    600     }
   597    601   
   598    602   delete_from_cleanup:
   599    603     sqlite3AuthContextPop(&sContext);

Changes to src/expr.c.

  1359   1359         }
  1360   1360       }
  1361   1361       pItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
  1362   1362       pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan);
  1363   1363       pItem->sortOrder = pOldItem->sortOrder;
  1364   1364       pItem->done = 0;
  1365   1365       pItem->bSpanIsTab = pOldItem->bSpanIsTab;
         1366  +    pItem->bSorterRef = pOldItem->bSorterRef;
  1366   1367       pItem->u = pOldItem->u;
  1367   1368     }
  1368   1369     return pNew;
  1369   1370   }
  1370   1371   
  1371   1372   /*
  1372   1373   ** If cursors, triggers, views and subqueries are all omitted from
................................................................................
  1821   1822         testcase( pExpr->op==TK_AGG_FUNCTION );
  1822   1823         testcase( pExpr->op==TK_AGG_COLUMN );
  1823   1824         if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){
  1824   1825           return WRC_Continue;
  1825   1826         }
  1826   1827         /* Fall through */
  1827   1828       case TK_IF_NULL_ROW:
         1829  +    case TK_REGISTER:
         1830  +      testcase( pExpr->op==TK_REGISTER );
  1828   1831         testcase( pExpr->op==TK_IF_NULL_ROW );
  1829   1832         pWalker->eCode = 0;
  1830   1833         return WRC_Abort;
  1831   1834       case TK_VARIABLE:
  1832   1835         if( pWalker->eCode==5 ){
  1833   1836           /* Silently convert bound parameters that appear inside of CREATE
  1834   1837           ** statements into a NULL when parsing the CREATE statement text out
................................................................................
  1838   1841           /* A bound parameter in a CREATE statement that originates from
  1839   1842           ** sqlite3_prepare() causes an error */
  1840   1843           pWalker->eCode = 0;
  1841   1844           return WRC_Abort;
  1842   1845         }
  1843   1846         /* Fall through */
  1844   1847       default:
  1845         -      testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail will disallow */
  1846         -      testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail will disallow */
         1848  +      testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail() disallows */
         1849  +      testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail() disallows */
  1847   1850         return WRC_Continue;
  1848   1851     }
  1849   1852   }
  1850   1853   static int exprIsConst(Expr *p, int initFlag, int iCur){
  1851   1854     Walker w;
  1852   1855     w.eCode = initFlag;
  1853   1856     w.xExprCallback = exprNodeIsConstant;
................................................................................
  2403   2406               if( aiMap ) aiMap[i] = j;
  2404   2407             }
  2405   2408     
  2406   2409             assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) );
  2407   2410             if( colUsed==(MASKBIT(nExpr)-1) ){
  2408   2411               /* If we reach this point, that means the index pIdx is usable */
  2409   2412               int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
  2410         -#ifndef SQLITE_OMIT_EXPLAIN
  2411         -            sqlite3VdbeAddOp4(v, OP_Explain, 0, 0, 0,
  2412         -              sqlite3MPrintf(db, "USING INDEX %s FOR IN-OPERATOR",pIdx->zName),
  2413         -              P4_DYNAMIC);
  2414         -#endif
         2413  +            ExplainQueryPlan((pParse, 0,
         2414  +                              "USING INDEX %s FOR IN-OPERATOR",pIdx->zName));
  2415   2415               sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb);
  2416   2416               sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
  2417   2417               VdbeComment((v, "%s", pIdx->zName));
  2418   2418               assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 );
  2419   2419               eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0];
  2420   2420     
  2421   2421               if( prRhsHasNull ){
................................................................................
  2602   2602     ** If all of the above are false, then we can run this code just once
  2603   2603     ** save the results, and reuse the same result on subsequent invocations.
  2604   2604     */
  2605   2605     if( !ExprHasProperty(pExpr, EP_VarSelect) ){
  2606   2606       jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
  2607   2607     }
  2608   2608   
  2609         -#ifndef SQLITE_OMIT_EXPLAIN
  2610         -  if( pParse->explain==2 ){
  2611         -    char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d",
  2612         -        jmpIfDynamic>=0?"":"CORRELATED ",
  2613         -        pExpr->op==TK_IN?"LIST":"SCALAR",
  2614         -        pParse->iNextSelectId
  2615         -    );
  2616         -    sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
  2617         -  }
  2618         -#endif
  2619         -
  2620   2609     switch( pExpr->op ){
  2621   2610       case TK_IN: {
  2622   2611         int addr;                   /* Address of OP_OpenEphemeral instruction */
  2623   2612         Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */
  2624   2613         KeyInfo *pKeyInfo = 0;      /* Key information */
  2625   2614         int nVal;                   /* Size of vector pLeft */
  2626   2615         
................................................................................
  2650   2639           **
  2651   2640           ** Generate code to write the results of the select into the temporary
  2652   2641           ** table allocated and opened above.
  2653   2642           */
  2654   2643           Select *pSelect = pExpr->x.pSelect;
  2655   2644           ExprList *pEList = pSelect->pEList;
  2656   2645   
         2646  +        ExplainQueryPlan((pParse, 1, "%sLIST SUBQUERY",
         2647  +            jmpIfDynamic>=0?"":"CORRELATED "
         2648  +        ));
  2657   2649           assert( !isRowid );
  2658   2650           /* If the LHS and RHS of the IN operator do not match, that
  2659   2651           ** error will have been caught long before we reach this point. */
  2660   2652           if( ALWAYS(pEList->nExpr==nVal) ){
  2661   2653             SelectDest dest;
  2662   2654             int i;
  2663   2655             sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable);
................................................................................
  2691   2683           ** a column, use numeric affinity.
  2692   2684           */
  2693   2685           char affinity;            /* Affinity of the LHS of the IN */
  2694   2686           int i;
  2695   2687           ExprList *pList = pExpr->x.pList;
  2696   2688           struct ExprList_item *pItem;
  2697   2689           int r1, r2, r3;
  2698         -
  2699   2690           affinity = sqlite3ExprAffinity(pLeft);
  2700   2691           if( !affinity ){
  2701   2692             affinity = SQLITE_AFF_BLOB;
  2702   2693           }
  2703   2694           if( pKeyInfo ){
  2704   2695             assert( sqlite3KeyInfoIsWriteable(pKeyInfo) );
  2705   2696             pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
................................................................................
  2772   2763   
  2773   2764         testcase( pExpr->op==TK_EXISTS );
  2774   2765         testcase( pExpr->op==TK_SELECT );
  2775   2766         assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT );
  2776   2767         assert( ExprHasProperty(pExpr, EP_xIsSelect) );
  2777   2768   
  2778   2769         pSel = pExpr->x.pSelect;
         2770  +      ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY",
         2771  +            jmpIfDynamic>=0?"":"CORRELATED "));
  2779   2772         nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1;
  2780   2773         sqlite3SelectDestInit(&dest, 0, pParse->nMem+1);
  2781   2774         pParse->nMem += nReg;
  2782   2775         if( pExpr->op==TK_SELECT ){
  2783   2776           dest.eDest = SRT_Mem;
  2784   2777           dest.iSdst = dest.iSDParm;
  2785   2778           dest.nSdst = nReg;
................................................................................
  3534   3527   
  3535   3528     assert( target>0 && target<=pParse->nMem );
  3536   3529     if( v==0 ){
  3537   3530       assert( pParse->db->mallocFailed );
  3538   3531       return 0;
  3539   3532     }
  3540   3533   
         3534  +expr_code_doover:
  3541   3535     if( pExpr==0 ){
  3542   3536       op = TK_NULL;
  3543   3537     }else{
  3544   3538       op = pExpr->op;
  3545   3539     }
  3546   3540     switch( op ){
  3547   3541       case TK_AGG_COLUMN: {
................................................................................
  3994   3988       case TK_BETWEEN: {
  3995   3989         exprCodeBetween(pParse, pExpr, target, 0, 0);
  3996   3990         return target;
  3997   3991       }
  3998   3992       case TK_SPAN:
  3999   3993       case TK_COLLATE: 
  4000   3994       case TK_UPLUS: {
  4001         -      return sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
         3995  +      pExpr = pExpr->pLeft;
         3996  +      goto expr_code_doover;
  4002   3997       }
  4003   3998   
  4004   3999       case TK_TRIGGER: {
  4005   4000         /* If the opcode is TK_TRIGGER, then the expression is a reference
  4006   4001         ** to a column in the new.* or old.* pseudo-tables available to
  4007   4002         ** trigger programs. In this case Expr.iTable is set to 1 for the
  4008   4003         ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn
................................................................................
  4032   4027   
  4033   4028         assert( pExpr->iTable==0 || pExpr->iTable==1 );
  4034   4029         assert( pExpr->iColumn>=-1 && pExpr->iColumn<pTab->nCol );
  4035   4030         assert( pTab->iPKey<0 || pExpr->iColumn!=pTab->iPKey );
  4036   4031         assert( p1>=0 && p1<(pTab->nCol*2+2) );
  4037   4032   
  4038   4033         sqlite3VdbeAddOp2(v, OP_Param, p1, target);
  4039         -      VdbeComment((v, "%s.%s -> $%d",
         4034  +      VdbeComment((v, "r[%d]=%s.%s", target,
  4040   4035           (pExpr->iTable ? "new" : "old"),
  4041         -        (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName),
  4042         -        target
         4036  +        (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName)
  4043   4037         ));
  4044   4038   
  4045   4039   #ifndef SQLITE_OMIT_FLOATING_POINT
  4046   4040         /* If the column has REAL affinity, it may currently be stored as an
  4047   4041         ** integer. Use OP_RealAffinity to make sure it is really real.
  4048   4042         **
  4049   4043         ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to
................................................................................
  4367   4361     assert( pList!=0 );
  4368   4362     assert( target>0 );
  4369   4363     assert( pParse->pVdbe!=0 );  /* Never gets this far otherwise */
  4370   4364     n = pList->nExpr;
  4371   4365     if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR;
  4372   4366     for(pItem=pList->a, i=0; i<n; i++, pItem++){
  4373   4367       Expr *pExpr = pItem->pExpr;
         4368  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         4369  +    if( pItem->bSorterRef ){
         4370  +      i--;
         4371  +      n--;
         4372  +    }else
         4373  +#endif
  4374   4374       if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){
  4375   4375         if( flags & SQLITE_ECEL_OMITREF ){
  4376   4376           i--;
  4377   4377           n--;
  4378   4378         }else{
  4379   4379           sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i);
  4380   4380         }
................................................................................
  4895   4895         return 1;
  4896   4896       }
  4897   4897       return 2;
  4898   4898     }
  4899   4899     if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){
  4900   4900       if( pA->op==TK_FUNCTION ){
  4901   4901         if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2;
         4902  +    }else if( pA->op==TK_COLLATE ){
         4903  +      if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2;
  4902   4904       }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){
  4903         -      return pA->op==TK_COLLATE ? 1 : 2;
         4905  +      return 2;
  4904   4906       }
  4905   4907     }
  4906   4908     if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2;
  4907   4909     if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){
  4908   4910       if( combinedFlags & EP_xIsSelect ) return 2;
  4909   4911       if( sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2;
  4910   4912       if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2;
  4911   4913       if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2;
  4912         -    if( ALWAYS((combinedFlags & EP_Reduced)==0) && pA->op!=TK_STRING ){
         4914  +    assert( (combinedFlags & EP_Reduced)==0 );
         4915  +    if( pA->op!=TK_STRING && pA->op!=TK_TRUEFALSE ){
  4913   4916         if( pA->iColumn!=pB->iColumn ) return 2;
  4914   4917         if( pA->iTable!=pB->iTable 
  4915   4918          && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2;
  4916   4919       }
  4917   4920     }
  4918   4921     return 0;
  4919   4922   }
................................................................................
  5251   5254   ** for additional information.
  5252   5255   */
  5253   5256   static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
  5254   5257     int i;
  5255   5258     NameContext *pNC = pWalker->u.pNC;
  5256   5259     Parse *pParse = pNC->pParse;
  5257   5260     SrcList *pSrcList = pNC->pSrcList;
  5258         -  AggInfo *pAggInfo = pNC->pAggInfo;
         5261  +  AggInfo *pAggInfo = pNC->uNC.pAggInfo;
  5259   5262   
         5263  +  assert( pNC->ncFlags & NC_UAggInfo );
  5260   5264     switch( pExpr->op ){
  5261   5265       case TK_AGG_COLUMN:
  5262   5266       case TK_COLUMN: {
  5263   5267         testcase( pExpr->op==TK_AGG_COLUMN );
  5264   5268         testcase( pExpr->op==TK_COLUMN );
  5265   5269         /* Check to see if the column is in one of the tables in the FROM
  5266   5270         ** clause of the aggregate query */

Changes to src/global.c.

   236    236      0,                         /* xVdbeBranch */
   237    237      0,                         /* pVbeBranchArg */
   238    238   #endif
   239    239   #ifndef SQLITE_UNTESTABLE
   240    240      0,                         /* xTestCallback */
   241    241   #endif
   242    242      0,                         /* bLocaltimeFault */
   243         -   0x7ffffffe                 /* iOnceResetThreshold */
          243  +   0x7ffffffe,                /* iOnceResetThreshold */
          244  +   SQLITE_DEFAULT_SORTERREF_SIZE   /* szSorterRef */
   244    245   };
   245    246   
   246    247   /*
   247    248   ** Hash table for global functions - functions common to all
   248    249   ** database connections.  After initialization, this table is
   249    250   ** read-only.
   250    251   */

Changes to src/insert.c.

   484    484   **      D: cleanup
   485    485   */
   486    486   void sqlite3Insert(
   487    487     Parse *pParse,        /* Parser context */
   488    488     SrcList *pTabList,    /* Name of table into which we are inserting */
   489    489     Select *pSelect,      /* A SELECT statement to use as the data source */
   490    490     IdList *pColumn,      /* Column names corresponding to IDLIST. */
   491         -  int onError           /* How to handle constraint errors */
          491  +  int onError,          /* How to handle constraint errors */
          492  +  Upsert *pUpsert       /* ON CONFLICT clauses for upsert, or NULL */
   492    493   ){
   493    494     sqlite3 *db;          /* The main database structure */
   494    495     Table *pTab;          /* The table to insert into.  aka TABLE */
   495    496     int i, j;             /* Loop counters */
   496    497     Vdbe *v;              /* Generate code into this virtual machine */
   497    498     Index *pIdx;          /* For looping over indices of the table */
   498    499     int nColumn;          /* Number of columns in the data */
................................................................................
   779    780     if( pColumn!=0 && nColumn!=pColumn->nId ){
   780    781       sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId);
   781    782       goto insert_cleanup;
   782    783     }
   783    784       
   784    785     /* Initialize the count of rows to be inserted
   785    786     */
   786         -  if( db->flags & SQLITE_CountRows ){
          787  +  if( (db->flags & SQLITE_CountRows)!=0
          788  +   && !pParse->nested
          789  +   && !pParse->pTriggerTab
          790  +  ){
   787    791       regRowCount = ++pParse->nMem;
   788    792       sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
   789    793     }
   790    794   
   791    795     /* If this is not a view, open the table and and all indices */
   792    796     if( !isView ){
   793    797       int nIdx;
................................................................................
   799    803       }
   800    804       for(i=0, pIdx=pTab->pIndex; i<nIdx; pIdx=pIdx->pNext, i++){
   801    805         assert( pIdx );
   802    806         aRegIdx[i] = ++pParse->nMem;
   803    807         pParse->nMem += pIdx->nColumn;
   804    808       }
   805    809     }
          810  +#ifndef SQLITE_OMIT_UPSERT
          811  +  if( pUpsert ){
          812  +    pTabList->a[0].iCursor = iDataCur;
          813  +    pUpsert->pUpsertSrc = pTabList;
          814  +    pUpsert->regData = regData;
          815  +    pUpsert->iDataCur = iDataCur;
          816  +    pUpsert->iIdxCur = iIdxCur;
          817  +    if( pUpsert->pUpsertTarget ){
          818  +      sqlite3UpsertAnalyzeTarget(pParse, pTabList, pUpsert);
          819  +    }
          820  +  }
          821  +#endif
          822  +
   806    823   
   807    824     /* This is the top of the main insertion loop */
   808    825     if( useTempTable ){
   809    826       /* This block codes the top of loop only.  The complete loop is the
   810    827       ** following pseudocode (template 4):
   811    828       **
   812    829       **         rewind temp table, if empty goto D
................................................................................
  1001   1018         sqlite3MayAbort(pParse);
  1002   1019       }else
  1003   1020   #endif
  1004   1021       {
  1005   1022         int isReplace;    /* Set to true if constraints may cause a replace */
  1006   1023         int bUseSeek;     /* True to use OPFLAG_SEEKRESULT */
  1007   1024         sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
  1008         -          regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0
         1025  +          regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert
  1009   1026         );
  1010   1027         sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0);
  1011   1028   
  1012   1029         /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE
  1013   1030         ** constraints or (b) there are no triggers and this table is not a
  1014   1031         ** parent table in a foreign key constraint. It is safe to set the
  1015   1032         ** flag in the second case as if any REPLACE constraint is hit, an
................................................................................
  1024   1041             regIns, aRegIdx, 0, appendFlag, bUseSeek
  1025   1042         );
  1026   1043       }
  1027   1044     }
  1028   1045   
  1029   1046     /* Update the count of rows that are inserted
  1030   1047     */
  1031         -  if( (db->flags & SQLITE_CountRows)!=0 ){
         1048  +  if( regRowCount ){
  1032   1049       sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1);
  1033   1050     }
  1034   1051   
  1035   1052     if( pTrigger ){
  1036   1053       /* Code AFTER triggers */
  1037   1054       sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, 
  1038   1055           pTab, regData-2-pTab->nCol, onError, endOfLoop);
................................................................................
  1061   1078     }
  1062   1079   
  1063   1080     /*
  1064   1081     ** Return the number of rows inserted. If this routine is 
  1065   1082     ** generating code because of a call to sqlite3NestedParse(), do not
  1066   1083     ** invoke the callback function.
  1067   1084     */
  1068         -  if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){
         1085  +  if( regRowCount ){
  1069   1086       sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1);
  1070   1087       sqlite3VdbeSetNumCols(v, 1);
  1071   1088       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC);
  1072   1089     }
  1073   1090   
  1074   1091   insert_cleanup:
  1075   1092     sqlite3SrcListDelete(db, pTabList);
  1076   1093     sqlite3ExprListDelete(db, pList);
         1094  +  sqlite3UpsertDelete(db, pUpsert);
  1077   1095     sqlite3SelectDelete(db, pSelect);
  1078   1096     sqlite3IdListDelete(db, pColumn);
  1079   1097     sqlite3DbFree(db, aRegIdx);
  1080   1098   }
  1081   1099   
  1082   1100   /* Make sure "isView" and other macros defined above are undefined. Otherwise
  1083   1101   ** they may interfere with compilation of other functions in this file
................................................................................
  1140   1158     }
  1141   1159     testcase( w.eCode==0 );
  1142   1160     testcase( w.eCode==CKCNSTRNT_COLUMN );
  1143   1161     testcase( w.eCode==CKCNSTRNT_ROWID );
  1144   1162     testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) );
  1145   1163     return !w.eCode;
  1146   1164   }
         1165  +
         1166  +/*
         1167  +** An instance of the ConstraintAddr object remembers the byte-code addresses
         1168  +** for sections of the constraint checks that deal with uniqueness constraints
         1169  +** on the rowid and on the upsert constraint.
         1170  +**
         1171  +** This information is passed into checkReorderConstraintChecks() to insert
         1172  +** some OP_Goto operations so that the rowid and upsert constraints occur
         1173  +** in the correct order relative to other constraints.
         1174  +*/
         1175  +typedef struct ConstraintAddr ConstraintAddr;
         1176  +struct ConstraintAddr {
         1177  +  int ipkTop;          /* Subroutine for rowid constraint check */
         1178  +  int upsertTop;       /* Label for upsert constraint check subroutine */
         1179  +  int upsertTop2;      /* Copy of upsertTop not cleared by the call */
         1180  +  int upsertBtm;       /* upsert constraint returns to this label */
         1181  +  int ipkBtm;          /* Return opcode rowid constraint check */
         1182  +};
         1183  +
         1184  +/*
         1185  +** Generate any OP_Goto operations needed to cause constraints to be
         1186  +** run that haven't already been run.
         1187  +*/
         1188  +static void reorderConstraintChecks(Vdbe *v, ConstraintAddr *p){
         1189  +  if( p->upsertTop ){
         1190  +    testcase( sqlite3VdbeLabelHasBeenResolved(v, p->upsertTop) );
         1191  +    sqlite3VdbeGoto(v, p->upsertTop);
         1192  +    VdbeComment((v, "call upsert subroutine"));
         1193  +    sqlite3VdbeResolveLabel(v, p->upsertBtm);
         1194  +    p->upsertTop = 0;
         1195  +  }
         1196  +  if( p->ipkTop ){
         1197  +    sqlite3VdbeGoto(v, p->ipkTop);
         1198  +    VdbeComment((v, "call rowid unique-check subroutine"));
         1199  +    sqlite3VdbeJumpHere(v, p->ipkBtm);
         1200  +    p->ipkTop = 0;
         1201  +  }
         1202  +}
  1147   1203   
  1148   1204   /*
  1149   1205   ** Generate code to do constraint checks prior to an INSERT or an UPDATE
  1150   1206   ** on table pTab.
  1151   1207   **
  1152   1208   ** The regNewData parameter is the first register in a range that contains
  1153   1209   ** the data to be inserted or the data after the update.  There will be
................................................................................
  1236   1292     int iIdxCur,         /* First index cursor */
  1237   1293     int regNewData,      /* First register in a range holding values to insert */
  1238   1294     int regOldData,      /* Previous content.  0 for INSERTs */
  1239   1295     u8 pkChng,           /* Non-zero if the rowid or PRIMARY KEY changed */
  1240   1296     u8 overrideError,    /* Override onError to this if not OE_Default */
  1241   1297     int ignoreDest,      /* Jump to this label on an OE_Ignore resolution */
  1242   1298     int *pbMayReplace,   /* OUT: Set to true if constraint may cause a replace */
  1243         -  int *aiChng          /* column i is unchanged if aiChng[i]<0 */
         1299  +  int *aiChng,         /* column i is unchanged if aiChng[i]<0 */
         1300  +  Upsert *pUpsert      /* ON CONFLICT clauses, if any.  NULL otherwise */
  1244   1301   ){
  1245   1302     Vdbe *v;             /* VDBE under constrution */
  1246   1303     Index *pIdx;         /* Pointer to one of the indices */
  1247   1304     Index *pPk = 0;      /* The PRIMARY KEY index */
  1248   1305     sqlite3 *db;         /* Database connection */
  1249   1306     int i;               /* loop counter */
  1250   1307     int ix;              /* Index loop counter */
  1251   1308     int nCol;            /* Number of columns */
  1252   1309     int onError;         /* Conflict resolution strategy */
  1253   1310     int addr1;           /* Address of jump instruction */
  1254   1311     int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */
  1255   1312     int nPkField;        /* Number of fields in PRIMARY KEY. 1 for ROWID tables */
  1256         -  int ipkTop = 0;      /* Top of the rowid change constraint check */
  1257         -  int ipkBottom = 0;   /* Bottom of the rowid change constraint check */
         1313  +  ConstraintAddr sAddr;/* Address information for constraint reordering */
         1314  +  Index *pUpIdx = 0;   /* Index to which to apply the upsert */
  1258   1315     u8 isUpdate;         /* True if this is an UPDATE operation */
  1259   1316     u8 bAffinityDone = 0;  /* True if the OP_Affinity operation has been run */
         1317  +  int upsertBypass = 0;  /* Address of Goto to bypass upsert subroutine */
  1260   1318   
  1261   1319     isUpdate = regOldData!=0;
  1262   1320     db = pParse->db;
  1263   1321     v = sqlite3GetVdbe(pParse);
  1264   1322     assert( v!=0 );
  1265   1323     assert( pTab->pSelect==0 );  /* This table is not a VIEW */
  1266   1324     nCol = pTab->nCol;
         1325  +  memset(&sAddr, 0, sizeof(sAddr));
  1267   1326     
  1268   1327     /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for
  1269   1328     ** normal rowid tables.  nPkField is the number of key fields in the 
  1270   1329     ** pPk index or 1 for a rowid table.  In other words, nPkField is the
  1271   1330     ** number of fields in the true primary key of the table. */
  1272   1331     if( HasRowid(pTab) ){
  1273   1332       pPk = 0;
................................................................................
  1358   1417                                 P5_ConstraintCheck);
  1359   1418         }
  1360   1419         sqlite3VdbeResolveLabel(v, allOk);
  1361   1420       }
  1362   1421       pParse->iSelfTab = 0;
  1363   1422     }
  1364   1423   #endif /* !defined(SQLITE_OMIT_CHECK) */
         1424  +
         1425  +  /* UNIQUE and PRIMARY KEY constraints should be handled in the following
         1426  +  ** order:
         1427  +  **
         1428  +  **   (1)  OE_Abort, OE_Fail, OE_Rollback, OE_Ignore
         1429  +  **   (2)  OE_Update
         1430  +  **   (3)  OE_Replace
         1431  +  **
         1432  +  ** OE_Fail and OE_Ignore must happen before any changes are made.
         1433  +  ** OE_Update guarantees that only a single row will change, so it
         1434  +  ** must happen before OE_Replace.  Technically, OE_Abort and OE_Rollback
         1435  +  ** could happen in any order, but they are grouped up front for
         1436  +  ** convenience.
         1437  +  **
         1438  +  ** Constraint checking code is generated in this order:
         1439  +  **   (A)  The rowid constraint
         1440  +  **   (B)  Unique index constraints that do not have OE_Replace as their
         1441  +  **        default conflict resolution strategy
         1442  +  **   (C)  Unique index that do use OE_Replace by default.
         1443  +  **
         1444  +  ** The ordering of (2) and (3) is accomplished by making sure the linked
         1445  +  ** list of indexes attached to a table puts all OE_Replace indexes last
         1446  +  ** in the list.  See sqlite3CreateIndex() for where that happens.
         1447  +  */
         1448  +
         1449  +  if( pUpsert ){
         1450  +    if( pUpsert->pUpsertTarget==0 ){
         1451  +      /* An ON CONFLICT DO NOTHING clause, without a constraint-target.
         1452  +      ** Make all unique constraint resolution be OE_Ignore */
         1453  +      assert( pUpsert->pUpsertSet==0 );
         1454  +      overrideError = OE_Ignore;
         1455  +      pUpsert = 0;
         1456  +    }else if( (pUpIdx = pUpsert->pUpsertIdx)!=0 ){
         1457  +      /* If the constraint-target is on some column other than
         1458  +      ** then ROWID, then we might need to move the UPSERT around
         1459  +      ** so that it occurs in the correct order. */
         1460  +      sAddr.upsertTop = sAddr.upsertTop2 = sqlite3VdbeMakeLabel(v);
         1461  +      sAddr.upsertBtm = sqlite3VdbeMakeLabel(v);
         1462  +    }
         1463  +  }
  1365   1464   
  1366   1465     /* If rowid is changing, make sure the new rowid does not previously
  1367   1466     ** exist in the table.
  1368   1467     */
  1369   1468     if( pkChng && pPk==0 ){
  1370   1469       int addrRowidOk = sqlite3VdbeMakeLabel(v);
  1371   1470   
................................................................................
  1372   1471       /* Figure out what action to take in case of a rowid collision */
  1373   1472       onError = pTab->keyConf;
  1374   1473       if( overrideError!=OE_Default ){
  1375   1474         onError = overrideError;
  1376   1475       }else if( onError==OE_Default ){
  1377   1476         onError = OE_Abort;
  1378   1477       }
         1478  +
         1479  +    /* figure out whether or not upsert applies in this case */
         1480  +    if( pUpsert && pUpsert->pUpsertIdx==0 ){
         1481  +      if( pUpsert->pUpsertSet==0 ){
         1482  +        onError = OE_Ignore;  /* DO NOTHING is the same as INSERT OR IGNORE */
         1483  +      }else{
         1484  +        onError = OE_Update;  /* DO UPDATE */
         1485  +      }
         1486  +    }
         1487  +
         1488  +    /* If the response to a rowid conflict is REPLACE but the response
         1489  +    ** to some other UNIQUE constraint is FAIL or IGNORE, then we need
         1490  +    ** to defer the running of the rowid conflict checking until after
         1491  +    ** the UNIQUE constraints have run.
         1492  +    */
         1493  +    assert( OE_Update>OE_Replace );
         1494  +    assert( OE_Ignore<OE_Replace );
         1495  +    assert( OE_Fail<OE_Replace );
         1496  +    assert( OE_Abort<OE_Replace );
         1497  +    assert( OE_Rollback<OE_Replace );
         1498  +    if( onError>=OE_Replace
         1499  +     && (pUpsert || onError!=overrideError)
         1500  +     && pTab->pIndex
         1501  +    ){
         1502  +      sAddr.ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1;
         1503  +    }
  1379   1504   
  1380   1505       if( isUpdate ){
  1381   1506         /* pkChng!=0 does not mean that the rowid has changed, only that
  1382   1507         ** it might have changed.  Skip the conflict logic below if the rowid
  1383   1508         ** is unchanged. */
  1384   1509         sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData);
  1385   1510         sqlite3VdbeChangeP5(v, SQLITE_NOTNULL);
  1386   1511         VdbeCoverage(v);
  1387   1512       }
  1388   1513   
  1389         -    /* If the response to a rowid conflict is REPLACE but the response
  1390         -    ** to some other UNIQUE constraint is FAIL or IGNORE, then we need
  1391         -    ** to defer the running of the rowid conflict checking until after
  1392         -    ** the UNIQUE constraints have run.
  1393         -    */
  1394         -    if( onError==OE_Replace && overrideError!=OE_Replace ){
  1395         -      for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
  1396         -        if( pIdx->onError==OE_Ignore || pIdx->onError==OE_Fail ){
  1397         -          ipkTop = sqlite3VdbeAddOp0(v, OP_Goto);
  1398         -          break;
  1399         -        }
  1400         -      }
  1401         -    }
  1402         -
  1403   1514       /* Check to see if the new rowid already exists in the table.  Skip
  1404   1515       ** the following conflict logic if it does not. */
         1516  +    VdbeNoopComment((v, "uniqueness check for ROWID"));
  1405   1517       sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData);
  1406   1518       VdbeCoverage(v);
  1407   1519   
  1408         -    /* Generate code that deals with a rowid collision */
  1409   1520       switch( onError ){
  1410   1521         default: {
  1411   1522           onError = OE_Abort;
  1412   1523           /* Fall thru into the next case */
  1413   1524         }
  1414   1525         case OE_Rollback:
  1415   1526         case OE_Abort:
  1416   1527         case OE_Fail: {
         1528  +        testcase( onError==OE_Rollback );
         1529  +        testcase( onError==OE_Abort );
         1530  +        testcase( onError==OE_Fail );
  1417   1531           sqlite3RowidConstraint(pParse, onError, pTab);
  1418   1532           break;
  1419   1533         }
  1420   1534         case OE_Replace: {
  1421   1535           /* If there are DELETE triggers on this table and the
  1422   1536           ** recursive-triggers flag is set, call GenerateRowDelete() to
  1423   1537           ** remove the conflicting row from the table. This will fire
................................................................................
  1446   1560           }
  1447   1561           if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){
  1448   1562             sqlite3MultiWrite(pParse);
  1449   1563             sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur,
  1450   1564                                      regNewData, 1, 0, OE_Replace, 1, -1);
  1451   1565           }else{
  1452   1566   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
  1453         -          if( HasRowid(pTab) ){
  1454         -            /* This OP_Delete opcode fires the pre-update-hook only. It does
  1455         -            ** not modify the b-tree. It is more efficient to let the coming
  1456         -            ** OP_Insert replace the existing entry than it is to delete the
  1457         -            ** existing entry and then insert a new one. */
  1458         -            sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP);
  1459         -            sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
  1460         -          }
         1567  +          assert( HasRowid(pTab) );
         1568  +          /* This OP_Delete opcode fires the pre-update-hook only. It does
         1569  +          ** not modify the b-tree. It is more efficient to let the coming
         1570  +          ** OP_Insert replace the existing entry than it is to delete the
         1571  +          ** existing entry and then insert a new one. */
         1572  +          sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP);
         1573  +          sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
  1461   1574   #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
  1462   1575             if( pTab->pIndex ){
  1463   1576               sqlite3MultiWrite(pParse);
  1464   1577               sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1);
  1465   1578             }
  1466   1579           }
  1467   1580           seenReplace = 1;
  1468   1581           break;
  1469   1582         }
         1583  +#ifndef SQLITE_OMIT_UPSERT
         1584  +      case OE_Update: {
         1585  +        sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, 0, iDataCur);
         1586  +        /* Fall through */
         1587  +      }
         1588  +#endif
  1470   1589         case OE_Ignore: {
  1471         -        /*assert( seenReplace==0 );*/
         1590  +        testcase( onError==OE_Ignore );
  1472   1591           sqlite3VdbeGoto(v, ignoreDest);
  1473   1592           break;
  1474   1593         }
  1475   1594       }
  1476   1595       sqlite3VdbeResolveLabel(v, addrRowidOk);
  1477         -    if( ipkTop ){
  1478         -      ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto);
  1479         -      sqlite3VdbeJumpHere(v, ipkTop);
         1596  +    if( sAddr.ipkTop ){
         1597  +      sAddr.ipkBtm = sqlite3VdbeAddOp0(v, OP_Goto);
         1598  +      sqlite3VdbeJumpHere(v, sAddr.ipkTop-1);
  1480   1599       }
  1481   1600     }
  1482   1601   
  1483   1602     /* Test all UNIQUE constraints by creating entries for each UNIQUE
  1484   1603     ** index and making sure that duplicate entries do not already exist.
  1485   1604     ** Compute the revised record entries for indices as we go.
  1486   1605     **
................................................................................
  1490   1609     for(ix=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, ix++){
  1491   1610       int regIdx;          /* Range of registers hold conent for pIdx */
  1492   1611       int regR;            /* Range of registers holding conflicting PK */
  1493   1612       int iThisCur;        /* Cursor for this UNIQUE index */
  1494   1613       int addrUniqueOk;    /* Jump here if the UNIQUE constraint is satisfied */
  1495   1614   
  1496   1615       if( aRegIdx[ix]==0 ) continue;  /* Skip indices that do not change */
         1616  +    if( pUpIdx==pIdx ){
         1617  +      addrUniqueOk = sAddr.upsertBtm;
         1618  +      upsertBypass = sqlite3VdbeGoto(v, 0);
         1619  +      VdbeComment((v, "Skip upsert subroutine"));
         1620  +      sqlite3VdbeResolveLabel(v, sAddr.upsertTop2);
         1621  +    }else{
         1622  +      addrUniqueOk = sqlite3VdbeMakeLabel(v);
         1623  +    }
         1624  +    VdbeNoopComment((v, "uniqueness check for %s", pIdx->zName));
  1497   1625       if( bAffinityDone==0 ){
  1498   1626         sqlite3TableAffinity(v, pTab, regNewData+1);
  1499   1627         bAffinityDone = 1;
  1500   1628       }
  1501   1629       iThisCur = iIdxCur+ix;
  1502         -    addrUniqueOk = sqlite3VdbeMakeLabel(v);
         1630  +
  1503   1631   
  1504   1632       /* Skip partial indices for which the WHERE clause is not true */
  1505   1633       if( pIdx->pPartIdxWhere ){
  1506   1634         sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]);
  1507   1635         pParse->iSelfTab = -(regNewData+1);
  1508   1636         sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk,
  1509   1637                               SQLITE_JUMPIFNULL);
................................................................................
  1554   1682         continue;  /* pIdx is not a UNIQUE index */
  1555   1683       }
  1556   1684       if( overrideError!=OE_Default ){
  1557   1685         onError = overrideError;
  1558   1686       }else if( onError==OE_Default ){
  1559   1687         onError = OE_Abort;
  1560   1688       }
         1689  +
         1690  +    /* Figure out if the upsert clause applies to this index */
         1691  +    if( pUpIdx==pIdx ){
         1692  +      if( pUpsert->pUpsertSet==0 ){
         1693  +        onError = OE_Ignore;  /* DO NOTHING is the same as INSERT OR IGNORE */
         1694  +      }else{
         1695  +        onError = OE_Update;  /* DO UPDATE */
         1696  +      }
         1697  +    }
         1698  +
         1699  +    /* Invoke subroutines to handle IPK replace and upsert prior to running
         1700  +    ** the first REPLACE constraint check. */
         1701  +    if( onError==OE_Replace ){
         1702  +      testcase( sAddr.ipkTop );
         1703  +      testcase( sAddr.upsertTop
         1704  +             && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) );
         1705  +      reorderConstraintChecks(v, &sAddr);
         1706  +    }
  1561   1707   
  1562   1708       /* Collision detection may be omitted if all of the following are true:
  1563   1709       **   (1) The conflict resolution algorithm is REPLACE
  1564   1710       **   (2) The table is a WITHOUT ROWID table
  1565   1711       **   (3) There are no secondary indexes on the table
  1566   1712       **   (4) No delete triggers need to be fired if there is a conflict
  1567   1713       **   (5) No FK constraint counters need to be updated if a conflict occurs.
................................................................................
  1637   1783             }
  1638   1784           }
  1639   1785         }
  1640   1786       }
  1641   1787   
  1642   1788       /* Generate code that executes if the new index entry is not unique */
  1643   1789       assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail
  1644         -        || onError==OE_Ignore || onError==OE_Replace );
         1790  +        || onError==OE_Ignore || onError==OE_Replace || onError==OE_Update );
  1645   1791       switch( onError ){
  1646   1792         case OE_Rollback:
  1647   1793         case OE_Abort:
  1648   1794         case OE_Fail: {
         1795  +        testcase( onError==OE_Rollback );
         1796  +        testcase( onError==OE_Abort );
         1797  +        testcase( onError==OE_Fail );
  1649   1798           sqlite3UniqueConstraint(pParse, onError, pIdx);
  1650   1799           break;
  1651   1800         }
         1801  +#ifndef SQLITE_OMIT_UPSERT
         1802  +      case OE_Update: {
         1803  +        sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, pIdx, iIdxCur+ix);
         1804  +        /* Fall through */
         1805  +      }
         1806  +#endif
  1652   1807         case OE_Ignore: {
         1808  +        testcase( onError==OE_Ignore );
  1653   1809           sqlite3VdbeGoto(v, ignoreDest);
  1654   1810           break;
  1655   1811         }
  1656   1812         default: {
  1657   1813           Trigger *pTrigger = 0;
  1658   1814           assert( onError==OE_Replace );
  1659   1815           sqlite3MultiWrite(pParse);
................................................................................
  1663   1819           sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur,
  1664   1820               regR, nPkField, 0, OE_Replace,
  1665   1821               (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur);
  1666   1822           seenReplace = 1;
  1667   1823           break;
  1668   1824         }
  1669   1825       }
  1670         -    sqlite3VdbeResolveLabel(v, addrUniqueOk);
         1826  +    if( pUpIdx==pIdx ){
         1827  +      sqlite3VdbeJumpHere(v, upsertBypass);
         1828  +    }else{
         1829  +      sqlite3VdbeResolveLabel(v, addrUniqueOk);
         1830  +    }
  1671   1831       sqlite3ExprCachePop(pParse);
  1672   1832       if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField);
         1833  +
  1673   1834     }
  1674         -  if( ipkTop ){
  1675         -    sqlite3VdbeGoto(v, ipkTop+1);
  1676         -    sqlite3VdbeJumpHere(v, ipkBottom);
  1677         -  }
         1835  +  testcase( sAddr.ipkTop!=0 );
         1836  +  testcase( sAddr.upsertTop
         1837  +         && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) );
         1838  +  reorderConstraintChecks(v, &sAddr);
  1678   1839     
  1679   1840     *pbMayReplace = seenReplace;
  1680   1841     VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace));
  1681   1842   }
  1682   1843   
  1683   1844   #ifdef SQLITE_ENABLE_NULL_TRIM
  1684   1845   /*

Changes to src/main.c.

   641    641       }
   642    642   
   643    643       case SQLITE_CONFIG_STMTJRNL_SPILL: {
   644    644         sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int);
   645    645         break;
   646    646       }
   647    647   
          648  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          649  +    case SQLITE_CONFIG_SORTERREF_SIZE: {
          650  +      int iVal = va_arg(ap, int);
          651  +      if( iVal<0 ){
          652  +        iVal = SQLITE_DEFAULT_SORTERREF_SIZE;
          653  +      }
          654  +      sqlite3GlobalConfig.szSorterRef = (u32)iVal;
          655  +      break;
          656  +    }
          657  +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */
          658  +
   648    659       default: {
   649    660         rc = SQLITE_ERROR;
   650    661         break;
   651    662       }
   652    663     }
   653    664     va_end(ap);
   654    665     return rc;
................................................................................
   822    833           { SQLITE_DBCONFIG_ENABLE_FKEY,           SQLITE_ForeignKeys    },
   823    834           { SQLITE_DBCONFIG_ENABLE_TRIGGER,        SQLITE_EnableTrigger  },
   824    835           { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer  },
   825    836           { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension  },
   826    837           { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,      SQLITE_NoCkptOnClose  },
   827    838           { SQLITE_DBCONFIG_ENABLE_QPSG,           SQLITE_EnableQPSG     },
   828    839           { SQLITE_DBCONFIG_TRIGGER_EQP,           SQLITE_TriggerEQP     },
          840  +        { SQLITE_DBCONFIG_RESET_DATABASE,        SQLITE_ResetDatabase  },
   829    841         };
   830    842         unsigned int i;
   831    843         rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
   832    844         for(i=0; i<ArraySize(aFlagOp); i++){
   833    845           if( aFlagOp[i].op==op ){
   834    846             int onoff = va_arg(ap, int);
   835    847             int *pRes = va_arg(ap, int*);
................................................................................
  3975   3987       */
  3976   3988       case SQLITE_TESTCTRL_OPTIMIZATIONS: {
  3977   3989         sqlite3 *db = va_arg(ap, sqlite3*);
  3978   3990         db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff);
  3979   3991         break;
  3980   3992       }
  3981   3993   
  3982         -#ifdef SQLITE_N_KEYWORD
  3983         -    /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord)
  3984         -    **
  3985         -    ** If zWord is a keyword recognized by the parser, then return the
  3986         -    ** number of keywords.  Or if zWord is not a keyword, return 0.
  3987         -    ** 
  3988         -    ** This test feature is only available in the amalgamation since
  3989         -    ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite
  3990         -    ** is built using separate source files.
  3991         -    */
  3992         -    case SQLITE_TESTCTRL_ISKEYWORD: {
  3993         -      const char *zWord = va_arg(ap, const char*);
  3994         -      int n = sqlite3Strlen30(zWord);
  3995         -      rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0;
  3996         -      break;
  3997         -    }
  3998         -#endif 
  3999         -
  4000   3994       /*   sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff);
  4001   3995       **
  4002   3996       ** If parameter onoff is non-zero, configure the wrappers so that all
  4003   3997       ** subsequent calls to localtime() and variants fail. If onoff is zero,
  4004   3998       ** undo this setting.
  4005   3999       */
  4006   4000       case SQLITE_TESTCTRL_LOCALTIME_FAULT: {

Changes to src/os_win.c.

   311    311   /*
   312    312    * The size of the buffer used by sqlite3_win32_write_debug().
   313    313    */
   314    314   #ifndef SQLITE_WIN32_DBG_BUF_SIZE
   315    315   #  define SQLITE_WIN32_DBG_BUF_SIZE   ((int)(4096-sizeof(DWORD)))
   316    316   #endif
   317    317   
   318         -/*
   319         - * The value used with sqlite3_win32_set_directory() to specify that
   320         - * the data directory should be changed.
   321         - */
   322         -#ifndef SQLITE_WIN32_DATA_DIRECTORY_TYPE
   323         -#  define SQLITE_WIN32_DATA_DIRECTORY_TYPE (1)
   324         -#endif
   325         -
   326         -/*
   327         - * The value used with sqlite3_win32_set_directory() to specify that
   328         - * the temporary directory should be changed.
   329         - */
   330         -#ifndef SQLITE_WIN32_TEMP_DIRECTORY_TYPE
   331         -#  define SQLITE_WIN32_TEMP_DIRECTORY_TYPE (2)
   332         -#endif
   333         -
   334    318   /*
   335    319    * If compiled with SQLITE_WIN32_MALLOC on Windows, we will use the
   336    320    * various Win32 API heap functions instead of our own.
   337    321    */
   338    322   #ifdef SQLITE_WIN32_MALLOC
   339    323   
   340    324   /*
................................................................................
  1923   1907   #ifndef SQLITE_OMIT_AUTOINIT
  1924   1908     if( sqlite3_initialize() ) return 0;
  1925   1909   #endif
  1926   1910     return winUtf8ToMbcs(zText, useAnsi);
  1927   1911   }
  1928   1912   
  1929   1913   /*
  1930         -** This function sets the data directory or the temporary directory based on
  1931         -** the provided arguments.  The type argument must be 1 in order to set the
  1932         -** data directory or 2 in order to set the temporary directory.  The zValue
  1933         -** argument is the name of the directory to use.  The return value will be
  1934         -** SQLITE_OK if successful.
         1914  +** This function is the same as sqlite3_win32_set_directory (below); however,
         1915  +** it accepts a UTF-8 string.
  1935   1916   */
  1936         -int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){
         1917  +int sqlite3_win32_set_directory8(
         1918  +  unsigned long type, /* Identifier for directory being set or reset */
         1919  +  const char *zValue  /* New value for directory being set or reset */
         1920  +){
  1937   1921     char **ppDirectory = 0;
  1938   1922   #ifndef SQLITE_OMIT_AUTOINIT
  1939   1923     int rc = sqlite3_initialize();
  1940   1924     if( rc ) return rc;
  1941   1925   #endif
  1942   1926     if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){
  1943   1927       ppDirectory = &sqlite3_data_directory;
................................................................................
  1945   1929       ppDirectory = &sqlite3_temp_directory;
  1946   1930     }
  1947   1931     assert( !ppDirectory || type==SQLITE_WIN32_DATA_DIRECTORY_TYPE
  1948   1932             || type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE
  1949   1933     );
  1950   1934     assert( !ppDirectory || sqlite3MemdebugHasType(*ppDirectory, MEMTYPE_HEAP) );
  1951   1935     if( ppDirectory ){
  1952         -    char *zValueUtf8 = 0;
         1936  +    char *zCopy = 0;
  1953   1937       if( zValue && zValue[0] ){
  1954         -      zValueUtf8 = winUnicodeToUtf8(zValue);
  1955         -      if ( zValueUtf8==0 ){
         1938  +      zCopy = sqlite3_mprintf("%s", zValue);
         1939  +      if ( zCopy==0 ){
  1956   1940           return SQLITE_NOMEM_BKPT;
  1957   1941         }
  1958   1942       }
  1959   1943       sqlite3_free(*ppDirectory);
  1960         -    *ppDirectory = zValueUtf8;
         1944  +    *ppDirectory = zCopy;
  1961   1945       return SQLITE_OK;
  1962   1946     }
  1963   1947     return SQLITE_ERROR;
  1964   1948   }
         1949  +
         1950  +/*
         1951  +** This function is the same as sqlite3_win32_set_directory (below); however,
         1952  +** it accepts a UTF-16 string.
         1953  +*/
         1954  +int sqlite3_win32_set_directory16(
         1955  +  unsigned long type, /* Identifier for directory being set or reset */
         1956  +  const void *zValue  /* New value for directory being set or reset */
         1957  +){
         1958  +  int rc;
         1959  +  char *zUtf8 = 0;
         1960  +  if( zValue ){
         1961  +    zUtf8 = sqlite3_win32_unicode_to_utf8(zValue);
         1962  +    if( zUtf8==0 ) return SQLITE_NOMEM_BKPT;
         1963  +  }
         1964  +  rc = sqlite3_win32_set_directory8(type, zUtf8);
         1965  +  if( zUtf8 ) sqlite3_free(zUtf8);
         1966  +  return rc;
         1967  +}
         1968  +
         1969  +/*
         1970  +** This function sets the data directory or the temporary directory based on
         1971  +** the provided arguments.  The type argument must be 1 in order to set the
         1972  +** data directory or 2 in order to set the temporary directory.  The zValue
         1973  +** argument is the name of the directory to use.  The return value will be
         1974  +** SQLITE_OK if successful.
         1975  +*/
         1976  +int sqlite3_win32_set_directory(
         1977  +  unsigned long type, /* Identifier for directory being set or reset */
         1978  +  void *zValue        /* New value for directory being set or reset */
         1979  +){
         1980  +  return sqlite3_win32_set_directory16(type, zValue);
         1981  +}
  1965   1982   
  1966   1983   /*
  1967   1984   ** The return value of winGetLastErrorMsg
  1968   1985   ** is zero if the error message fits in the buffer, or non-zero
  1969   1986   ** otherwise (if the message was truncated).
  1970   1987   */
  1971   1988   static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){

Changes to src/parse.y.

    20     20   
    21     21   // The type of the data attached to each token is Token.  This is also the
    22     22   // default type for non-terminals.
    23     23   //
    24     24   %token_type {Token}
    25     25   %default_type {Token}
    26     26   
    27         -// The generated parser function takes a 4th argument as follows:
    28         -%extra_argument {Parse *pParse}
           27  +// An extra argument to the constructor for the parser, which is available
           28  +// to all actions.
           29  +%extra_context {Parse *pParse}
    29     30   
    30     31   // This code runs whenever there is a syntax error
    31     32   //
    32     33   %syntax_error {
    33     34     UNUSED_PARAMETER(yymajor);  /* Silence some compiler warnings */
    34     35     if( TOKEN.z[0] ){
    35     36       sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
................................................................................
   110    111   } // end %include
   111    112   
   112    113   // Input is a single SQL command
   113    114   input ::= cmdlist.
   114    115   cmdlist ::= cmdlist ecmd.
   115    116   cmdlist ::= ecmd.
   116    117   ecmd ::= SEMI.
   117         -ecmd ::= explain cmdx SEMI.
   118         -explain ::= .
          118  +ecmd ::= cmdx SEMI.
   119    119   %ifndef SQLITE_OMIT_EXPLAIN
          120  +ecmd ::= explain cmdx.
   120    121   explain ::= EXPLAIN.              { pParse->explain = 1; }
   121    122   explain ::= EXPLAIN QUERY PLAN.   { pParse->explain = 2; }
   122    123   %endif  SQLITE_OMIT_EXPLAIN
   123    124   cmdx ::= cmd.           { sqlite3FinishCoding(pParse); }
   124    125   
   125    126   ///////////////////// Begin and end transactions. ////////////////////////////
   126    127   //
................................................................................
   201    202   
   202    203   // The following directive causes tokens ABORT, AFTER, ASC, etc. to
   203    204   // fallback to ID if they will not parse as their original value.
   204    205   // This obviates the need for the "id" nonterminal.
   205    206   //
   206    207   %fallback ID
   207    208     ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
   208         -  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
          209  +  CONFLICT DATABASE DEFERRED DESC DETACH DO
          210  +  EACH END EXCLUSIVE EXPLAIN FAIL FOR
   209    211     IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
   210    212     QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
   211    213     ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
   212    214   %ifdef SQLITE_OMIT_COMPOUND_SELECT
   213    215     EXCEPT INTERSECT UNION
   214    216   %endif SQLITE_OMIT_COMPOUND_SELECT
   215    217     REINDEX RENAME CTIME_KW IF
................................................................................
   235    237   %right ESCAPE.
   236    238   %left BITAND BITOR LSHIFT RSHIFT.
   237    239   %left PLUS MINUS.
   238    240   %left STAR SLASH REM.
   239    241   %left CONCAT.
   240    242   %left COLLATE.
   241    243   %right BITNOT.
          244  +%nonassoc ON.
   242    245   
   243    246   // An IDENTIFIER can be a generic identifier, or one of several
   244    247   // keywords.  Any non-standard keyword can also be an identifier.
   245    248   //
   246    249   %token_class id  ID|INDEXED.
   247    250   
   248    251   
................................................................................
   683    686   
   684    687   %type fullname {SrcList*}
   685    688   %destructor fullname {sqlite3SrcListDelete(pParse->db, $$);}
   686    689   fullname(A) ::= nm(X).  
   687    690      {A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/}
   688    691   fullname(A) ::= nm(X) DOT nm(Y).  
   689    692      {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/}
          693  +
          694  +%type xfullname {SrcList*}
          695  +%destructor xfullname {sqlite3SrcListDelete(pParse->db, $$);}
          696  +xfullname(A) ::= nm(X).  
          697  +   {A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/}
          698  +xfullname(A) ::= nm(X) DOT nm(Y).  
          699  +   {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/}
          700  +xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z).  {
          701  +   A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/
          702  +   if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z);
          703  +}
          704  +xfullname(A) ::= nm(X) AS nm(Z). {  
          705  +   A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/
          706  +   if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z);
          707  +}
   690    708   
   691    709   %type joinop {int}
   692    710   joinop(X) ::= COMMA|JOIN.              { X = JT_INNER; }
   693    711   joinop(X) ::= JOIN_KW(A) JOIN.
   694    712                     {X = sqlite3JoinType(pParse,&A,0,0);  /*X-overwrites-A*/}
   695    713   joinop(X) ::= JOIN_KW(A) nm(B) JOIN.
   696    714                     {X = sqlite3JoinType(pParse,&A,&B,0); /*X-overwrites-A*/}
   697    715   joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN.
   698    716                     {X = sqlite3JoinType(pParse,&A,&B,&C);/*X-overwrites-A*/}
   699    717   
          718  +// There is a parsing abiguity in an upsert statement that uses a
          719  +// SELECT on the RHS of a the INSERT:
          720  +//
          721  +//      INSERT INTO tab SELECT * FROM aaa JOIN bbb ON CONFLICT ...
          722  +//                                        here ----^^
          723  +//
          724  +// When the ON token is encountered, the parser does not know if it is
          725  +// the beginning of an ON CONFLICT clause, or the beginning of an ON
          726  +// clause associated with the JOIN.  The conflict is resolved in favor
          727  +// of the JOIN.  If an ON CONFLICT clause is intended, insert a dummy
          728  +// WHERE clause in between, like this:
          729  +//
          730  +//      INSERT INTO tab SELECT * FROM aaa JOIN bbb WHERE true ON CONFLICT ...
          731  +//
          732  +// The [AND] and [OR] precedence marks in the rules for on_opt cause the
          733  +// ON in this context to always be interpreted as belonging to the JOIN.
          734  +//
   700    735   %type on_opt {Expr*}
   701    736   %destructor on_opt {sqlite3ExprDelete(pParse->db, $$);}
   702         -on_opt(N) ::= ON expr(E).   {N = E;}
   703         -on_opt(N) ::= .             {N = 0;}
          737  +on_opt(N) ::= ON expr(E).  {N = E;}
          738  +on_opt(N) ::= .     [OR]   {N = 0;}
   704    739   
   705    740   // Note that this block abuses the Token type just a little. If there is
   706    741   // no "INDEXED BY" clause, the returned token is empty (z==0 && n==0). If
   707    742   // there is an INDEXED BY clause, then the token is populated as per normal,
   708    743   // with z pointing to the token data and n containing the number of bytes
   709    744   // in the token.
   710    745   //
................................................................................
   777    812                            {A = sqlite3PExpr(pParse,TK_LIMIT,X,Y);}
   778    813   limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). 
   779    814                            {A = sqlite3PExpr(pParse,TK_LIMIT,Y,X);}
   780    815   
   781    816   /////////////////////////// The DELETE statement /////////////////////////////
   782    817   //
   783    818   %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   784         -cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
          819  +cmd ::= with DELETE FROM xfullname(X) indexed_opt(I) where_opt(W) 
   785    820           orderby_opt(O) limit_opt(L). {
   786    821     sqlite3SrcListIndexedBy(pParse, X, &I);
   787    822     sqlite3DeleteFrom(pParse,X,W,O,L);
   788    823   }
   789    824   %endif
   790    825   %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   791         -cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
          826  +cmd ::= with DELETE FROM xfullname(X) indexed_opt(I) where_opt(W). {
   792    827     sqlite3SrcListIndexedBy(pParse, X, &I);
   793    828     sqlite3DeleteFrom(pParse,X,W,0,0);
   794    829   }
   795    830   %endif
   796    831   
   797    832   %type where_opt {Expr*}
   798    833   %destructor where_opt {sqlite3ExprDelete(pParse->db, $$);}
................................................................................
   799    834   
   800    835   where_opt(A) ::= .                    {A = 0;}
   801    836   where_opt(A) ::= WHERE expr(X).       {A = X;}
   802    837   
   803    838   ////////////////////////// The UPDATE command ////////////////////////////////
   804    839   //
   805    840   %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   806         -cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
          841  +cmd ::= with UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y)
   807    842           where_opt(W) orderby_opt(O) limit_opt(L).  {
   808    843     sqlite3SrcListIndexedBy(pParse, X, &I);
   809    844     sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   810         -  sqlite3Update(pParse,X,Y,W,R,O,L);
          845  +  sqlite3Update(pParse,X,Y,W,R,O,L,0);
   811    846   }
   812    847   %endif
   813    848   %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   814         -cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
          849  +cmd ::= with UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y)
   815    850           where_opt(W).  {
   816    851     sqlite3SrcListIndexedBy(pParse, X, &I);
   817    852     sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   818         -  sqlite3Update(pParse,X,Y,W,R,0,0);
          853  +  sqlite3Update(pParse,X,Y,W,R,0,0,0);
   819    854   }
   820    855   %endif
   821    856   
   822    857   %type setlist {ExprList*}
   823    858   %destructor setlist {sqlite3ExprListDelete(pParse->db, $$);}
   824    859   
   825    860   setlist(A) ::= setlist(A) COMMA nm(X) EQ expr(Y). {
................................................................................
   835    870   }
   836    871   setlist(A) ::= LP idlist(X) RP EQ expr(Y). {
   837    872     A = sqlite3ExprListAppendVector(pParse, 0, X, Y);
   838    873   }
   839    874   
   840    875   ////////////////////////// The INSERT command /////////////////////////////////
   841    876   //
   842         -cmd ::= with insert_cmd(R) INTO fullname(X) idlist_opt(F) select(S). {
   843         -  sqlite3Insert(pParse, X, S, F, R);
          877  +cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) select(S)
          878  +        upsert(U). {
          879  +  sqlite3Insert(pParse, X, S, F, R, U);
   844    880   }
   845         -cmd ::= with insert_cmd(R) INTO fullname(X) idlist_opt(F) DEFAULT VALUES.
          881  +cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) DEFAULT VALUES.
   846    882   {
   847         -  sqlite3Insert(pParse, X, 0, F, R);
          883  +  sqlite3Insert(pParse, X, 0, F, R, 0);
   848    884   }
   849    885   
          886  +%type upsert {Upsert*}
          887  +
          888  +// Because upsert only occurs at the tip end of the INSERT rule for cmd,
          889  +// there is never a case where the value of the upsert pointer will not
          890  +// be destroyed by the cmd action.  So comment-out the destructor to
          891  +// avoid unreachable code.
          892  +//%destructor upsert {sqlite3UpsertDelete(pParse->db,$$);}
          893  +upsert(A) ::= . { A = 0; }
          894  +upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW)
          895  +              DO UPDATE SET setlist(Z) where_opt(W).
          896  +              { A = sqlite3UpsertNew(pParse->db,T,TW,Z,W);}
          897  +upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW) DO NOTHING.
          898  +              { A = sqlite3UpsertNew(pParse->db,T,TW,0,0); }
          899  +upsert(A) ::= ON CONFLICT DO NOTHING.
          900  +              { A = sqlite3UpsertNew(pParse->db,0,0,0,0); }
          901  +
   850    902   %type insert_cmd {int}
   851    903   insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
   852    904   insert_cmd(A) ::= REPLACE.            {A = OE_Replace;}
   853    905   
   854    906   %type idlist_opt {IdList*}
   855    907   %destructor idlist_opt {sqlite3IdListDelete(pParse->db, $$);}
   856    908   %type idlist {IdList*}
................................................................................
  1388   1440   // UPDATE 
  1389   1441   trigger_cmd(A) ::=
  1390   1442      UPDATE(B) orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z) scanpt(E).  
  1391   1443      {A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R, B.z, E);}
  1392   1444   
  1393   1445   // INSERT
  1394   1446   trigger_cmd(A) ::= scanpt(B) insert_cmd(R) INTO
  1395         -                      trnm(X) idlist_opt(F) select(S) scanpt(Z).
  1396         -   {A = sqlite3TriggerInsertStep(pParse->db,&X,F,S,R,B,Z);/*A-overwrites-R*/}
  1397         -
         1447  +                      trnm(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). {
         1448  +   A = sqlite3TriggerInsertStep(pParse->db,&X,F,S,R,U,B,Z);/*A-overwrites-R*/
         1449  +}
  1398   1450   // DELETE
  1399   1451   trigger_cmd(A) ::= DELETE(B) FROM trnm(X) tridxby where_opt(Y) scanpt(E).
  1400   1452      {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y, B.z, E);}
  1401   1453   
  1402   1454   // SELECT
  1403   1455   trigger_cmd(A) ::= scanpt(B) select(X) scanpt(E).
  1404   1456      {A = sqlite3TriggerSelectStep(pParse->db, X, B, E); /*A-overwrites-X*/}

Changes to src/prepare.c.

   144    144     Db *pDb;
   145    145     char const *azArg[4];
   146    146     int meta[5];
   147    147     InitData initData;
   148    148     const char *zMasterName;
   149    149     int openedTransaction = 0;
   150    150   
          151  +  assert( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 );
   151    152     assert( iDb>=0 && iDb<db->nDb );
   152    153     assert( db->aDb[iDb].pSchema );
   153    154     assert( sqlite3_mutex_held(db->mutex) );
   154    155     assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
   155    156   
   156    157     db->init.busy = 1;
   157    158   
................................................................................
   213    214     **    meta[9]   unused
   214    215     **
   215    216     ** Note: The #defined SQLITE_UTF* symbols in sqliteInt.h correspond to
   216    217     ** the possible values of meta[4].
   217    218     */
   218    219     for(i=0; i<ArraySize(meta); i++){
   219    220       sqlite3BtreeGetMeta(pDb->pBt, i+1, (u32 *)&meta[i]);
          221  +  }
          222  +  if( (db->flags & SQLITE_ResetDatabase)!=0 ){
          223  +    memset(meta, 0, sizeof(meta));
   220    224     }
   221    225     pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1];
   222    226   
   223    227     /* If opening a non-empty database, check the text encoding. For the
   224    228     ** main database, set sqlite3.enc to the encoding of the main database.
   225    229     ** For an attached db, it is an error if the encoding is not the same
   226    230     ** as sqlite3.enc.
................................................................................
   373    377     /* Do the main schema first */
   374    378     if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){
   375    379       rc = sqlite3InitOne(db, 0, pzErrMsg);
   376    380       if( rc ) return rc;
   377    381     }
   378    382     /* All other schemas after the main schema. The "temp" schema must be last */
   379    383     for(i=db->nDb-1; i>0; i--){
          384  +    assert( i==1 || sqlite3BtreeHoldsMutex(db->aDb[i].pBt) );
   380    385       if( !DbHasProperty(db, i, DB_SchemaLoaded) ){
   381    386         rc = sqlite3InitOne(db, i, pzErrMsg);
   382    387         if( rc ) return rc;
   383    388       }
   384    389     }
   385    390     if( commit_internal ){
   386    391       sqlite3CommitInternalChanges(db);
................................................................................
   394    399   */
   395    400   int sqlite3ReadSchema(Parse *pParse){
   396    401     int rc = SQLITE_OK;
   397    402     sqlite3 *db = pParse->db;
   398    403     assert( sqlite3_mutex_held(db->mutex) );
   399    404     if( !db->init.busy ){
   400    405       rc = sqlite3Init(db, &pParse->zErrMsg);
   401         -  }
   402         -  if( rc!=SQLITE_OK ){
   403         -    pParse->rc = rc;
   404         -    pParse->nErr++;
          406  +    if( rc!=SQLITE_OK ){
          407  +      pParse->rc = rc;
          408  +      pParse->nErr++;
          409  +    }else if( db->noSharedCache ){
          410  +      db->mDbFlags |= DBFLAG_SchemaKnownOk;
          411  +    }
   405    412     }
   406    413     return rc;
   407    414   }
   408    415   
   409    416   
   410    417   /*
   411    418   ** Check schema cookies in all databases.  If any cookie is out
................................................................................
   608    615     }
   609    616     rc = sParse.rc;
   610    617   
   611    618   #ifndef SQLITE_OMIT_EXPLAIN
   612    619     if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){
   613    620       static const char * const azColName[] = {
   614    621          "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",
   615         -       "selectid", "order", "from", "detail"
          622  +       "id", "parent", "notused", "detail"
   616    623       };
   617    624       int iFirst, mx;
   618    625       if( sParse.explain==2 ){
   619    626         sqlite3VdbeSetNumCols(sParse.pVdbe, 4);
   620    627         iFirst = 8;
   621    628         mx = 12;
   622    629       }else{

Changes to src/resolve.c.

   187    187     int cntTab = 0;                   /* Number of matching table names */
   188    188     int nSubquery = 0;                /* How many levels of subquery */
   189    189     sqlite3 *db = pParse->db;         /* The database connection */
   190    190     struct SrcList_item *pItem;       /* Use for looping over pSrcList items */
   191    191     struct SrcList_item *pMatch = 0;  /* The matching pSrcList item */
   192    192     NameContext *pTopNC = pNC;        /* First namecontext in the list */
   193    193     Schema *pSchema = 0;              /* Schema of the expression */
   194         -  int isTrigger = 0;                /* True if resolved to a trigger column */
          194  +  int eNewExprOp = TK_COLUMN;       /* New value for pExpr->op on success */
   195    195     Table *pTab = 0;                  /* Table hold the row */
   196    196     Column *pCol;                     /* A column of pTab */
   197    197   
   198    198     assert( pNC );     /* the name context cannot be NULL. */
   199    199     assert( zCol );    /* The Z in X.Y.Z cannot be NULL */
   200    200     assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
   201    201   
................................................................................
   292    292           if( (pMatch->fg.jointype & JT_LEFT)!=0 ){
   293    293             ExprSetProperty(pExpr, EP_CanBeNull);
   294    294           }
   295    295           pSchema = pExpr->pTab->pSchema;
   296    296         }
   297    297       } /* if( pSrcList ) */
   298    298   
   299         -#ifndef SQLITE_OMIT_TRIGGER
          299  +#if !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT)
   300    300       /* If we have not already resolved the name, then maybe 
   301         -    ** it is a new.* or old.* trigger argument reference
          301  +    ** it is a new.* or old.* trigger argument reference.  Or
          302  +    ** maybe it is an excluded.* from an upsert.
   302    303       */
   303         -    if( zDb==0 && zTab!=0 && cntTab==0 && pParse->pTriggerTab!=0 ){
   304         -      int op = pParse->eTriggerOp;
   305         -      assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT );
   306         -      if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){
   307         -        pExpr->iTable = 1;
   308         -        pTab = pParse->pTriggerTab;
   309         -      }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){
   310         -        pExpr->iTable = 0;
   311         -        pTab = pParse->pTriggerTab;
   312         -      }else{
   313         -        pTab = 0;
          304  +    if( zDb==0 && zTab!=0 && cntTab==0 ){
          305  +      pTab = 0;
          306  +#ifndef SQLITE_OMIT_TRIGGER
          307  +      if( pParse->pTriggerTab!=0 ){
          308  +        int op = pParse->eTriggerOp;
          309  +        assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT );
          310  +        if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){
          311  +          pExpr->iTable = 1;
          312  +          pTab = pParse->pTriggerTab;
          313  +        }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){
          314  +          pExpr->iTable = 0;
          315  +          pTab = pParse->pTriggerTab;
          316  +        }
   314    317         }
          318  +#endif /* SQLITE_OMIT_TRIGGER */
          319  +#ifndef SQLITE_OMIT_UPSERT
          320  +      if( (pNC->ncFlags & NC_UUpsert)!=0 ){
          321  +        Upsert *pUpsert = pNC->uNC.pUpsert;
          322  +        if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){
          323  +          pTab = pUpsert->pUpsertSrc->a[0].pTab;
          324  +          pExpr->iTable = 2;
          325  +        }
          326  +      }
          327  +#endif /* SQLITE_OMIT_UPSERT */
   315    328   
   316    329         if( pTab ){ 
   317    330           int iCol;
   318    331           pSchema = pTab->pSchema;
   319    332           cntTab++;
   320    333           for(iCol=0, pCol=pTab->aCol; iCol<pTab->nCol; iCol++, pCol++){
   321    334             if( sqlite3StrICmp(pCol->zName, zCol)==0 ){
................................................................................
   327    340           }
   328    341           if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){
   329    342             /* IMP: R-51414-32910 */
   330    343             iCol = -1;
   331    344           }
   332    345           if( iCol<pTab->nCol ){
   333    346             cnt++;
   334         -          if( iCol<0 ){
   335         -            pExpr->affinity = SQLITE_AFF_INTEGER;
   336         -          }else if( pExpr->iTable==0 ){
   337         -            testcase( iCol==31 );
   338         -            testcase( iCol==32 );
   339         -            pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
   340         -          }else{
   341         -            testcase( iCol==31 );
   342         -            testcase( iCol==32 );
   343         -            pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
          347  +#ifndef SQLITE_OMIT_UPSERT
          348  +          if( pExpr->iTable==2 ){
          349  +            testcase( iCol==(-1) );
          350  +            pExpr->iTable = pNC->uNC.pUpsert->regData + iCol;
          351  +            eNewExprOp = TK_REGISTER;
          352  +          }else
          353  +#endif /* SQLITE_OMIT_UPSERT */
          354  +          {
          355  +#ifndef SQLITE_OMIT_TRIGGER
          356  +            if( iCol<0 ){
          357  +              pExpr->affinity = SQLITE_AFF_INTEGER;
          358  +            }else if( pExpr->iTable==0 ){
          359  +              testcase( iCol==31 );
          360  +              testcase( iCol==32 );
          361  +              pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
          362  +            }else{
          363  +              testcase( iCol==31 );
          364  +              testcase( iCol==32 );
          365  +              pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
          366  +            }
          367  +            pExpr->pTab = pTab;
          368  +            pExpr->iColumn = (i16)iCol;
          369  +            eNewExprOp = TK_TRIGGER;
          370  +#endif /* SQLITE_OMIT_TRIGGER */
   344    371             }
   345         -          pExpr->iColumn = (i16)iCol;
   346         -          pExpr->pTab = pTab;
   347         -          isTrigger = 1;
   348    372           }
   349    373         }
   350    374       }
   351         -#endif /* !defined(SQLITE_OMIT_TRIGGER) */
          375  +#endif /* !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) */
   352    376   
   353    377       /*
   354    378       ** Perhaps the name is a reference to the ROWID
   355    379       */
   356    380       if( cnt==0
   357    381        && cntTab==1
   358    382        && pMatch
................................................................................
   379    403       **
   380    404       ** The ability to use an output result-set column in the WHERE, GROUP BY,
   381    405       ** or HAVING clauses, or as part of a larger expression in the ORDER BY
   382    406       ** clause is not standard SQL.  This is a (goofy) SQLite extension, that
   383    407       ** is supported for backwards compatibility only. Hence, we issue a warning
   384    408       ** on sqlite3_log() whenever the capability is used.
   385    409       */
   386         -    if( (pEList = pNC->pEList)!=0
   387         -     && zTab==0
          410  +    if( (pNC->ncFlags & NC_UEList)!=0
   388    411        && cnt==0
          412  +     && zTab==0
   389    413       ){
          414  +      pEList = pNC->uNC.pEList;
          415  +      assert( pEList!=0 );
   390    416         for(j=0; j<pEList->nExpr; j++){
   391    417           char *zAs = pEList->a[j].zName;
   392    418           if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){
   393    419             Expr *pOrig;
   394    420             assert( pExpr->pLeft==0 && pExpr->pRight==0 );
   395    421             assert( pExpr->x.pList==0 );
   396    422             assert( pExpr->x.pSelect==0 );
................................................................................
   479    505   
   480    506     /* Clean up and return
   481    507     */
   482    508     sqlite3ExprDelete(db, pExpr->pLeft);
   483    509     pExpr->pLeft = 0;
   484    510     sqlite3ExprDelete(db, pExpr->pRight);
   485    511     pExpr->pRight = 0;
   486         -  pExpr->op = (isTrigger ? TK_TRIGGER : TK_COLUMN);
          512  +  pExpr->op = eNewExprOp;
   487    513     ExprSetProperty(pExpr, EP_Leaf);
   488    514   lookupname_end:
   489    515     if( cnt==1 ){
   490    516       assert( pNC!=0 );
   491    517       if( !ExprHasProperty(pExpr, EP_Alias) ){
   492    518         sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList);
   493    519       }
................................................................................
   911    937     pEList = pSelect->pEList;
   912    938   
   913    939     /* Resolve all names in the ORDER BY term expression
   914    940     */
   915    941     memset(&nc, 0, sizeof(nc));
   916    942     nc.pParse = pParse;
   917    943     nc.pSrcList = pSelect->pSrc;
   918         -  nc.pEList = pEList;
   919         -  nc.ncFlags = NC_AllowAgg;
          944  +  nc.uNC.pEList = pEList;
          945  +  nc.ncFlags = NC_AllowAgg|NC_UEList;
   920    946     nc.nErr = 0;
   921    947     db = pParse->db;
   922    948     savedSuppErr = db->suppressErr;
   923    949     db->suppressErr = 1;
   924    950     rc = sqlite3ResolveExprNames(&nc, pE);
   925    951     db->suppressErr = savedSuppErr;
   926    952     if( rc ) return 0;
................................................................................
  1295   1321       ** other expressions in the SELECT statement. This is so that
  1296   1322       ** expressions in the WHERE clause (etc.) can refer to expressions by
  1297   1323       ** aliases in the result set.
  1298   1324       **
  1299   1325       ** Minor point: If this is the case, then the expression will be
  1300   1326       ** re-evaluated for each reference to it.
  1301   1327       */
  1302         -    sNC.pEList = p->pEList;
         1328  +    assert( (sNC.ncFlags & (NC_UAggInfo|NC_UUpsert))==0 );
         1329  +    sNC.uNC.pEList = p->pEList;
         1330  +    sNC.ncFlags |= NC_UEList;
  1303   1331       if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort;
  1304   1332       if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort;
  1305   1333   
  1306   1334       /* Resolve names in table-valued-function arguments */
  1307   1335       for(i=0; i<p->pSrc->nSrc; i++){
  1308   1336         struct SrcList_item *pItem = &p->pSrc->a[i];
  1309   1337         if( pItem->fg.isTabFunc
................................................................................
  1528   1556   ** Any errors cause an error message to be set in pParse.
  1529   1557   */
  1530   1558   void sqlite3ResolveSelfReference(
  1531   1559     Parse *pParse,      /* Parsing context */
  1532   1560     Table *pTab,        /* The table being referenced */
  1533   1561     int type,           /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */
  1534   1562     Expr *pExpr,        /* Expression to resolve.  May be NULL. */
  1535         -  ExprList *pList     /* Expression list to resolve.  May be NUL. */
         1563  +  ExprList *pList     /* Expression list to resolve.  May be NULL. */
  1536   1564   ){
  1537   1565     SrcList sSrc;                   /* Fake SrcList for pParse->pNewTable */
  1538   1566     NameContext sNC;                /* Name context for pParse->pNewTable */
  1539   1567   
  1540   1568     assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr );
  1541   1569     memset(&sNC, 0, sizeof(sNC));
  1542   1570     memset(&sSrc, 0, sizeof(sSrc));

Changes to src/select.c.

    17     17   /*
    18     18   ** Trace output macros
    19     19   */
    20     20   #if SELECTTRACE_ENABLED
    21     21   /***/ int sqlite3SelectTrace = 0;
    22     22   # define SELECTTRACE(K,P,S,X)  \
    23     23     if(sqlite3SelectTrace&(K))   \
    24         -    sqlite3DebugPrintf("%s/%p: ",(S)->zSelName,(S)),\
           24  +    sqlite3DebugPrintf("%s/%d/%p: ",(S)->zSelName,(P)->addrExplain,(S)),\
    25     25       sqlite3DebugPrintf X
    26     26   #else
    27     27   # define SELECTTRACE(K,P,S,X)
    28     28   #endif
    29     29   
    30     30   
    31     31   /*
................................................................................
    40     40     int tabTnct;    /* Ephemeral table used for DISTINCT processing */
    41     41     int addrTnct;   /* Address of OP_OpenEphemeral opcode for tabTnct */
    42     42   };
    43     43   
    44     44   /*
    45     45   ** An instance of the following object is used to record information about
    46     46   ** the ORDER BY (or GROUP BY) clause of query is being coded.
           47  +**
           48  +** The aDefer[] array is used by the sorter-references optimization. For
           49  +** example, assuming there is no index that can be used for the ORDER BY,
           50  +** for the query:
           51  +**
           52  +**     SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10;
           53  +**
           54  +** it may be more efficient to add just the "a" values to the sorter, and
           55  +** retrieve the associated "bigblob" values directly from table t1 as the
           56  +** 10 smallest "a" values are extracted from the sorter.
           57  +**
           58  +** When the sorter-reference optimization is used, there is one entry in the
           59  +** aDefer[] array for each database table that may be read as values are
           60  +** extracted from the sorter.
    47     61   */
    48     62   typedef struct SortCtx SortCtx;
    49     63   struct SortCtx {
    50     64     ExprList *pOrderBy;   /* The ORDER BY (or GROUP BY clause) */
    51     65     int nOBSat;           /* Number of ORDER BY terms satisfied by indices */
    52     66     int iECursor;         /* Cursor number for the sorter */
    53     67     int regReturn;        /* Register holding block-output return address */
    54     68     int labelBkOut;       /* Start label for the block-output subroutine */
    55     69     int addrSortIndex;    /* Address of the OP_SorterOpen or OP_OpenEphemeral */
    56     70     int labelDone;        /* Jump here when done, ex: LIMIT reached */
    57     71     u8 sortFlags;         /* Zero or more SORTFLAG_* bits */
    58     72     u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */
           73  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
           74  +  u8 nDefer;            /* Number of valid entries in aDefer[] */
           75  +  struct DeferredCsr {
           76  +    Table *pTab;        /* Table definition */
           77  +    int iCsr;           /* Cursor number for table */
           78  +    int nKey;           /* Number of PK columns for table pTab (>=1) */
           79  +  } aDefer[4];
           80  +#endif
           81  +  struct RowLoadInfo *pDeferredRowLoad;  /* Deferred row loading info or NULL */
    59     82   };
    60     83   #define SORTFLAG_UseSorter  0x01   /* Use SorterOpen instead of OpenEphemeral */
    61     84   
    62     85   /*
    63     86   ** Delete all the content of a Select structure.  Deallocate the structure
    64     87   ** itself only if bFree is true.
    65     88   */
................................................................................
   509    532   /* Forward reference */
   510    533   static KeyInfo *keyInfoFromExprList(
   511    534     Parse *pParse,       /* Parsing context */
   512    535     ExprList *pList,     /* Form the KeyInfo object from this ExprList */
   513    536     int iStart,          /* Begin with this column of pList */
   514    537     int nExtra           /* Add this many extra columns to the end */
   515    538   );
          539  +
          540  +/*
          541  +** An instance of this object holds information (beyond pParse and pSelect)
          542  +** needed to load the next result row that is to be added to the sorter.
          543  +*/
          544  +typedef struct RowLoadInfo RowLoadInfo;
          545  +struct RowLoadInfo {
          546  +  int regResult;               /* Store results in array of registers here */
          547  +  u8 ecelFlags;                /* Flag argument to ExprCodeExprList() */
          548  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          549  +  ExprList *pExtra;            /* Extra columns needed by sorter refs */
          550  +  int regExtraResult;          /* Where to load the extra columns */
          551  +#endif
          552  +};
          553  +
          554  +/*
          555  +** This routine does the work of loading query data into an array of
          556  +** registers so that it can be added to the sorter.
          557  +*/
          558  +static void innerLoopLoadRow(
          559  +  Parse *pParse,             /* Statement under construction */
          560  +  Select *pSelect,           /* The query being coded */
          561  +  RowLoadInfo *pInfo         /* Info needed to complete the row load */
          562  +){
          563  +  sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult,
          564  +                          0, pInfo->ecelFlags);
          565  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          566  +  if( pInfo->pExtra ){
          567  +    sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0);
          568  +    sqlite3ExprListDelete(pParse->db, pInfo->pExtra);
          569  +  }
          570  +#endif
          571  +}
          572  +
          573  +/*
          574  +** Code the OP_MakeRecord instruction that generates the entry to be
          575  +** added into the sorter.
          576  +**
          577  +** Return the register in which the result is stored.
          578  +*/
          579  +static int makeSorterRecord(
          580  +  Parse *pParse,
          581  +  SortCtx *pSort,
          582  +  Select *pSelect,
          583  +  int regBase,
          584  +  int nBase
          585  +){
          586  +  int nOBSat = pSort->nOBSat;
          587  +  Vdbe *v = pParse->pVdbe;
          588  +  int regOut = ++pParse->nMem;
          589  +  if( pSort->pDeferredRowLoad ){
          590  +    innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad);
          591  +  }
          592  +  sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut);
          593  +  return regOut;
          594  +}
   516    595   
   517    596   /*
   518    597   ** Generate code that will push the record in registers regData
   519    598   ** through regData+nData-1 onto the sorter.
   520    599   */
   521    600   static void pushOntoSorter(
   522    601     Parse *pParse,         /* Parser context */
   523    602     SortCtx *pSort,        /* Information about the ORDER BY clause */
   524    603     Select *pSelect,       /* The whole SELECT statement */
   525    604     int regData,           /* First register holding data to be sorted */
   526    605     int regOrigData,       /* First register holding data before packing */
   527         -  int nData,             /* Number of elements in the data array */
          606  +  int nData,             /* Number of elements in the regData data array */
   528    607     int nPrefixReg         /* No. of reg prior to regData available for use */
   529    608   ){
   530    609     Vdbe *v = pParse->pVdbe;                         /* Stmt under construction */
   531    610     int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0);
   532    611     int nExpr = pSort->pOrderBy->nExpr;              /* No. of ORDER BY terms */
   533    612     int nBase = nExpr + bSeq + nData;                /* Fields in sorter record */
   534    613     int regBase;                                     /* Regs for sorter record */
   535         -  int regRecord = ++pParse->nMem;                  /* Assembled sorter record */
          614  +  int regRecord = 0;                               /* Assembled sorter record */
   536    615     int nOBSat = pSort->nOBSat;                      /* ORDER BY terms to skip */
   537    616     int op;                            /* Opcode to add sorter record to sorter */
   538    617     int iLimit;                        /* LIMIT counter */
          618  +  int iSkip = 0;                     /* End of the sorter insert loop */
   539    619   
   540    620     assert( bSeq==0 || bSeq==1 );
          621  +
          622  +  /* Three cases:
          623  +  **   (1) The data to be sorted has already been packed into a Record
          624  +  **       by a prior OP_MakeRecord.  In this case nData==1 and regData
          625  +  **       will be completely unrelated to regOrigData.
          626  +  **   (2) All output columns are included in the sort record.  In that
          627  +  **       case regData==regOrigData.
          628  +  **   (3) Some output columns are omitted from the sort record due to
          629  +  **       the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the
          630  +  **       SQLITE_ECEL_OMITREF optimization.  In that case, regOrigData==0
          631  +  **       to prevent this routine from trying to copy values that might
          632  +  **       not exist.
          633  +  */
   541    634     assert( nData==1 || regData==regOrigData || regOrigData==0 );
          635  +
   542    636     if( nPrefixReg ){
   543    637       assert( nPrefixReg==nExpr+bSeq );
   544         -    regBase = regData - nExpr - bSeq;
          638  +    regBase = regData - nPrefixReg;
   545    639     }else{
   546    640       regBase = pParse->nMem + 1;
   547    641       pParse->nMem += nBase;
   548    642     }
   549    643     assert( pSelect->iOffset==0 || pSelect->iLimit!=0 );
   550    644     iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit;
   551    645     pSort->labelDone = sqlite3VdbeMakeLabel(v);
................................................................................
   553    647                             SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0));
   554    648     if( bSeq ){
   555    649       sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr);
   556    650     }
   557    651     if( nPrefixReg==0 && nData>0 ){
   558    652       sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData);
   559    653     }
   560         -  sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord);
   561    654     if( nOBSat>0 ){
   562    655       int regPrevKey;   /* The first nOBSat columns of the previous row */
   563    656       int addrFirst;    /* Address of the OP_IfNot opcode */
   564    657       int addrJmp;      /* Address of the OP_Jump opcode */
   565    658       VdbeOp *pOp;      /* Opcode that opens the sorter */
   566    659       int nKey;         /* Number of sorting key columns, including OP_Sequence */
   567    660       KeyInfo *pKI;     /* Original KeyInfo on the sorter table */
   568    661   
          662  +    regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
   569    663       regPrevKey = pParse->nMem+1;
   570    664       pParse->nMem += pSort->nOBSat;
   571    665       nKey = nExpr - pSort->nOBSat + bSeq;
   572    666       if( bSeq ){
   573    667         addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); 
   574    668       }else{
   575    669         addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor);
................................................................................
   595    689         sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone);
   596    690         VdbeCoverage(v);
   597    691       }
   598    692       sqlite3VdbeJumpHere(v, addrFirst);
   599    693       sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat);
   600    694       sqlite3VdbeJumpHere(v, addrJmp);
   601    695     }
          696  +  if( iLimit ){
          697  +    /* At this point the values for the new sorter entry are stored
          698  +    ** in an array of registers. They need to be composed into a record
          699  +    ** and inserted into the sorter if either (a) there are currently
          700  +    ** less than LIMIT+OFFSET items or (b) the new record is smaller than 
          701  +    ** the largest record currently in the sorter. If (b) is true and there
          702  +    ** are already LIMIT+OFFSET items in the sorter, delete the largest
          703  +    ** entry before inserting the new one. This way there are never more 
          704  +    ** than LIMIT+OFFSET items in the sorter.
          705  +    **
          706  +    ** If the new record does not need to be inserted into the sorter,
          707  +    ** jump to the next iteration of the loop. Or, if the
          708  +    ** pSort->bOrderedInnerLoop flag is set to indicate that the inner
          709  +    ** loop delivers items in sorted order, jump to the next iteration
          710  +    ** of the outer loop.
          711  +    */
          712  +    int iCsr = pSort->iECursor;
          713  +    sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4);
          714  +    VdbeCoverage(v);
          715  +    sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0);
          716  +    iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE,
          717  +                                 iCsr, 0, regBase+nOBSat, nExpr-nOBSat);
          718  +    VdbeCoverage(v);
          719  +    sqlite3VdbeAddOp1(v, OP_Delete, iCsr);
          720  +  }
          721  +  if( regRecord==0 ){
          722  +    regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
          723  +  }
   602    724     if( pSort->sortFlags & SORTFLAG_UseSorter ){
   603    725       op = OP_SorterInsert;
   604    726     }else{
   605    727       op = OP_IdxInsert;
   606    728     }
   607    729     sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord,
   608    730                          regBase+nOBSat, nBase-nOBSat);
   609         -  if( iLimit ){
   610         -    int addr;
   611         -    int r1 = 0;
   612         -    /* Fill the sorter until it contains LIMIT+OFFSET entries.  (The iLimit
   613         -    ** register is initialized with value of LIMIT+OFFSET.)  After the sorter
   614         -    ** fills up, delete the least entry in the sorter after each insert.
   615         -    ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */
   616         -    addr = sqlite3VdbeAddOp1(v, OP_IfNotZero, iLimit); VdbeCoverage(v);
   617         -    sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
   618         -    if( pSort->bOrderedInnerLoop ){
   619         -      r1 = ++pParse->nMem;
   620         -      sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1);
   621         -      VdbeComment((v, "seq"));
   622         -    }
   623         -    sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
   624         -    if( pSort->bOrderedInnerLoop ){
   625         -      /* If the inner loop is driven by an index such that values from
   626         -      ** the same iteration of the inner loop are in sorted order, then
   627         -      ** immediately jump to the next iteration of an inner loop if the
   628         -      ** entry from the current iteration does not fit into the top
   629         -      ** LIMIT+OFFSET entries of the sorter. */
   630         -      int iBrk = sqlite3VdbeCurrentAddr(v) + 2;
   631         -      sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1);
   632         -      sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
   633         -      VdbeCoverage(v);
   634         -    }
   635         -    sqlite3VdbeJumpHere(v, addr);
          731  +  if( iSkip ){
          732  +    assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 );
          733  +    sqlite3VdbeChangeP2(v, iSkip,
          734  +         sqlite3VdbeCurrentAddr(v) + pSort->bOrderedInnerLoop);
   636    735     }
   637    736   }
   638    737   
   639    738   /*
   640    739   ** Add code to implement the OFFSET
   641    740   */
   642    741   static void codeOffset(
................................................................................
   674    773     sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v);
   675    774     sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);
   676    775     sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, iMem, N);
   677    776     sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   678    777     sqlite3ReleaseTempReg(pParse, r1);
   679    778   }
   680    779   
          780  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          781  +/*
          782  +** This function is called as part of inner-loop generation for a SELECT
          783  +** statement with an ORDER BY that is not optimized by an index. It 
          784  +** determines the expressions, if any, that the sorter-reference 
          785  +** optimization should be used for. The sorter-reference optimization
          786  +** is used for SELECT queries like:
          787  +**
          788  +**   SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10
          789  +**
          790  +** If the optimization is used for expression "bigblob", then instead of
          791  +** storing values read from that column in the sorter records, the PK of
          792  +** the row from table t1 is stored instead. Then, as records are extracted from
          793  +** the sorter to return to the user, the required value of bigblob is
          794  +** retrieved directly from table t1. If the values are very large, this 
          795  +** can be more efficient than storing them directly in the sorter records.
          796  +**
          797  +** The ExprList_item.bSorterRef flag is set for each expression in pEList 
          798  +** for which the sorter-reference optimization should be enabled. 
          799  +** Additionally, the pSort->aDefer[] array is populated with entries
          800  +** for all cursors required to evaluate all selected expressions. Finally.
          801  +** output variable (*ppExtra) is set to an expression list containing
          802  +** expressions for all extra PK values that should be stored in the
          803  +** sorter records.
          804  +*/
          805  +static void selectExprDefer(
          806  +  Parse *pParse,                  /* Leave any error here */
          807  +  SortCtx *pSort,                 /* Sorter context */
          808  +  ExprList *pEList,               /* Expressions destined for sorter */
          809  +  ExprList **ppExtra              /* Expressions to append to sorter record */
          810  +){
          811  +  int i;
          812  +  int nDefer = 0;
          813  +  ExprList *pExtra = 0;
          814  +  for(i=0; i<pEList->nExpr; i++){
          815  +    struct ExprList_item *pItem = &pEList->a[i];
          816  +    if( pItem->u.x.iOrderByCol==0 ){
          817  +      Expr *pExpr = pItem->pExpr;
          818  +      Table *pTab = pExpr->pTab;
          819  +      if( pExpr->op==TK_COLUMN && pTab && !IsVirtual(pTab)
          820  +       && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF)
          821  +      ){
          822  +        int j;
          823  +        for(j=0; j<nDefer; j++){
          824  +          if( pSort->aDefer[j].iCsr==pExpr->iTable ) break;
          825  +        }
          826  +        if( j==nDefer ){
          827  +          if( nDefer==ArraySize(pSort->aDefer) ){
          828  +            continue;
          829  +          }else{
          830  +            int nKey = 1;
          831  +            int k;
          832  +            Index *pPk = 0;
          833  +            if( !HasRowid(pTab) ){
          834  +              pPk = sqlite3PrimaryKeyIndex(pTab);
          835  +              nKey = pPk->nKeyCol;
          836  +            }
          837  +            for(k=0; k<nKey; k++){
          838  +              Expr *pNew = sqlite3PExpr(pParse, TK_COLUMN, 0, 0);
          839  +              if( pNew ){
          840  +                pNew->iTable = pExpr->iTable;
          841  +                pNew->pTab = pExpr->pTab;
          842  +                pNew->iColumn = pPk ? pPk->aiColumn[k] : -1;
          843  +                pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew);
          844  +              }
          845  +            }
          846  +            pSort->aDefer[nDefer].pTab = pExpr->pTab;
          847  +            pSort->aDefer[nDefer].iCsr = pExpr->iTable;
          848  +            pSort->aDefer[nDefer].nKey = nKey;
          849  +            nDefer++;
          850  +          }
          851  +        }
          852  +        pItem->bSorterRef = 1;
          853  +      }
          854  +    }
          855  +  }
          856  +  pSort->nDefer = (u8)nDefer;
          857  +  *ppExtra = pExtra;
          858  +}
          859  +#endif
          860  +
   681    861   /*
   682    862   ** This routine generates the code for the inside of the inner loop
   683    863   ** of a SELECT.
   684    864   **
   685    865   ** If srcTab is negative, then the p->pEList expressions
   686    866   ** are evaluated in order to get the data for this row.  If srcTab is
   687    867   ** zero or more, then data is pulled from srcTab and p->pEList is used only 
................................................................................
   700    880     Vdbe *v = pParse->pVdbe;
   701    881     int i;
   702    882     int hasDistinct;            /* True if the DISTINCT keyword is present */
   703    883     int eDest = pDest->eDest;   /* How to dispose of results */
   704    884     int iParm = pDest->iSDParm; /* First argument to disposal method */
   705    885     int nResultCol;             /* Number of result columns */
   706    886     int nPrefixReg = 0;         /* Number of extra registers before regResult */
          887  +  RowLoadInfo sRowLoadInfo;   /* Info for deferred row loading */
   707    888   
   708    889     /* Usually, regResult is the first cell in an array of memory cells
   709    890     ** containing the current result row. In this case regOrig is set to the
   710    891     ** same value. However, if the results are being sent to the sorter, the
   711    892     ** values for any expressions that are also part of the sort-key are omitted
   712    893     ** from this array. In this case regOrig is set to zero.  */
   713    894     int regResult;              /* Start of memory holding current results */
................................................................................
   746    927     regOrig = regResult = pDest->iSdst;
   747    928     if( srcTab>=0 ){
   748    929       for(i=0; i<nResultCol; i++){
   749    930         sqlite3VdbeAddOp3(v, OP_Column, srcTab, i, regResult+i);
   750    931         VdbeComment((v, "%s", p->pEList->a[i].zName));
   751    932       }
   752    933     }else if( eDest!=SRT_Exists ){
          934  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          935  +    ExprList *pExtra = 0;
          936  +#endif
   753    937       /* If the destination is an EXISTS(...) expression, the actual
   754    938       ** values returned by the SELECT are not required.
   755    939       */
   756         -    u8 ecelFlags;
          940  +    u8 ecelFlags;    /* "ecel" is an abbreviation of "ExprCodeExprList" */
          941  +    ExprList *pEList;
   757    942       if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){
   758    943         ecelFlags = SQLITE_ECEL_DUP;
   759    944       }else{
   760    945         ecelFlags = 0;
   761    946       }
   762    947       if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){
   763    948         /* For each expression in p->pEList that is a copy of an expression in
   764    949         ** the ORDER BY clause (pSort->pOrderBy), set the associated 
   765    950         ** iOrderByCol value to one more than the index of the ORDER BY 
   766    951         ** expression within the sort-key that pushOntoSorter() will generate.
   767    952         ** This allows the p->pEList field to be omitted from the sorted record,
   768    953         ** saving space and CPU cycles.  */
   769    954         ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF);
          955  +
   770    956         for(i=pSort->nOBSat; i<pSort->pOrderBy->nExpr; i++){
   771    957           int j;
   772    958           if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){
   773    959             p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat;
   774    960           }
   775    961         }
   776         -      regOrig = 0;
          962  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          963  +      selectExprDefer(pParse, pSort, p->pEList, &pExtra);
          964  +      if( pExtra && pParse->db->mallocFailed==0 ){
          965  +        /* If there are any extra PK columns to add to the sorter records,
          966  +        ** allocate extra memory cells and adjust the OpenEphemeral 
          967  +        ** instruction to account for the larger records. This is only
          968  +        ** required if there are one or more WITHOUT ROWID tables with
          969  +        ** composite primary keys in the SortCtx.aDefer[] array.  */
          970  +        VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
          971  +        pOp->p2 += (pExtra->nExpr - pSort->nDefer);
          972  +        pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer);
          973  +        pParse->nMem += pExtra->nExpr;
          974  +      }
          975  +#endif
          976  +
          977  +      /* Adjust nResultCol to account for columns that are omitted
          978  +      ** from the sorter by the optimizations in this branch */
          979  +      pEList = p->pEList;
          980  +      for(i=0; i<pEList->nExpr; i++){
          981  +        if( pEList->a[i].u.x.iOrderByCol>0
          982  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          983  +         || pEList->a[i].bSorterRef
          984  +#endif
          985  +        ){
          986  +          nResultCol--;
          987  +          regOrig = 0;
          988  +        }
          989  +      }
          990  +
          991  +      testcase( regOrig );
          992  +      testcase( eDest==SRT_Set );
          993  +      testcase( eDest==SRT_Mem );
          994  +      testcase( eDest==SRT_Coroutine );
          995  +      testcase( eDest==SRT_Output );
   777    996         assert( eDest==SRT_Set || eDest==SRT_Mem 
   778    997              || eDest==SRT_Coroutine || eDest==SRT_Output );
   779    998       }
   780         -    nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult,
   781         -                                         0,ecelFlags);
          999  +    sRowLoadInfo.regResult = regResult;
         1000  +    sRowLoadInfo.ecelFlags = ecelFlags;
         1001  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1002  +    sRowLoadInfo.pExtra = pExtra;
         1003  +    sRowLoadInfo.regExtraResult = regResult + nResultCol;
         1004  +    if( pExtra ) nResultCol += pExtra->nExpr;
         1005  +#endif
         1006  +    if( p->iLimit
         1007  +     && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 
         1008  +     && nPrefixReg>0
         1009  +    ){
         1010  +      assert( pSort!=0 );
         1011  +      assert( hasDistinct==0 );
         1012  +      pSort->pDeferredRowLoad = &sRowLoadInfo;
         1013  +    }else{
         1014  +      innerLoopLoadRow(pParse, p, &sRowLoadInfo);
         1015  +    }
   782   1016     }
   783   1017   
   784   1018     /* If the DISTINCT keyword was present on the SELECT statement
   785   1019     ** and this row has been seen before, then do not make this row
   786   1020     ** part of the result.
   787   1021     */
   788   1022     if( hasDistinct ){
................................................................................
   890   1124           sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0);
   891   1125           VdbeCoverage(v);
   892   1126           sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1,regResult,nResultCol);
   893   1127           assert( pSort==0 );
   894   1128         }
   895   1129   #endif
   896   1130         if( pSort ){
   897         -        pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg);
         1131  +        assert( regResult==regOrig );
         1132  +        pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg);
   898   1133         }else{
   899   1134           int r2 = sqlite3GetTempReg(pParse);
   900   1135           sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2);
   901   1136           sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2);
   902   1137           sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
   903   1138           sqlite3ReleaseTempReg(pParse, r2);
   904   1139         }
................................................................................
  1157   1392   **
  1158   1393   **   "USE TEMP B-TREE FOR xxx"
  1159   1394   **
  1160   1395   ** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which
  1161   1396   ** is determined by the zUsage argument.
  1162   1397   */
  1163   1398   static void explainTempTable(Parse *pParse, const char *zUsage){
  1164         -  if( pParse->explain==2 ){
  1165         -    Vdbe *v = pParse->pVdbe;
  1166         -    char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage);
  1167         -    sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
  1168         -  }
         1399  +  ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s", zUsage));
  1169   1400   }
  1170   1401   
  1171   1402   /*
  1172   1403   ** Assign expression b to lvalue a. A second, no-op, version of this macro
  1173   1404   ** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code
  1174   1405   ** in sqlite3Select() to assign values to structure member variables that
  1175   1406   ** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the
................................................................................
  1179   1410   
  1180   1411   #else
  1181   1412   /* No-op versions of the explainXXX() functions and macros. */
  1182   1413   # define explainTempTable(y,z)
  1183   1414   # define explainSetInteger(y,z)
  1184   1415   #endif
  1185   1416   
  1186         -#if !defined(SQLITE_OMIT_EXPLAIN) && !defined(SQLITE_OMIT_COMPOUND_SELECT)
  1187         -/*
  1188         -** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function
  1189         -** is a no-op. Otherwise, it adds a single row of output to the EQP result,
  1190         -** where the caption is of one of the two forms:
  1191         -**
  1192         -**   "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)"
  1193         -**   "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)"
  1194         -**
  1195         -** where iSub1 and iSub2 are the integers passed as the corresponding
  1196         -** function parameters, and op is the text representation of the parameter
  1197         -** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT,
  1198         -** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is 
  1199         -** false, or the second form if it is true.
  1200         -*/
  1201         -static void explainComposite(
  1202         -  Parse *pParse,                  /* Parse context */
  1203         -  int op,                         /* One of TK_UNION, TK_EXCEPT etc. */
  1204         -  int iSub1,                      /* Subquery id 1 */
  1205         -  int iSub2,                      /* Subquery id 2 */
  1206         -  int bUseTmp                     /* True if a temp table was used */
  1207         -){
  1208         -  assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL );
  1209         -  if( pParse->explain==2 ){
  1210         -    Vdbe *v = pParse->pVdbe;
  1211         -    char *zMsg = sqlite3MPrintf(
  1212         -        pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2,
  1213         -        bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op)
  1214         -    );
  1215         -    sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
  1216         -  }
  1217         -}
  1218         -#else
  1219         -/* No-op versions of the explainXXX() functions and macros. */
  1220         -# define explainComposite(v,w,x,y,z)
  1221         -#endif
  1222   1417   
  1223   1418   /*
  1224   1419   ** If the inner loop was generated using a non-null pOrderBy argument,
  1225   1420   ** then the results were placed in a sorter.  After the loop is terminated
  1226   1421   ** we need to run the sorter and output the results.  The following
  1227   1422   ** routine generates the code needed to do that.
  1228   1423   */
................................................................................
  1232   1427     SortCtx *pSort,   /* Information on the ORDER BY clause */
  1233   1428     int nColumn,      /* Number of columns of data */
  1234   1429     SelectDest *pDest /* Write the sorted results here */
  1235   1430   ){
  1236   1431     Vdbe *v = pParse->pVdbe;                     /* The prepared statement */
  1237   1432     int addrBreak = pSort->labelDone;            /* Jump here to exit loop */
  1238   1433     int addrContinue = sqlite3VdbeMakeLabel(v);  /* Jump here for next cycle */
  1239         -  int addr;
         1434  +  int addr;                       /* Top of output loop. Jump for Next. */
  1240   1435     int addrOnce = 0;
  1241   1436     int iTab;
  1242   1437     ExprList *pOrderBy = pSort->pOrderBy;
  1243   1438     int eDest = pDest->eDest;
  1244   1439     int iParm = pDest->iSDParm;
  1245   1440     int regRow;
  1246   1441     int regRowid;
  1247   1442     int iCol;
  1248         -  int nKey;
         1443  +  int nKey;                       /* Number of key columns in sorter record */
  1249   1444     int iSortTab;                   /* Sorter cursor to read from */
  1250         -  int nSortData;                  /* Trailing values to read from sorter */
  1251   1445     int i;
  1252   1446     int bSeq;                       /* True if sorter record includes seq. no. */
         1447  +  int nRefKey = 0;
  1253   1448     struct ExprList_item *aOutEx = p->pEList->a;
  1254   1449   
  1255   1450     assert( addrBreak<0 );
  1256   1451     if( pSort->labelBkOut ){
  1257   1452       sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
  1258   1453       sqlite3VdbeGoto(v, addrBreak);
  1259   1454       sqlite3VdbeResolveLabel(v, pSort->labelBkOut);
  1260   1455     }
         1456  +
         1457  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1458  +  /* Open any cursors needed for sorter-reference expressions */
         1459  +  for(i=0; i<pSort->nDefer; i++){
         1460  +    Table *pTab = pSort->aDefer[i].pTab;
         1461  +    int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
         1462  +    sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead);
         1463  +    nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey);
         1464  +  }
         1465  +#endif
         1466  +
  1261   1467     iTab = pSort->iECursor;
  1262   1468     if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){
  1263   1469       regRowid = 0;
  1264   1470       regRow = pDest->iSdst;
  1265         -    nSortData = nColumn;
  1266   1471     }else{
  1267   1472       regRowid = sqlite3GetTempReg(pParse);
  1268   1473       regRow = sqlite3GetTempRange(pParse, nColumn);
  1269         -    nSortData = nColumn;
  1270   1474     }
  1271   1475     nKey = pOrderBy->nExpr - pSort->nOBSat;
  1272   1476     if( pSort->sortFlags & SORTFLAG_UseSorter ){
  1273   1477       int regSortOut = ++pParse->nMem;
  1274   1478       iSortTab = pParse->nTab++;
  1275   1479       if( pSort->labelBkOut ){
  1276   1480         addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
  1277   1481       }
  1278         -    sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData);
         1482  +    sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, 
         1483  +        nKey+1+nColumn+nRefKey);
  1279   1484       if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
  1280   1485       addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
  1281   1486       VdbeCoverage(v);
  1282   1487       codeOffset(v, p->iOffset, addrContinue);
  1283   1488       sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab);
  1284   1489       bSeq = 0;
  1285   1490     }else{
  1286   1491       addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
  1287   1492       codeOffset(v, p->iOffset, addrContinue);
  1288   1493       iSortTab = iTab;
  1289   1494       bSeq = 1;
  1290   1495     }
  1291         -  for(i=0, iCol=nKey+bSeq-1; i<nSortData; i++){
         1496  +  for(i=0, iCol=nKey+bSeq-1; i<nColumn; i++){
         1497  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1498  +    if( aOutEx[i].bSorterRef ) continue;
         1499  +#endif
  1292   1500       if( aOutEx[i].u.x.iOrderByCol==0 ) iCol++;
  1293   1501     }
  1294         -  for(i=nSortData-1; i>=0; i--){
  1295         -    int iRead;
  1296         -    if( aOutEx[i].u.x.iOrderByCol ){
  1297         -      iRead = aOutEx[i].u.x.iOrderByCol-1;
  1298         -    }else{
  1299         -      iRead = iCol--;
  1300         -    }
  1301         -    sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
  1302         -    VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));
         1502  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1503  +  if( pSort->nDefer ){
         1504  +    int iKey = iCol+1;
         1505  +    int regKey = sqlite3GetTempRange(pParse, nRefKey);
         1506  +
         1507  +    for(i=0; i<pSort->nDefer; i++){
         1508  +      int iCsr = pSort->aDefer[i].iCsr;
         1509  +      Table *pTab = pSort->aDefer[i].pTab;
         1510  +      int nKey = pSort->aDefer[i].nKey;
         1511  +
         1512  +      sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
         1513  +      if( HasRowid(pTab) ){
         1514  +        sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey);
         1515  +        sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, 
         1516  +            sqlite3VdbeCurrentAddr(v)+1, regKey);
         1517  +      }else{
         1518  +        int k;
         1519  +        int iJmp;
         1520  +        assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey );
         1521  +        for(k=0; k<nKey; k++){
         1522  +          sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey+k);
         1523  +        }
         1524  +        iJmp = sqlite3VdbeCurrentAddr(v);
         1525  +        sqlite3VdbeAddOp4Int(v, OP_SeekGE, iCsr, iJmp+2, regKey, nKey);
         1526  +        sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp+3, regKey, nKey);
         1527  +        sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
         1528  +      }
         1529  +    }
         1530  +    sqlite3ReleaseTempRange(pParse, regKey, nRefKey);
         1531  +  }
         1532  +#endif
         1533  +  for(i=nColumn-1; i>=0; i--){
         1534  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1535  +    if( aOutEx[i].bSorterRef ){
         1536  +      sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i);
         1537  +    }else
         1538  +#endif
         1539  +    {
         1540  +      int iRead;
         1541  +      if( aOutEx[i].u.x.iOrderByCol ){
         1542  +        iRead = aOutEx[i].u.x.iOrderByCol-1;
         1543  +      }else{
         1544  +        iRead = iCol--;
         1545  +      }
         1546  +      sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
         1547  +      VdbeComment((v, "%s", aOutEx[i].zName?aOutEx[i].zName : aOutEx[i].zSpan));
         1548  +    }
  1303   1549     }
  1304   1550     switch( eDest ){
  1305   1551       case SRT_Table:
  1306   1552       case SRT_EphemTab: {
  1307   1553         sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid);
  1308   1554         sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid);
  1309   1555         sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
................................................................................
  1613   1859   #ifndef SQLITE_OMIT_EXPLAIN
  1614   1860     /* If this is an EXPLAIN, skip this step */
  1615   1861     if( pParse->explain ){
  1616   1862       return;
  1617   1863     }
  1618   1864   #endif
  1619   1865   
  1620         -  if( pParse->colNamesSet || db->mallocFailed ) return;
         1866  +  if( pParse->colNamesSet ) return;
  1621   1867     /* Column names are determined by the left-most term of a compound select */
  1622   1868     while( pSelect->pPrior ) pSelect = pSelect->pPrior;
  1623   1869     SELECTTRACE(1,pParse,pSelect,("generating column names\n"));
  1624   1870     pTabList = pSelect->pSrc;
  1625   1871     pEList = pSelect->pEList;
  1626   1872     assert( v!=0 );
  1627   1873     assert( pTabList!=0 );
................................................................................
  2140   2386     }
  2141   2387   
  2142   2388     /* Detach the ORDER BY clause from the compound SELECT */
  2143   2389     p->pOrderBy = 0;
  2144   2390   
  2145   2391     /* Store the results of the setup-query in Queue. */
  2146   2392     pSetup->pNext = 0;
         2393  +  ExplainQueryPlan((pParse, 1, "SETUP"));
  2147   2394     rc = sqlite3Select(pParse, pSetup, &destQueue);
  2148   2395     pSetup->pNext = p;
  2149   2396     if( rc ) goto end_of_recursive_query;
  2150   2397   
  2151   2398     /* Find the next row in the Queue and output that row */
  2152   2399     addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v);
  2153   2400   
................................................................................
  2174   2421     /* Execute the recursive SELECT taking the single row in Current as
  2175   2422     ** the value for the recursive-table. Store the results in the Queue.
  2176   2423     */
  2177   2424     if( p->selFlags & SF_Aggregate ){
  2178   2425       sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported");
  2179   2426     }else{
  2180   2427       p->pPrior = 0;
         2428  +    ExplainQueryPlan((pParse, 1, "RECURSIVE STEP"));
  2181   2429       sqlite3Select(pParse, p, &destQueue);
  2182   2430       assert( p->pPrior==0 );
  2183   2431       p->pPrior = pSetup;
  2184   2432     }
  2185   2433   
  2186   2434     /* Keep running the loop until the Queue is empty */
  2187   2435     sqlite3VdbeGoto(v, addrTop);
................................................................................
  2219   2467   ** Since the limit is exactly 1, we only need to evalutes the left-most VALUES.
  2220   2468   */
  2221   2469   static int multiSelectValues(
  2222   2470     Parse *pParse,        /* Parsing context */
  2223   2471     Select *p,            /* The right-most of SELECTs to be coded */
  2224   2472     SelectDest *pDest     /* What to do with query results */
  2225   2473   ){
  2226         -  Select *pPrior;
  2227         -  Select *pRightmost = p;
  2228   2474     int nRow = 1;
  2229   2475     int rc = 0;
         2476  +  int bShowAll = p->pLimit==0;
  2230   2477     assert( p->selFlags & SF_MultiValue );
  2231   2478     do{
  2232   2479       assert( p->selFlags & SF_Values );
  2233   2480       assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) );
  2234   2481       assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr );
  2235   2482       if( p->pPrior==0 ) break;
  2236   2483       assert( p->pPrior->pNext==p );
  2237   2484       p = p->pPrior;
  2238         -    nRow++;
         2485  +    nRow += bShowAll;
  2239   2486     }while(1);
         2487  +  ExplainQueryPlan((pParse, 0, "SCAN %d CONSTANT ROW%s", nRow,
         2488  +                    nRow==1 ? "" : "S"));
  2240   2489     while( p ){
  2241         -    pPrior = p->pPrior;
  2242         -    p->pPrior = 0;
  2243         -    rc = sqlite3Select(pParse, p, pDest);
  2244         -    p->pPrior = pPrior;
  2245         -    if( rc || pRightmost->pLimit ) break;
         2490  +    selectInnerLoop(pParse, p, -1, 0, 0, pDest, 1, 1);
         2491  +    if( !bShowAll ) break;
  2246   2492       p->nSelectRow = nRow;
  2247   2493       p = p->pNext;
  2248   2494     }
  2249   2495     return rc;
  2250   2496   }
  2251   2497   
  2252   2498   /*
................................................................................
  2287   2533   ){
  2288   2534     int rc = SQLITE_OK;   /* Success code from a subroutine */
  2289   2535     Select *pPrior;       /* Another SELECT immediately to our left */
  2290   2536     Vdbe *v;              /* Generate code to this VDBE */
  2291   2537     SelectDest dest;      /* Alternative data destination */
  2292   2538     Select *pDelete = 0;  /* Chain of simple selects to delete */
  2293   2539     sqlite3 *db;          /* Database connection */
  2294         -#ifndef SQLITE_OMIT_EXPLAIN
  2295         -  int iSub1 = 0;        /* EQP id of left-hand query */
  2296         -  int iSub2 = 0;        /* EQP id of right-hand query */
  2297         -#endif
  2298   2540   
  2299   2541     /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs.  Only
  2300   2542     ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT.
  2301   2543     */
  2302   2544     assert( p && p->pPrior );  /* Calling function guarantees this much */
  2303   2545     assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION );
  2304   2546     db = pParse->db;
................................................................................
  2341   2583     }else
  2342   2584   #endif
  2343   2585   
  2344   2586     /* Compound SELECTs that have an ORDER BY clause are handled separately.
  2345   2587     */
  2346   2588     if( p->pOrderBy ){
  2347   2589       return multiSelectOrderBy(pParse, p, pDest);
  2348         -  }else
  2349         -
  2350         -  /* Generate code for the left and right SELECT statements.
  2351         -  */
  2352         -  switch( p->op ){
  2353         -    case TK_ALL: {
  2354         -      int addr = 0;
  2355         -      int nLimit;
  2356         -      assert( !pPrior->pLimit );
  2357         -      pPrior->iLimit = p->iLimit;
  2358         -      pPrior->iOffset = p->iOffset;
  2359         -      pPrior->pLimit = p->pLimit;
  2360         -      explainSetInteger(iSub1, pParse->iNextSelectId);
  2361         -      rc = sqlite3Select(pParse, pPrior, &dest);
  2362         -      p->pLimit = 0;
  2363         -      if( rc ){
  2364         -        goto multi_select_end;
  2365         -      }
  2366         -      p->pPrior = 0;
  2367         -      p->iLimit = pPrior->iLimit;
  2368         -      p->iOffset = pPrior->iOffset;
  2369         -      if( p->iLimit ){
  2370         -        addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v);
  2371         -        VdbeComment((v, "Jump ahead if LIMIT reached"));
  2372         -        if( p->iOffset ){
  2373         -          sqlite3VdbeAddOp3(v, OP_OffsetLimit,
  2374         -                            p->iLimit, p->iOffset+1, p->iOffset);
  2375         -        }
  2376         -      }
  2377         -      explainSetInteger(iSub2, pParse->iNextSelectId);
  2378         -      rc = sqlite3Select(pParse, p, &dest);
  2379         -      testcase( rc!=SQLITE_OK );
  2380         -      pDelete = p->pPrior;
  2381         -      p->pPrior = pPrior;
  2382         -      p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
  2383         -      if( pPrior->pLimit
  2384         -       && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit)
  2385         -       && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) 
  2386         -      ){
  2387         -        p->nSelectRow = sqlite3LogEst((u64)nLimit);
  2388         -      }
  2389         -      if( addr ){
  2390         -        sqlite3VdbeJumpHere(v, addr);
  2391         -      }
  2392         -      break;
  2393         -    }
  2394         -    case TK_EXCEPT:
  2395         -    case TK_UNION: {
  2396         -      int unionTab;    /* Cursor number of the temporary table holding result */
  2397         -      u8 op = 0;       /* One of the SRT_ operations to apply to self */
  2398         -      int priorOp;     /* The SRT_ operation to apply to prior selects */
  2399         -      Expr *pLimit;    /* Saved values of p->nLimit  */
  2400         -      int addr;
  2401         -      SelectDest uniondest;
  2402         -
  2403         -      testcase( p->op==TK_EXCEPT );
  2404         -      testcase( p->op==TK_UNION );
  2405         -      priorOp = SRT_Union;
  2406         -      if( dest.eDest==priorOp ){
  2407         -        /* We can reuse a temporary table generated by a SELECT to our
  2408         -        ** right.
         2590  +  }else{
         2591  +
         2592  +#ifndef SQLITE_OMIT_EXPLAIN
         2593  +    if( pPrior->pPrior==0 ){
         2594  +      ExplainQueryPlan((pParse, 1, "COMPOUND QUERY"));
         2595  +      ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY"));
         2596  +    }
         2597  +#endif
         2598  +
         2599  +    /* Generate code for the left and right SELECT statements.
         2600  +    */
         2601  +    switch( p->op ){
         2602  +      case TK_ALL: {
         2603  +        int addr = 0;
         2604  +        int nLimit;
         2605  +        assert( !pPrior->pLimit );
         2606  +        pPrior->iLimit = p->iLimit;
         2607  +        pPrior->iOffset = p->iOffset;
         2608  +        pPrior->pLimit = p->pLimit;
         2609  +        rc = sqlite3Select(pParse, pPrior, &dest);
         2610  +        p->pLimit = 0;
         2611  +        if( rc ){
         2612  +          goto multi_select_end;
         2613  +        }
         2614  +        p->pPrior = 0;
         2615  +        p->iLimit = pPrior->iLimit;
         2616  +        p->iOffset = pPrior->iOffset;
         2617  +        if( p->iLimit ){
         2618  +          addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v);
         2619  +          VdbeComment((v, "Jump ahead if LIMIT reached"));
         2620  +          if( p->iOffset ){
         2621  +            sqlite3VdbeAddOp3(v, OP_OffsetLimit,
         2622  +                              p->iLimit, p->iOffset+1, p->iOffset);
         2623  +          }
         2624  +        }
         2625  +        ExplainQueryPlan((pParse, 1, "UNION ALL"));
         2626  +        rc = sqlite3Select(pParse, p, &dest);
         2627  +        testcase( rc!=SQLITE_OK );
         2628  +        pDelete = p->pPrior;
         2629  +        p->pPrior = pPrior;
         2630  +        p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
         2631  +        if( pPrior->pLimit
         2632  +         && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit)
         2633  +         && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) 
         2634  +        ){
         2635  +          p->nSelectRow = sqlite3LogEst((u64)nLimit);
         2636  +        }
         2637  +        if( addr ){
         2638  +          sqlite3VdbeJumpHere(v, addr);
         2639  +        }
         2640  +        break;
         2641  +      }
         2642  +      case TK_EXCEPT:
         2643  +      case TK_UNION: {
         2644  +        int unionTab;    /* Cursor number of the temp table holding result */
         2645  +        u8 op = 0;       /* One of the SRT_ operations to apply to self */
         2646  +        int priorOp;     /* The SRT_ operation to apply to prior selects */
         2647  +        Expr *pLimit;    /* Saved values of p->nLimit  */
         2648  +        int addr;
         2649  +        SelectDest uniondest;
         2650  +  
         2651  +        testcase( p->op==TK_EXCEPT );
         2652  +        testcase( p->op==TK_UNION );
         2653  +        priorOp = SRT_Union;
         2654  +        if( dest.eDest==priorOp ){
         2655  +          /* We can reuse a temporary table generated by a SELECT to our
         2656  +          ** right.
         2657  +          */
         2658  +          assert( p->pLimit==0 );      /* Not allowed on leftward elements */
         2659  +          unionTab = dest.iSDParm;
         2660  +        }else{
         2661  +          /* We will need to create our own temporary table to hold the
         2662  +          ** intermediate results.
         2663  +          */
         2664  +          unionTab = pParse->nTab++;
         2665  +          assert( p->pOrderBy==0 );
         2666  +          addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0);
         2667  +          assert( p->addrOpenEphm[0] == -1 );
         2668  +          p->addrOpenEphm[0] = addr;
         2669  +          findRightmost(p)->selFlags |= SF_UsesEphemeral;
         2670  +          assert( p->pEList );
         2671  +        }
         2672  +  
         2673  +        /* Code the SELECT statements to our left
         2674  +        */
         2675  +        assert( !pPrior->pOrderBy );
         2676  +        sqlite3SelectDestInit(&uniondest, priorOp, unionTab);
         2677  +        rc = sqlite3Select(pParse, pPrior, &uniondest);
         2678  +        if( rc ){
         2679  +          goto multi_select_end;
         2680  +        }
         2681  +  
         2682  +        /* Code the current SELECT statement
         2683  +        */
         2684  +        if( p->op==TK_EXCEPT ){
         2685  +          op = SRT_Except;
         2686  +        }else{
         2687  +          assert( p->op==TK_UNION );
         2688  +          op = SRT_Union;
         2689  +        }
         2690  +        p->pPrior = 0;
         2691  +        pLimit = p->pLimit;
         2692  +        p->pLimit = 0;
         2693  +        uniondest.eDest = op;
         2694  +        ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
         2695  +                          selectOpName(p->op)));
         2696  +        rc = sqlite3Select(pParse, p, &uniondest);
         2697  +        testcase( rc!=SQLITE_OK );
         2698  +        /* Query flattening in sqlite3Select() might refill p->pOrderBy.
         2699  +        ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */
         2700  +        sqlite3ExprListDelete(db, p->pOrderBy);
         2701  +        pDelete = p->pPrior;
         2702  +        p->pPrior = pPrior;
         2703  +        p->pOrderBy = 0;
         2704  +        if( p->op==TK_UNION ){
         2705  +          p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
         2706  +        }
         2707  +        sqlite3ExprDelete(db, p->pLimit);
         2708  +        p->pLimit = pLimit;
         2709  +        p->iLimit = 0;
         2710  +        p->iOffset = 0;
         2711  +  
         2712  +        /* Convert the data in the temporary table into whatever form
         2713  +        ** it is that we currently need.
         2714  +        */
         2715  +        assert( unionTab==dest.iSDParm || dest.eDest!=priorOp );
         2716  +        if( dest.eDest!=priorOp ){
         2717  +          int iCont, iBreak, iStart;
         2718  +          assert( p->pEList );
         2719  +          iBreak = sqlite3VdbeMakeLabel(v);
         2720  +          iCont = sqlite3VdbeMakeLabel(v);
         2721  +          computeLimitRegisters(pParse, p, iBreak);
         2722  +          sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v);
         2723  +          iStart = sqlite3VdbeCurrentAddr(v);
         2724  +          selectInnerLoop(pParse, p, unionTab,
         2725  +                          0, 0, &dest, iCont, iBreak);
         2726  +          sqlite3VdbeResolveLabel(v, iCont);
         2727  +          sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v);
         2728  +          sqlite3VdbeResolveLabel(v, iBreak);
         2729  +          sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0);
         2730  +        }
         2731  +        break;
         2732  +      }
         2733  +      default: assert( p->op==TK_INTERSECT ); {
         2734  +        int tab1, tab2;
         2735  +        int iCont, iBreak, iStart;
         2736  +        Expr *pLimit;
         2737  +        int addr;
         2738  +        SelectDest intersectdest;
         2739  +        int r1;
         2740  +  
         2741  +        /* INTERSECT is different from the others since it requires
         2742  +        ** two temporary tables.  Hence it has its own case.  Begin
         2743  +        ** by allocating the tables we will need.
  2409   2744           */
  2410         -        assert( p->pLimit==0 );      /* Not allowed on leftward elements */
  2411         -        unionTab = dest.iSDParm;
  2412         -      }else{
  2413         -        /* We will need to create our own temporary table to hold the
  2414         -        ** intermediate results.
  2415         -        */
  2416         -        unionTab = pParse->nTab++;
         2745  +        tab1 = pParse->nTab++;
         2746  +        tab2 = pParse->nTab++;
  2417   2747           assert( p->pOrderBy==0 );
  2418         -        addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0);
         2748  +  
         2749  +        addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0);
  2419   2750           assert( p->addrOpenEphm[0] == -1 );
  2420   2751           p->addrOpenEphm[0] = addr;
  2421   2752           findRightmost(p)->selFlags |= SF_UsesEphemeral;
  2422   2753           assert( p->pEList );
  2423         -      }
  2424         -
  2425         -      /* Code the SELECT statements to our left
  2426         -      */
  2427         -      assert( !pPrior->pOrderBy );
  2428         -      sqlite3SelectDestInit(&uniondest, priorOp, unionTab);
  2429         -      explainSetInteger(iSub1, pParse->iNextSelectId);
  2430         -      rc = sqlite3Select(pParse, pPrior, &uniondest);
  2431         -      if( rc ){
  2432         -        goto multi_select_end;
  2433         -      }
  2434         -
  2435         -      /* Code the current SELECT statement
  2436         -      */
  2437         -      if( p->op==TK_EXCEPT ){
  2438         -        op = SRT_Except;
  2439         -      }else{
  2440         -        assert( p->op==TK_UNION );
  2441         -        op = SRT_Union;
  2442         -      }
  2443         -      p->pPrior = 0;
  2444         -      pLimit = p->pLimit;
  2445         -      p->pLimit = 0;
  2446         -      uniondest.eDest = op;
  2447         -      explainSetInteger(iSub2, pParse->iNextSelectId);
  2448         -      rc = sqlite3Select(pParse, p, &uniondest);
  2449         -      testcase( rc!=SQLITE_OK );
  2450         -      /* Query flattening in sqlite3Select() might refill p->pOrderBy.
  2451         -      ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */
  2452         -      sqlite3ExprListDelete(db, p->pOrderBy);
  2453         -      pDelete = p->pPrior;
  2454         -      p->pPrior = pPrior;
  2455         -      p->pOrderBy = 0;
  2456         -      if( p->op==TK_UNION ){
  2457         -        p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
  2458         -      }
  2459         -      sqlite3ExprDelete(db, p->pLimit);
  2460         -      p->pLimit = pLimit;
  2461         -      p->iLimit = 0;
  2462         -      p->iOffset = 0;
  2463         -
  2464         -      /* Convert the data in the temporary table into whatever form
  2465         -      ** it is that we currently need.
  2466         -      */
  2467         -      assert( unionTab==dest.iSDParm || dest.eDest!=priorOp );
  2468         -      if( dest.eDest!=priorOp ){
  2469         -        int iCont, iBreak, iStart;
         2754  +  
         2755  +        /* Code the SELECTs to our left into temporary table "tab1".
         2756  +        */
         2757  +        sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1);
         2758  +        rc = sqlite3Select(pParse, pPrior, &intersectdest);
         2759  +        if( rc ){
         2760  +          goto multi_select_end;
         2761  +        }
         2762  +  
         2763  +        /* Code the current SELECT into temporary table "tab2"
         2764  +        */
         2765  +        addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0);
         2766  +        assert( p->addrOpenEphm[1] == -1 );
         2767  +        p->addrOpenEphm[1] = addr;
         2768  +        p->pPrior = 0;
         2769  +        pLimit = p->pLimit;
         2770  +        p->pLimit = 0;
         2771  +        intersectdest.iSDParm = tab2;
         2772  +        ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
         2773  +                          selectOpName(p->op)));
         2774  +        rc = sqlite3Select(pParse, p, &intersectdest);
         2775  +        testcase( rc!=SQLITE_OK );
         2776  +        pDelete = p->pPrior;
         2777  +        p->pPrior = pPrior;
         2778  +        if( p->nSelectRow>pPrior->nSelectRow ){
         2779  +          p->nSelectRow = pPrior->nSelectRow;
         2780  +        }
         2781  +        sqlite3ExprDelete(db, p->pLimit);
         2782  +        p->pLimit = pLimit;
         2783  +  
         2784  +        /* Generate code to take the intersection of the two temporary
         2785  +        ** tables.
         2786  +        */
  2470   2787           assert( p->pEList );
  2471   2788           iBreak = sqlite3VdbeMakeLabel(v);
  2472   2789           iCont = sqlite3VdbeMakeLabel(v);
  2473   2790           computeLimitRegisters(pParse, p, iBreak);
  2474         -        sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v);
  2475         -        iStart = sqlite3VdbeCurrentAddr(v);
  2476         -        selectInnerLoop(pParse, p, unionTab,
         2791  +        sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v);
         2792  +        r1 = sqlite3GetTempReg(pParse);
         2793  +        iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1);
         2794  +        sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0);
         2795  +        VdbeCoverage(v);
         2796  +        sqlite3ReleaseTempReg(pParse, r1);
         2797  +        selectInnerLoop(pParse, p, tab1,
  2477   2798                           0, 0, &dest, iCont, iBreak);
  2478   2799           sqlite3VdbeResolveLabel(v, iCont);
  2479         -        sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v);
         2800  +        sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v);
  2480   2801           sqlite3VdbeResolveLabel(v, iBreak);
  2481         -        sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0);
  2482         -      }
  2483         -      break;
  2484         -    }
  2485         -    default: assert( p->op==TK_INTERSECT ); {
  2486         -      int tab1, tab2;
  2487         -      int iCont, iBreak, iStart;
  2488         -      Expr *pLimit;
  2489         -      int addr;
  2490         -      SelectDest intersectdest;
  2491         -      int r1;
  2492         -
  2493         -      /* INTERSECT is different from the others since it requires
  2494         -      ** two temporary tables.  Hence it has its own case.  Begin
  2495         -      ** by allocating the tables we will need.
  2496         -      */
  2497         -      tab1 = pParse->nTab++;
  2498         -      tab2 = pParse->nTab++;
  2499         -      assert( p->pOrderBy==0 );
  2500         -
  2501         -      addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0);
  2502         -      assert( p->addrOpenEphm[0] == -1 );
  2503         -      p->addrOpenEphm[0] = addr;
  2504         -      findRightmost(p)->selFlags |= SF_UsesEphemeral;
  2505         -      assert( p->pEList );
  2506         -
  2507         -      /* Code the SELECTs to our left into temporary table "tab1".
  2508         -      */
  2509         -      sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1);
  2510         -      explainSetInteger(iSub1, pParse->iNextSelectId);
  2511         -      rc = sqlite3Select(pParse, pPrior, &intersectdest);
  2512         -      if( rc ){
  2513         -        goto multi_select_end;
  2514         -      }
  2515         -
  2516         -      /* Code the current SELECT into temporary table "tab2"
  2517         -      */
  2518         -      addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0);
  2519         -      assert( p->addrOpenEphm[1] == -1 );
  2520         -      p->addrOpenEphm[1] = addr;
  2521         -      p->pPrior = 0;
  2522         -      pLimit = p->pLimit;
  2523         -      p->pLimit = 0;
  2524         -      intersectdest.iSDParm = tab2;
  2525         -      explainSetInteger(iSub2, pParse->iNextSelectId);
  2526         -      rc = sqlite3Select(pParse, p, &intersectdest);
  2527         -      testcase( rc!=SQLITE_OK );
  2528         -      pDelete = p->pPrior;
  2529         -      p->pPrior = pPrior;
  2530         -      if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
  2531         -      sqlite3ExprDelete(db, p->pLimit);
  2532         -      p->pLimit = pLimit;
  2533         -
  2534         -      /* Generate code to take the intersection of the two temporary
  2535         -      ** tables.
  2536         -      */
  2537         -      assert( p->pEList );
  2538         -      iBreak = sqlite3VdbeMakeLabel(v);
  2539         -      iCont = sqlite3VdbeMakeLabel(v);
  2540         -      computeLimitRegisters(pParse, p, iBreak);
  2541         -      sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v);
  2542         -      r1 = sqlite3GetTempReg(pParse);
  2543         -      iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1);
  2544         -      sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v);
  2545         -      sqlite3ReleaseTempReg(pParse, r1);
  2546         -      selectInnerLoop(pParse, p, tab1,
  2547         -                      0, 0, &dest, iCont, iBreak);
  2548         -      sqlite3VdbeResolveLabel(v, iCont);
  2549         -      sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v);
  2550         -      sqlite3VdbeResolveLabel(v, iBreak);
  2551         -      sqlite3VdbeAddOp2(v, OP_Close, tab2, 0);
  2552         -      sqlite3VdbeAddOp2(v, OP_Close, tab1, 0);
  2553         -      break;
  2554         -    }
  2555         -  }
  2556         -
  2557         -  explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL);
  2558         -
         2802  +        sqlite3VdbeAddOp2(v, OP_Close, tab2, 0);
         2803  +        sqlite3VdbeAddOp2(v, OP_Close, tab1, 0);
         2804  +        break;
         2805  +      }
         2806  +    }
         2807  +  
         2808  +  #ifndef SQLITE_OMIT_EXPLAIN
         2809  +    if( p->pNext==0 ){
         2810  +      ExplainQueryPlanPop(pParse);
         2811  +    }
         2812  +  #endif
         2813  +  }
         2814  +  
  2559   2815     /* Compute collating sequences used by 
  2560   2816     ** temporary tables needed to implement the compound select.
  2561   2817     ** Attach the KeyInfo structure to all temporary tables.
  2562   2818     **
  2563   2819     ** This section is run by the right-most SELECT statement only.
  2564   2820     ** SELECT statements to the left always skip this part.  The right-most
  2565   2821     ** SELECT might also skip this part if it has no ORDER BY clause and
................................................................................
  2889   3145     int op;               /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */
  2890   3146     KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */
  2891   3147     KeyInfo *pKeyMerge;   /* Comparison information for merging rows */
  2892   3148     sqlite3 *db;          /* Database connection */
  2893   3149     ExprList *pOrderBy;   /* The ORDER BY clause */
  2894   3150     int nOrderBy;         /* Number of terms in the ORDER BY clause */
  2895   3151     int *aPermute;        /* Mapping from ORDER BY terms to result set columns */
  2896         -#ifndef SQLITE_OMIT_EXPLAIN
  2897         -  int iSub1;            /* EQP id of left-hand query */
  2898         -  int iSub2;            /* EQP id of right-hand query */
  2899         -#endif
  2900   3152   
  2901   3153     assert( p->pOrderBy!=0 );
  2902   3154     assert( pKeyDup==0 ); /* "Managed" code needs this.  Ticket #3382. */
  2903   3155     db = pParse->db;
  2904   3156     v = pParse->pVdbe;
  2905   3157     assert( v!=0 );       /* Already thrown the error if VDBE alloc failed */
  2906   3158     labelEnd = sqlite3VdbeMakeLabel(v);
................................................................................
  3012   3264     regAddrA = ++pParse->nMem;
  3013   3265     regAddrB = ++pParse->nMem;
  3014   3266     regOutA = ++pParse->nMem;
  3015   3267     regOutB = ++pParse->nMem;
  3016   3268     sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA);
  3017   3269     sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB);
  3018   3270   
         3271  +  ExplainQueryPlan((pParse, 1, "MERGE (%s)", selectOpName(p->op)));
         3272  +
  3019   3273     /* Generate a coroutine to evaluate the SELECT statement to the
  3020   3274     ** left of the compound operator - the "A" select.
  3021   3275     */
  3022   3276     addrSelectA = sqlite3VdbeCurrentAddr(v) + 1;
  3023   3277     addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA);
  3024   3278     VdbeComment((v, "left SELECT"));
  3025   3279     pPrior->iLimit = regLimitA;
  3026         -  explainSetInteger(iSub1, pParse->iNextSelectId);
         3280  +  ExplainQueryPlan((pParse, 1, "LEFT"));
  3027   3281     sqlite3Select(pParse, pPrior, &destA);
  3028   3282     sqlite3VdbeEndCoroutine(v, regAddrA);
  3029   3283     sqlite3VdbeJumpHere(v, addr1);
  3030   3284   
  3031   3285     /* Generate a coroutine to evaluate the SELECT statement on 
  3032   3286     ** the right - the "B" select
  3033   3287     */
................................................................................
  3034   3288     addrSelectB = sqlite3VdbeCurrentAddr(v) + 1;
  3035   3289     addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB);
  3036   3290     VdbeComment((v, "right SELECT"));
  3037   3291     savedLimit = p->iLimit;
  3038   3292     savedOffset = p->iOffset;
  3039   3293     p->iLimit = regLimitB;
  3040   3294     p->iOffset = 0;  
  3041         -  explainSetInteger(iSub2, pParse->iNextSelectId);
         3295  +  ExplainQueryPlan((pParse, 1, "RIGHT"));
  3042   3296     sqlite3Select(pParse, p, &destB);
  3043   3297     p->iLimit = savedLimit;
  3044   3298     p->iOffset = savedOffset;
  3045   3299     sqlite3VdbeEndCoroutine(v, regAddrB);
  3046   3300   
  3047   3301     /* Generate a subroutine that outputs the current row of the A
  3048   3302     ** select as the next output row of the compound select.
................................................................................
  3146   3400       sqlite3SelectDelete(db, p->pPrior);
  3147   3401     }
  3148   3402     p->pPrior = pPrior;
  3149   3403     pPrior->pNext = p;
  3150   3404   
  3151   3405     /*** TBD:  Insert subroutine calls to close cursors on incomplete
  3152   3406     **** subqueries ****/
  3153         -  explainComposite(pParse, p->op, iSub1, iSub2, 0);
         3407  +  ExplainQueryPlanPop(pParse);
  3154   3408     return pParse->nErr!=0;
  3155   3409   }
  3156   3410   #endif
  3157   3411   
  3158   3412   #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
  3159   3413   
  3160   3414   /* An instance of the SubstContext object describes an substitution edit
................................................................................
  3633   3887       if( pNew==0 ){
  3634   3888         p->pPrior = pPrior;
  3635   3889       }else{
  3636   3890         pNew->pPrior = pPrior;
  3637   3891         if( pPrior ) pPrior->pNext = pNew;
  3638   3892         pNew->pNext = p;
  3639   3893         p->pPrior = pNew;
  3640         -      SELECTTRACE(2,pParse,p,
  3641         -         ("compound-subquery flattener creates %s.%p as peer\n",
  3642         -         pNew->zSelName, pNew));
         3894  +      SELECTTRACE(2,pParse,p,("compound-subquery flattener"
         3895  +                              " creates %s.%p as peer\n",pNew->zSelName, pNew));
  3643   3896       }
  3644   3897       if( db->mallocFailed ) return 1;
  3645   3898     }
  3646   3899   
  3647   3900     /* Begin flattening the iFrom-th entry of the FROM clause 
  3648   3901     ** in the outer query.
  3649   3902     */
................................................................................
  4935   5188   static void explainSimpleCount(
  4936   5189     Parse *pParse,                  /* Parse context */
  4937   5190     Table *pTab,                    /* Table being queried */
  4938   5191     Index *pIdx                     /* Index used to optimize scan, or NULL */
  4939   5192   ){
  4940   5193     if( pParse->explain==2 ){
  4941   5194       int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx)));
  4942         -    char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s%s%s",
         5195  +    sqlite3VdbeExplain(pParse, 0, "SCAN TABLE %s%s%s",
  4943   5196           pTab->zName,
  4944   5197           bCover ? " USING COVERING INDEX " : "",
  4945   5198           bCover ? pIdx->zName : ""
  4946   5199       );
  4947         -    sqlite3VdbeAddOp4(
  4948         -        pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC
  4949         -    );
  4950   5200     }
  4951   5201   }
  4952   5202   #else
  4953   5203   # define explainSimpleCount(a,b,c)
  4954   5204   #endif
  4955   5205   
  4956   5206   /*
................................................................................
  5155   5405     SortCtx sSort;         /* Info on how to code the ORDER BY clause */
  5156   5406     AggInfo sAggInfo;      /* Information used by aggregate queries */
  5157   5407     int iEnd;              /* Address of the end of the query */
  5158   5408     sqlite3 *db;           /* The database connection */
  5159   5409     ExprList *pMinMaxOrderBy = 0;  /* Added ORDER BY for min/max queries */
  5160   5410     u8 minMaxFlag;                 /* Flag for min/max queries */
  5161   5411   
  5162         -#ifndef SQLITE_OMIT_EXPLAIN
  5163         -  int iRestoreSelectId = pParse->iSelectId;
  5164         -  pParse->iSelectId = pParse->iNextSelectId++;
  5165         -#endif
  5166         -
  5167   5412     db = pParse->db;
         5413  +  v = sqlite3GetVdbe(pParse);
  5168   5414     if( p==0 || db->mallocFailed || pParse->nErr ){
  5169   5415       return 1;
  5170   5416     }
  5171   5417     if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1;
  5172   5418     memset(&sAggInfo, 0, sizeof(sAggInfo));
  5173   5419   #if SELECTTRACE_ENABLED
  5174         -  SELECTTRACE(1,pParse,p, ("begin processing:\n"));
         5420  +  SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain));
  5175   5421     if( sqlite3SelectTrace & 0x100 ){
  5176   5422       sqlite3TreeViewSelect(0, p, 0);
  5177   5423     }
  5178   5424   #endif
  5179   5425   
  5180   5426     assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo );
  5181   5427     assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo );
................................................................................
  5198   5444     pTabList = p->pSrc;
  5199   5445     if( pParse->nErr || db->mallocFailed ){
  5200   5446       goto select_end;
  5201   5447     }
  5202   5448     assert( p->pEList!=0 );
  5203   5449     isAgg = (p->selFlags & SF_Aggregate)!=0;
  5204   5450   #if SELECTTRACE_ENABLED
  5205         -  if( sqlite3SelectTrace & 0x100 ){
  5206         -    SELECTTRACE(0x100,pParse,p, ("after name resolution:\n"));
         5451  +  if( sqlite3SelectTrace & 0x104 ){
         5452  +    SELECTTRACE(0x104,pParse,p, ("after name resolution:\n"));
  5207   5453       sqlite3TreeViewSelect(0, p, 0);
  5208   5454     }
  5209   5455   #endif
  5210   5456   
  5211         -  /* Get a pointer the VDBE under construction, allocating a new VDBE if one
  5212         -  ** does not already exist */
  5213         -  v = sqlite3GetVdbe(pParse);
  5214         -  if( v==0 ) goto select_end;
  5215   5457     if( pDest->eDest==SRT_Output ){
  5216   5458       generateColumnNames(pParse, p);
  5217   5459     }
  5218   5460   
  5219   5461     /* Try to various optimizations (flattening subqueries, and strength
  5220   5462     ** reduction of join operators) in the FROM clause up into the main query
  5221   5463     */
................................................................................
  5300   5542   
  5301   5543   #ifndef SQLITE_OMIT_COMPOUND_SELECT
  5302   5544     /* Handle compound SELECT statements using the separate multiSelect()
  5303   5545     ** procedure.
  5304   5546     */
  5305   5547     if( p->pPrior ){
  5306   5548       rc = multiSelect(pParse, p, pDest);
  5307         -    explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  5308   5549   #if SELECTTRACE_ENABLED
  5309         -    SELECTTRACE(1,pParse,p,("end compound-select processing\n"));
         5550  +    SELECTTRACE(0x1,pParse,p,("end compound-select processing\n"));
         5551  +    if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){
         5552  +      sqlite3TreeViewSelect(0, p, 0);
         5553  +    }
  5310   5554   #endif
         5555  +    if( p->pNext==0 ) ExplainQueryPlanPop(pParse);
  5311   5556       return rc;
  5312   5557     }
  5313   5558   #endif
  5314   5559   
  5315   5560     /* For each term in the FROM clause, do two things:
  5316   5561     ** (1) Authorized unreferenced tables
  5317   5562     ** (2) Generate code for all sub-queries
................................................................................
  5415   5660         int addrTop = sqlite3VdbeCurrentAddr(v)+1;
  5416   5661        
  5417   5662         pItem->regReturn = ++pParse->nMem;
  5418   5663         sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop);
  5419   5664         VdbeComment((v, "%s", pItem->pTab->zName));
  5420   5665         pItem->addrFillSub = addrTop;
  5421   5666         sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn);
  5422         -      explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
         5667  +      ExplainQueryPlan((pParse, 1, "CO-ROUTINE 0x%p", pSub));
  5423   5668         sqlite3Select(pParse, pSub, &dest);
  5424   5669         pItem->pTab->nRowLogEst = pSub->nSelectRow;
  5425   5670         pItem->fg.viaCoroutine = 1;
  5426   5671         pItem->regResult = dest.iSdst;
  5427   5672         sqlite3VdbeEndCoroutine(v, pItem->regReturn);
  5428   5673         sqlite3VdbeJumpHere(v, addrTop-1);
  5429   5674         sqlite3ClearTempRegCache(pParse);
................................................................................
  5450   5695           VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName));
  5451   5696         }else{
  5452   5697           VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName));
  5453   5698         }
  5454   5699         pPrior = isSelfJoinView(pTabList, pItem);
  5455   5700         if( pPrior ){
  5456   5701           sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor);
  5457         -        explainSetInteger(pItem->iSelectId, pPrior->iSelectId);
  5458   5702           assert( pPrior->pSelect!=0 );
  5459   5703           pSub->nSelectRow = pPrior->pSelect->nSelectRow;
  5460   5704         }else{
  5461   5705           sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
  5462         -        explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
         5706  +        ExplainQueryPlan((pParse, 1, "MATERIALIZE 0x%p", pSub));
  5463   5707           sqlite3Select(pParse, pSub, &dest);
  5464   5708         }
  5465   5709         pItem->pTab->nRowLogEst = pSub->nSelectRow;
  5466   5710         if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);
  5467   5711         retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn);
  5468   5712         VdbeComment((v, "end %s", pItem->pTab->zName));
  5469   5713         sqlite3VdbeChangeP1(v, topAddr, retAddr);
................................................................................
  5682   5926       /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in
  5683   5927       ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the
  5684   5928       ** SELECT statement.
  5685   5929       */
  5686   5930       memset(&sNC, 0, sizeof(sNC));
  5687   5931       sNC.pParse = pParse;
  5688   5932       sNC.pSrcList = pTabList;
  5689         -    sNC.pAggInfo = &sAggInfo;
         5933  +    sNC.uNC.pAggInfo = &sAggInfo;
         5934  +    VVA_ONLY( sNC.ncFlags = NC_UAggInfo; )
  5690   5935       sAggInfo.mnReg = pParse->nMem+1;
  5691   5936       sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
  5692   5937       sAggInfo.pGroupBy = pGroupBy;
  5693   5938       sqlite3ExprAnalyzeAggList(&sNC, pEList);
  5694   5939       sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy);
  5695   5940       if( pHaving ){
  5696   5941         if( pGroupBy ){
................................................................................
  6071   6316   
  6072   6317     /* If there is an ORDER BY clause, then we need to sort the results
  6073   6318     ** and send them to the callback one by one.
  6074   6319     */
  6075   6320     if( sSort.pOrderBy ){
  6076   6321       explainTempTable(pParse,
  6077   6322                        sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY");
         6323  +    assert( p->pEList==pEList );
  6078   6324       generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest);
  6079   6325     }
  6080   6326   
  6081   6327     /* Jump here to skip this query
  6082   6328     */
  6083   6329     sqlite3VdbeResolveLabel(v, iEnd);
  6084   6330   
................................................................................
  6086   6332     ** set the return code to 1. Otherwise 0. */
  6087   6333     rc = (pParse->nErr>0);
  6088   6334   
  6089   6335     /* Control jumps to here if an error is encountered above, or upon
  6090   6336     ** successful coding of the SELECT.
  6091   6337     */
  6092   6338   select_end:
  6093         -  explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  6094   6339     sqlite3ExprListDelete(db, pMinMaxOrderBy);
  6095   6340     sqlite3DbFree(db, sAggInfo.aCol);
  6096   6341     sqlite3DbFree(db, sAggInfo.aFunc);
  6097   6342   #if SELECTTRACE_ENABLED
  6098         -  SELECTTRACE(1,pParse,p,("end processing\n"));
         6343  +  SELECTTRACE(0x1,pParse,p,("end processing\n"));
         6344  +  if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){
         6345  +    sqlite3TreeViewSelect(0, p, 0);
         6346  +  }
  6099   6347   #endif
         6348  +  ExplainQueryPlanPop(pParse);
  6100   6349     return rc;
  6101   6350   }

Changes to src/shell.c.in.

   431    431   /*
   432    432   ** Render output like fprintf().  This should not be used on anything that
   433    433   ** includes string formatting (e.g. "%s").
   434    434   */
   435    435   #if !defined(raw_printf)
   436    436   # define raw_printf fprintf
   437    437   #endif
          438  +
          439  +/* Indicate out-of-memory and exit. */
          440  +static void shell_out_of_memory(void){
          441  +  raw_printf(stderr,"Error: out of memory\n");
          442  +  exit(1);
          443  +}
   438    444   
   439    445   /*
   440    446   ** Write I/O traces to the following stream.
   441    447   */
   442    448   #ifdef SQLITE_ENABLE_IOTRACE
   443    449   static FILE *iotrace = 0;
   444    450   #endif
................................................................................
   764    770   ** because it contains non-alphanumeric characters, or because it is an
   765    771   ** SQLite keyword.  Be conservative in this estimate:  When in doubt assume
   766    772   ** that quoting is required.
   767    773   **
   768    774   ** Return '"' if quoting is required.  Return 0 if no quoting is required.
   769    775   */
   770    776   static char quoteChar(const char *zName){
   771         -  /* All SQLite keywords, in alphabetical order */
   772         -  static const char *azKeywords[] = {
   773         -    "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
   774         -    "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
   775         -    "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
   776         -    "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
   777         -    "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
   778         -    "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
   779         -    "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
   780         -    "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
   781         -    "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
   782         -    "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
   783         -    "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
   784         -    "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
   785         -    "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
   786         -    "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
   787         -    "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
   788         -    "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
   789         -    "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
   790         -    "WITH", "WITHOUT",
   791         -  };
   792         -  int i, lwr, upr, mid, c;
          777  +  int i;
   793    778     if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
   794    779     for(i=0; zName[i]; i++){
   795    780       if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
   796    781     }
   797         -  lwr = 0;
   798         -  upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
   799         -  while( lwr<=upr ){
   800         -    mid = (lwr+upr)/2;
   801         -    c = sqlite3_stricmp(azKeywords[mid], zName);
   802         -    if( c==0 ) return '"';
   803         -    if( c<0 ){
   804         -      lwr = mid+1;
   805         -    }else{
   806         -      upr = mid-1;
   807         -    }
   808         -  }
   809         -  return 0;
          782  +  return sqlite3_keyword_check(zName, i) ? '"' : 0;
   810    783   }
   811    784   
   812    785   /*
   813    786   ** Construct a fake object name and column list to describe the structure
   814    787   ** of the view, virtual table, or table valued function zSchema.zName.
   815    788   */
   816    789   static char *shellFakeSchema(
................................................................................
  1001    974   
  1002    975   typedef struct ExpertInfo ExpertInfo;
  1003    976   struct ExpertInfo {
  1004    977     sqlite3expert *pExpert;
  1005    978     int bVerbose;
  1006    979   };
  1007    980   
          981  +/* A single line in the EQP output */
          982  +typedef struct EQPGraphRow EQPGraphRow;
          983  +struct EQPGraphRow {
          984  +  int iEqpId;           /* ID for this row */
          985  +  int iParentId;        /* ID of the parent row */
          986  +  EQPGraphRow *pNext;   /* Next row in sequence */
          987  +  char zText[1];        /* Text to display for this row */
          988  +};
          989  +
          990  +/* All EQP output is collected into an instance of the following */
          991  +typedef struct EQPGraph EQPGraph;
          992  +struct EQPGraph {
          993  +  EQPGraphRow *pRow;    /* Linked list of all rows of the EQP output */
          994  +  EQPGraphRow *pLast;   /* Last element of the pRow list */
          995  +  char zPrefix[100];    /* Graph prefix */
          996  +};
          997  +
  1008    998   /*
  1009    999   ** State information about the database connection is contained in an
  1010   1000   ** instance of the following structure.
  1011   1001   */
  1012   1002   typedef struct ShellState ShellState;
  1013   1003   struct ShellState {
  1014   1004     sqlite3 *db;           /* The database */
  1015   1005     u8 autoExplain;        /* Automatically turn on .explain mode */
  1016   1006     u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
         1007  +  u8 autoEQPtest;        /* autoEQP is in test mode */
  1017   1008     u8 statsOn;            /* True to display memory stats before each finalize */
  1018   1009     u8 scanstatsOn;        /* True to display scan stats before each finalize */
  1019   1010     u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  1020   1011     u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
         1012  +  u8 nEqpLevel;          /* Depth of the EQP output graph */
         1013  +  unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  1021   1014     int outCount;          /* Revert to stdout when reaching zero */
  1022   1015     int cnt;               /* Number of records displayed so far */
  1023   1016     FILE *out;             /* Write results here */
  1024   1017     FILE *traceOut;        /* Output for sqlite3_trace() */
  1025   1018     int nErr;              /* Number of errors seen */
  1026   1019     int mode;              /* An output mode setting */
  1027   1020     int modePrior;         /* Saved mode */
................................................................................
  1047   1040     char *zFreeOnClose;         /* Filename to free when closing */
  1048   1041     const char *zVfs;           /* Name of VFS to use */
  1049   1042     sqlite3_stmt *pStmt;   /* Current statement if any. */
  1050   1043     FILE *pLog;            /* Write log output here */
  1051   1044     int *aiIndent;         /* Array of indents used in MODE_Explain */
  1052   1045     int nIndent;           /* Size of array aiIndent[] */
  1053   1046     int iIndent;           /* Index of current op in aiIndent[] */
         1047  +  EQPGraph sGraph;       /* Information for the graphical EXPLAIN QUERY PLAN */
  1054   1048   #if defined(SQLITE_ENABLE_SESSION)
  1055   1049     int nSession;             /* Number of active sessions */
  1056   1050     OpenSession aSession[4];  /* Array of sessions.  [0] is in focus. */
  1057   1051   #endif
  1058   1052     ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
  1059   1053   };
  1060   1054   
  1061   1055   
  1062   1056   /* Allowed values for ShellState.autoEQP
  1063   1057   */
  1064         -#define AUTOEQP_off      0
  1065         -#define AUTOEQP_on       1
  1066         -#define AUTOEQP_trigger  2
  1067         -#define AUTOEQP_full     3
         1058  +#define AUTOEQP_off      0           /* Automatic EXPLAIN QUERY PLAN is off */
         1059  +#define AUTOEQP_on       1           /* Automatic EQP is on */
         1060  +#define AUTOEQP_trigger  2           /* On and also show plans for triggers */
         1061  +#define AUTOEQP_full     3           /* Show full EXPLAIN */
  1068   1062   
  1069   1063   /* Allowed values for ShellState.openMode
  1070   1064   */
  1071   1065   #define SHELL_OPEN_UNSPEC     0      /* No open-mode specified */
  1072   1066   #define SHELL_OPEN_NORMAL     1      /* Normal database file */
  1073   1067   #define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
  1074   1068   #define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
................................................................................
  1103   1097   #define MODE_Insert   5  /* Generate SQL "insert" statements */
  1104   1098   #define MODE_Quote    6  /* Quote values as for SQL */
  1105   1099   #define MODE_Tcl      7  /* Generate ANSI-C or TCL quoted elements */
  1106   1100   #define MODE_Csv      8  /* Quote strings, numbers are plain */
  1107   1101   #define MODE_Explain  9  /* Like MODE_Column, but do not truncate data */
  1108   1102   #define MODE_Ascii   10  /* Use ASCII unit and record separators (0x1F/0x1E) */
  1109   1103   #define MODE_Pretty  11  /* Pretty-print schemas */
         1104  +#define MODE_EQP     12  /* Converts EXPLAIN QUERY PLAN output into a graph */
  1110   1105   
  1111   1106   static const char *modeDescr[] = {
  1112   1107     "line",
  1113   1108     "column",
  1114   1109     "list",
  1115   1110     "semi",
  1116   1111     "html",
................................................................................
  1117   1112     "insert",
  1118   1113     "quote",
  1119   1114     "tcl",
  1120   1115     "csv",
  1121   1116     "explain",
  1122   1117     "ascii",
  1123   1118     "prettyprint",
         1119  +  "eqp"
  1124   1120   };
  1125   1121   
  1126   1122   /*
  1127   1123   ** These are the column/row/line separators used by the various
  1128   1124   ** import/export modes.
  1129   1125   */
  1130   1126   #define SEP_Column    "|"
................................................................................
  1665   1661       if( z[i]=='\n' ) return 1;
  1666   1662       if( IsSpace(z[i]) ) continue;
  1667   1663       if( z[i]=='-' && z[i+1]=='-' ) return 1;
  1668   1664       return 0;
  1669   1665     }
  1670   1666     return 1;
  1671   1667   }
  1672         -    
         1668  +
         1669  +/*
         1670  +** Add a new entry to the EXPLAIN QUERY PLAN data
         1671  +*/
         1672  +static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){
         1673  +  EQPGraphRow *pNew;
         1674  +  int nText = strlen30(zText);
         1675  +  if( p->autoEQPtest ){
         1676  +    utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText);
         1677  +  }
         1678  +  pNew = sqlite3_malloc64( sizeof(*pNew) + nText );
         1679  +  if( pNew==0 ) shell_out_of_memory();
         1680  +  pNew->iEqpId = iEqpId;
         1681  +  pNew->iParentId = p2;
         1682  +  memcpy(pNew->zText, zText, nText+1);
         1683  +  pNew->pNext = 0;
         1684  +  if( p->sGraph.pLast ){
         1685  +    p->sGraph.pLast->pNext = pNew;
         1686  +  }else{
         1687  +    p->sGraph.pRow = pNew;
         1688  +  }
         1689  +  p->sGraph.pLast = pNew;
         1690  +}
         1691  +
         1692  +/*
         1693  +** Free and reset the EXPLAIN QUERY PLAN data that has been collected
         1694  +** in p->sGraph.
         1695  +*/
         1696  +static void eqp_reset(ShellState *p){
         1697  +  EQPGraphRow *pRow, *pNext;
         1698  +  for(pRow = p->sGraph.pRow; pRow; pRow = pNext){
         1699  +    pNext = pRow->pNext;
         1700  +    sqlite3_free(pRow);
         1701  +  }
         1702  +  memset(&p->sGraph, 0, sizeof(p->sGraph));
         1703  +}
         1704  +
         1705  +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after
         1706  +** pOld, or return the first such line if pOld is NULL
         1707  +*/
         1708  +static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){
         1709  +  EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow;
         1710  +  while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext;
         1711  +  return pRow;
         1712  +}
         1713  +
         1714  +/* Render a single level of the graph that has iEqpId as its parent.  Called
         1715  +** recursively to render sublevels.
         1716  +*/
         1717  +static void eqp_render_level(ShellState *p, int iEqpId){
         1718  +  EQPGraphRow *pRow, *pNext;
         1719  +  int n = strlen30(p->sGraph.zPrefix);
         1720  +  char *z;
         1721  +  for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){
         1722  +    pNext = eqp_next_row(p, iEqpId, pRow);
         1723  +    z = pRow->zText;
         1724  +    utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z);
         1725  +    if( n<sizeof(p->sGraph.zPrefix)-7 ){
         1726  +      memcpy(&p->sGraph.zPrefix[n], pNext ? "|  " : "   ", 4);
         1727  +      eqp_render_level(p, pRow->iEqpId);
         1728  +      p->sGraph.zPrefix[n] = 0;
         1729  +    }
         1730  +  }
         1731  +}
         1732  +
         1733  +/*
         1734  +** Display and reset the EXPLAIN QUERY PLAN data
         1735  +*/
         1736  +static void eqp_render(ShellState *p){
         1737  +  EQPGraphRow *pRow = p->sGraph.pRow;
         1738  +  if( pRow ){
         1739  +    if( pRow->zText[0]=='-' ){
         1740  +      if( pRow->pNext==0 ){
         1741  +        eqp_reset(p);
         1742  +        return;
         1743  +      }
         1744  +      utf8_printf(p->out, "%s\n", pRow->zText+3);
         1745  +      p->sGraph.pRow = pRow->pNext;
         1746  +      sqlite3_free(pRow);
         1747  +    }else{
         1748  +      utf8_printf(p->out, "QUERY PLAN\n");
         1749  +    }
         1750  +    p->sGraph.zPrefix[0] = 0;
         1751  +    eqp_render_level(p, 0);
         1752  +    eqp_reset(p);
         1753  +  }
         1754  +}
  1673   1755   
  1674   1756   /*
  1675   1757   ** This is the callback routine that the shell
  1676   1758   ** invokes for each row of a query result.
  1677   1759   */
  1678   1760   static int shell_callback(
  1679   1761     void *pArg,
................................................................................
  2016   2098         for(i=0; i<nArg; i++){
  2017   2099           if( i>0 ) utf8_printf(p->out, "%s", p->colSeparator);
  2018   2100           utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue);
  2019   2101         }
  2020   2102         utf8_printf(p->out, "%s", p->rowSeparator);
  2021   2103         break;
  2022   2104       }
         2105  +    case MODE_EQP: {
         2106  +      eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]);
         2107  +      break;
         2108  +    }
  2023   2109     }
  2024   2110     return 0;
  2025   2111   }
  2026   2112   
  2027   2113   /*
  2028   2114   ** This is the callback routine that the SQLite library
  2029   2115   ** invokes for each row of a query result.
................................................................................
  2114   2200       p->zDestTable = 0;
  2115   2201     }
  2116   2202     if( zName==0 ) return;
  2117   2203     cQuote = quoteChar(zName);
  2118   2204     n = strlen30(zName);
  2119   2205     if( cQuote ) n += n+2;
  2120   2206     z = p->zDestTable = malloc( n+1 );
  2121         -  if( z==0 ){
  2122         -    raw_printf(stderr,"Error: out of memory\n");
  2123         -    exit(1);
  2124         -  }
         2207  +  if( z==0 ) shell_out_of_memory();
  2125   2208     n = 0;
  2126   2209     if( cQuote ) z[n++] = cQuote;
  2127   2210     for(i=0; zName[i]; i++){
  2128   2211       z[n++] = zName[i];
  2129   2212       if( zName[i]==cQuote ) z[n++] = cQuote;
  2130   2213     }
  2131   2214     if( cQuote ) z[n++] = cQuote;
................................................................................
  2857   2940           if( pArg->autoEQP>=AUTOEQP_trigger ){
  2858   2941             sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
  2859   2942           }
  2860   2943           zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
  2861   2944           rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
  2862   2945           if( rc==SQLITE_OK ){
  2863   2946             while( sqlite3_step(pExplain)==SQLITE_ROW ){
  2864         -            raw_printf(pArg->out,"--EQP-- %d,",sqlite3_column_int(pExplain, 0));
  2865         -            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 1));
  2866         -            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 2));
  2867         -            utf8_printf(pArg->out,"%s\n", sqlite3_column_text(pExplain, 3));
         2947  +            const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
         2948  +            int iEqpId = sqlite3_column_int(pExplain, 0);
         2949  +            int iParentId = sqlite3_column_int(pExplain, 1);
         2950  +            if( zEQPLine[0]=='-' ) eqp_render(pArg);
         2951  +            eqp_append(pArg, iEqpId, iParentId, zEQPLine);
  2868   2952             }
         2953  +          eqp_render(pArg);
  2869   2954           }
  2870   2955           sqlite3_finalize(pExplain);
  2871   2956           sqlite3_free(zEQP);
  2872   2957           if( pArg->autoEQP>=AUTOEQP_full ){
  2873   2958             /* Also do an EXPLAIN for ".eqp full" mode */
  2874   2959             zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
  2875   2960             rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
................................................................................
  2889   2974             sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  2890   2975           }
  2891   2976           restore_debug_trace_modes();
  2892   2977         }
  2893   2978   
  2894   2979         if( pArg ){
  2895   2980           pArg->cMode = pArg->mode;
  2896         -        if( pArg->autoExplain
  2897         -         && sqlite3_column_count(pStmt)==8
  2898         -         && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0
  2899         -        ){
  2900         -          pArg->cMode = MODE_Explain;
         2981  +        if( pArg->autoExplain ){
         2982  +          if( sqlite3_column_count(pStmt)==8
         2983  +           && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0
         2984  +          ){
         2985  +            pArg->cMode = MODE_Explain;
         2986  +          }
         2987  +          if( sqlite3_column_count(pStmt)==4
         2988  +           && sqlite3_strlike("EXPLAIN QUERY PLAN%", zStmtSql,0)==0 ){
         2989  +            pArg->cMode = MODE_EQP;
         2990  +          }
  2901   2991           }
  2902   2992   
  2903   2993           /* If the shell is currently in ".explain" mode, gather the extra
  2904   2994           ** data required to add indents to the output.*/
  2905   2995           if( pArg->cMode==MODE_Explain ){
  2906   2996             explain_data_prepare(pArg, pStmt);
  2907   2997           }
  2908   2998         }
  2909   2999   
  2910   3000         exec_prepared_stmt(pArg, pStmt);
  2911   3001         explain_data_delete(pArg);
         3002  +      eqp_render(pArg);
  2912   3003   
  2913   3004         /* print usage stats if stats on */
  2914   3005         if( pArg && pArg->statsOn ){
  2915   3006           display_stats(db, pArg, 0);
  2916   3007         }
  2917   3008   
  2918   3009         /* print loop-counters if required */
................................................................................
  2982   3073     rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  2983   3074     sqlite3_free(zSql);
  2984   3075     if( rc ) return 0;
  2985   3076     while( sqlite3_step(pStmt)==SQLITE_ROW ){
  2986   3077       if( nCol>=nAlloc-2 ){
  2987   3078         nAlloc = nAlloc*2 + nCol + 10;
  2988   3079         azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
  2989         -      if( azCol==0 ){
  2990         -        raw_printf(stderr, "Error: out of memory\n");
  2991         -        exit(1);
  2992         -      }
         3080  +      if( azCol==0 ) shell_out_of_memory();
  2993   3081       }
  2994   3082       azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
  2995   3083       if( sqlite3_column_int(pStmt, 5) ){
  2996   3084         nPK++;
  2997   3085         if( nPK==1
  2998   3086          && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2),
  2999   3087                             "INTEGER")==0
................................................................................
  3238   3326     ".bail on|off           Stop after hitting an error.  Default OFF\n"
  3239   3327     ".binary on|off         Turn binary output on or off.  Default OFF\n"
  3240   3328     ".cd DIRECTORY          Change the working directory to DIRECTORY\n"
  3241   3329     ".changes on|off        Show number of rows changed by SQL\n"
  3242   3330     ".check GLOB            Fail if output since .testcase does not match\n"
  3243   3331     ".clone NEWDB           Clone data into NEWDB from the existing database\n"
  3244   3332     ".databases             List names and files of attached databases\n"
         3333  +  ".dbconfig ?op? ?val?   List or change sqlite3_db_config() options\n"
  3245   3334     ".dbinfo ?DB?           Show status information about the database\n"
  3246   3335     ".dump ?TABLE? ...      Dump the database in an SQL text format\n"
  3247   3336     "                         If TABLE specified, only dump tables matching\n"
  3248   3337     "                         LIKE pattern TABLE.\n"
  3249   3338     ".echo on|off           Turn command echo on or off\n"
  3250   3339     ".eqp on|off|full       Enable or disable automatic EXPLAIN QUERY PLAN\n"
  3251   3340     ".excel                 Display the output of next command in a spreadsheet\n"
................................................................................
  3480   3569   
  3481   3570   /*
  3482   3571   ** Make sure the database is open.  If it is not, then open it.  If
  3483   3572   ** the database fails to open, print an error message and exit.
  3484   3573   */
  3485   3574   static void open_db(ShellState *p, int keepAlive){
  3486   3575     if( p->db==0 ){
  3487         -    sqlite3_initialize();
  3488   3576       if( p->openMode==SHELL_OPEN_UNSPEC && access(p->zDbFilename,0)==0 ){
  3489   3577         p->openMode = (u8)deduceDatabaseType(p->zDbFilename, 0);
  3490   3578       }
  3491   3579       switch( p->openMode ){
  3492   3580         case SHELL_OPEN_APPENDVFS: {
  3493   3581           sqlite3_open_v2(p->zDbFilename, &p->db, 
  3494   3582              SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
................................................................................
  3783   3871   };
  3784   3872   
  3785   3873   /* Append a single byte to z[] */
  3786   3874   static void import_append_char(ImportCtx *p, int c){
  3787   3875     if( p->n+1>=p->nAlloc ){
  3788   3876       p->nAlloc += p->nAlloc + 100;
  3789   3877       p->z = sqlite3_realloc64(p->z, p->nAlloc);
  3790         -    if( p->z==0 ){
  3791         -      raw_printf(stderr, "out of memory\n");
  3792         -      exit(1);
  3793         -    }
         3878  +    if( p->z==0 ) shell_out_of_memory();
  3794   3879     }
  3795   3880     p->z[p->n++] = (char)c;
  3796   3881   }
  3797   3882   
  3798   3883   /* Read a single field of CSV text.  Compatible with rfc4180 and extended
  3799   3884   ** with the option of having a separator other than ",".
  3800   3885   **
................................................................................
  3947   4032       utf8_printf(stderr, "Error %d: %s on [%s]\n",
  3948   4033               sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
  3949   4034               zQuery);
  3950   4035       goto end_data_xfer;
  3951   4036     }
  3952   4037     n = sqlite3_column_count(pQuery);
  3953   4038     zInsert = sqlite3_malloc64(200 + nTable + n*3);
  3954         -  if( zInsert==0 ){
  3955         -    raw_printf(stderr, "out of memory\n");
  3956         -    goto end_data_xfer;
  3957         -  }
         4039  +  if( zInsert==0 ) shell_out_of_memory();
  3958   4040     sqlite3_snprintf(200+nTable,zInsert,
  3959   4041                      "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable);
  3960   4042     i = strlen30(zInsert);
  3961   4043     for(j=1; j<n; j++){
  3962   4044       memcpy(zInsert+i, ",?", 2);
  3963   4045       i += 2;
  3964   4046     }
................................................................................
  4291   4373   */
  4292   4374   static int shellDatabaseError(sqlite3 *db){
  4293   4375     const char *zErr = sqlite3_errmsg(db);
  4294   4376     utf8_printf(stderr, "Error: %s\n", zErr);
  4295   4377     return 1;
  4296   4378   }
  4297   4379   
  4298         -/*
  4299         -** Print an out-of-memory message to stderr and return 1.
  4300         -*/
  4301         -static int shellNomemError(void){
  4302         -  raw_printf(stderr, "Error: out of memory\n");
  4303         -  return 1;
  4304         -}
  4305         -
  4306   4380   /*
  4307   4381   ** Compare the pattern in zGlob[] against the text in z[].  Return TRUE
  4308   4382   ** if they match and FALSE (0) if they do not match.
  4309   4383   **
  4310   4384   ** Globbing rules:
  4311   4385   **
  4312   4386   **      '*'       Matches any sequence of zero or more characters.
................................................................................
  5695   5769       if( zErrMsg ){
  5696   5770         utf8_printf(stderr,"Error: %s\n", zErrMsg);
  5697   5771         sqlite3_free(zErrMsg);
  5698   5772         rc = 1;
  5699   5773       }
  5700   5774     }else
  5701   5775   
  5702         -  if( c=='d' && strncmp(azArg[0], "dbinfo", n)==0 ){
         5776  +  if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){
         5777  +    static const struct DbConfigChoices {const char *zName; int op;} aDbConfig[] = {
         5778  +        { "enable_fkey",      SQLITE_DBCONFIG_ENABLE_FKEY            },
         5779  +        { "enable_trigger",   SQLITE_DBCONFIG_ENABLE_TRIGGER         },
         5780  +        { "fts3_tokenizer",   SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER  },
         5781  +        { "load_extension",   SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION  },
         5782  +        { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE       },
         5783  +        { "enable_qpsg",      SQLITE_DBCONFIG_ENABLE_QPSG            },
         5784  +        { "trigger_eqp",      SQLITE_DBCONFIG_TRIGGER_EQP            },
         5785  +        { "reset_database",   SQLITE_DBCONFIG_RESET_DATABASE         },
         5786  +    };
         5787  +    int ii, v;
         5788  +    open_db(p, 0);
         5789  +    for(ii=0; ii<ArraySize(aDbConfig); ii++){
         5790  +      if( nArg>1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue;
         5791  +      if( nArg>=3 ){
         5792  +        sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0);
         5793  +      }
         5794  +      sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v);
         5795  +      utf8_printf(p->out, "%18s %s\n", aDbConfig[ii].zName, v ? "on" : "off");
         5796  +      if( nArg>1 ) break;
         5797  +    }
         5798  +    if( nArg>1 && ii==ArraySize(aDbConfig) ){
         5799  +      utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]);
         5800  +      utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n");
         5801  +    }   
         5802  +  }else
         5803  +
         5804  +  if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){
  5703   5805       rc = shell_dbinfo_command(p, nArg, azArg);
  5704   5806     }else
  5705   5807   
  5706   5808     if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){
  5707   5809       const char *zLike = 0;
  5708   5810       int i;
  5709   5811       int savedShowHeader = p->showHeader;
................................................................................
  5798   5900         raw_printf(stderr, "Usage: .echo on|off\n");
  5799   5901         rc = 1;
  5800   5902       }
  5801   5903     }else
  5802   5904   
  5803   5905     if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){
  5804   5906       if( nArg==2 ){
         5907  +      p->autoEQPtest = 0;
  5805   5908         if( strcmp(azArg[1],"full")==0 ){
  5806   5909           p->autoEQP = AUTOEQP_full;
  5807   5910         }else if( strcmp(azArg[1],"trigger")==0 ){
  5808   5911           p->autoEQP = AUTOEQP_trigger;
         5912  +      }else if( strcmp(azArg[1],"test")==0 ){
         5913  +        p->autoEQP = AUTOEQP_on;
         5914  +        p->autoEQPtest = 1;
  5809   5915         }else{
  5810   5916           p->autoEQP = (u8)booleanValue(azArg[1]);
  5811   5917         }
  5812   5918       }else{
  5813   5919         raw_printf(stderr, "Usage: .eqp off|on|trigger|full\n");
  5814   5920         rc = 1;
  5815   5921       }
................................................................................
  5992   6098         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
  5993   6099         return 1;
  5994   6100       }
  5995   6101       sCtx.cColSep = p->colSeparator[0];
  5996   6102       sCtx.cRowSep = p->rowSeparator[0];
  5997   6103       zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
  5998   6104       if( zSql==0 ){
  5999         -      raw_printf(stderr, "Error: out of memory\n");
  6000   6105         xCloser(sCtx.in);
  6001         -      return 1;
         6106  +      shell_out_of_memory();
  6002   6107       }
  6003   6108       nByte = strlen30(zSql);
  6004   6109       rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  6005   6110       import_append_char(&sCtx, 0);    /* To ensure sCtx.z is allocated */
  6006   6111       if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){
  6007   6112         char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable);
  6008   6113         char cSep = '(';
................................................................................
  6039   6144       }
  6040   6145       nCol = sqlite3_column_count(pStmt);
  6041   6146       sqlite3_finalize(pStmt);
  6042   6147       pStmt = 0;
  6043   6148       if( nCol==0 ) return 0; /* no columns, no error */
  6044   6149       zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 );
  6045   6150       if( zSql==0 ){
  6046         -      raw_printf(stderr, "Error: out of memory\n");
  6047   6151         xCloser(sCtx.in);
  6048         -      return 1;
         6152  +      shell_out_of_memory();
  6049   6153       }
  6050   6154       sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
  6051   6155       j = strlen30(zSql);
  6052   6156       for(i=1; i<nCol; i++){
  6053   6157         zSql[j++] = ',';
  6054   6158         zSql[j++] = '?';
  6055   6159       }
................................................................................
  6117   6221   #ifndef SQLITE_UNTESTABLE
  6118   6222     if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){
  6119   6223       char *zSql;
  6120   6224       char *zCollist = 0;
  6121   6225       sqlite3_stmt *pStmt;
  6122   6226       int tnum = 0;
  6123   6227       int i;
  6124         -    if( nArg!=3 ){
  6125         -      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n");
         6228  +    if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){
         6229  +      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n"
         6230  +                          "       .imposter off\n");
  6126   6231         rc = 1;
  6127   6232         goto meta_command_exit;
  6128   6233       }
  6129   6234       open_db(p, 0);
         6235  +    if( nArg==2 ){
         6236  +      sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1);
         6237  +      goto meta_command_exit;
         6238  +    }
  6130   6239       zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_master"
  6131   6240                              " WHERE name='%q' AND type='index'", azArg[1]);
  6132   6241       sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  6133   6242       sqlite3_free(zSql);
  6134   6243       if( sqlite3_step(pStmt)==SQLITE_ROW ){
  6135   6244         tnum = sqlite3_column_int(pStmt, 0);
  6136   6245       }
................................................................................
  7307   7416         sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC);
  7308   7417       }
  7309   7418       while( sqlite3_step(pStmt)==SQLITE_ROW ){
  7310   7419         if( nRow>=nAlloc ){
  7311   7420           char **azNew;
  7312   7421           int n2 = nAlloc*2 + 10;
  7313   7422           azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2);
  7314         -        if( azNew==0 ){
  7315         -          rc = shellNomemError();
  7316         -          break;
  7317         -        }
         7423  +        if( azNew==0 ) shell_out_of_memory();
  7318   7424           nAlloc = n2;
  7319   7425           azResult = azNew;
  7320   7426         }
  7321   7427         azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
  7322         -      if( 0==azResult[nRow] ){
  7323         -        rc = shellNomemError();
  7324         -        break;
  7325         -      }
         7428  +      if( 0==azResult[nRow] ) shell_out_of_memory();
  7326   7429         nRow++;
  7327   7430       }
  7328   7431       if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
  7329   7432         rc = shellDatabaseError(p->db);
  7330   7433       }
  7331   7434   
  7332   7435       /* Pretty-print the contents of array azResult[] to the output */
................................................................................
  7379   7482         { "always",             SQLITE_TESTCTRL_ALWAYS,        "BOOLEAN"            },
  7380   7483         { "assert",             SQLITE_TESTCTRL_ASSERT,        "BOOLEAN"            },
  7381   7484       /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS, ""          },*/
  7382   7485       /*{ "bitvec_test",        SQLITE_TESTCTRL_BITVEC_TEST,   ""                },*/
  7383   7486         { "byteorder",          SQLITE_TESTCTRL_BYTEORDER,     ""                   },
  7384   7487       /*{ "fault_install",      SQLITE_TESTCTRL_FAULT_INSTALL, ""                }, */
  7385   7488         { "imposter",           SQLITE_TESTCTRL_IMPOSTER,   "SCHEMA ON/OFF ROOTPAGE"},
  7386         -#ifdef SQLITE_N_KEYWORD
  7387         -      { "iskeyword",          SQLITE_TESTCTRL_ISKEYWORD,     "IDENTIFIER"         },
  7388         -#endif
  7389   7489         { "localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN"           },
  7390   7490         { "never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN"            },
  7391   7491         { "optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK"       },
  7392   7492   #ifdef YYCOVERAGE
  7393   7493         { "parser_coverage",    SQLITE_TESTCTRL_PARSER_COVERAGE, ""                 },
  7394   7494   #endif
  7395   7495         { "pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,  "OFFSET  "           },
................................................................................
  7493   7593             if( nArg==3 ){
  7494   7594               int opt = booleanValue(azArg[2]);
  7495   7595               rc2 = sqlite3_test_control(testctrl, opt);
  7496   7596               isOk = 3;
  7497   7597             }
  7498   7598             break;
  7499   7599   
  7500         -        /* sqlite3_test_control(int, char *) */
  7501         -#ifdef SQLITE_N_KEYWORD
  7502         -        case SQLITE_TESTCTRL_ISKEYWORD:
  7503         -          if( nArg==3 ){
  7504         -            const char *opt = azArg[2];
  7505         -            rc2 = sqlite3_test_control(testctrl, opt);
  7506         -            isOk = 1;
  7507         -          }
  7508         -          break;
  7509         -#endif
  7510         -
  7511   7600           case SQLITE_TESTCTRL_IMPOSTER:
  7512   7601             if( nArg==5 ){
  7513   7602               rc2 = sqlite3_test_control(testctrl, p->db,
  7514   7603                             azArg[2],
  7515   7604                             integerValue(azArg[3]),
  7516   7605                             integerValue(azArg[4]));
  7517   7606               isOk = 3;
................................................................................
  7889   7978       if( line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){
  7890   7979         memcpy(zLine,";",2);
  7891   7980       }
  7892   7981       nLine = strlen30(zLine);
  7893   7982       if( nSql+nLine+2>=nAlloc ){
  7894   7983         nAlloc = nSql+nLine+100;
  7895   7984         zSql = realloc(zSql, nAlloc);
  7896         -      if( zSql==0 ){
  7897         -        raw_printf(stderr, "Error: out of memory\n");
  7898         -        exit(1);
  7899         -      }
         7985  +      if( zSql==0 ) shell_out_of_memory();
  7900   7986       }
  7901   7987       nSqlPrior = nSql;
  7902   7988       if( nSql==0 ){
  7903   7989         int i;
  7904   7990         for(i=0; zLine[i] && IsSpace(zLine[i]); i++){}
  7905   7991         assert( nAlloc>0 && zSql!=0 );
  7906   7992         memcpy(zSql, zLine+i, nLine+1-i);
................................................................................
  8021   8107     if (sqliterc == NULL) {
  8022   8108       home_dir = find_home_dir(0);
  8023   8109       if( home_dir==0 ){
  8024   8110         raw_printf(stderr, "-- warning: cannot find home directory;"
  8025   8111                         " cannot read ~/.sqliterc\n");
  8026   8112         return;
  8027   8113       }
  8028         -    sqlite3_initialize();
  8029   8114       zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir);
  8030   8115       sqliterc = zBuf;
  8031   8116     }
  8032   8117     in = fopen(sqliterc,"rb");
  8033   8118     if( in ){
  8034   8119       if( stdin_is_interactive ){
  8035   8120         utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc);
................................................................................
  8072   8157   #endif
  8073   8158     "   -newline SEP         set output row separator. Default: '\\n'\n"
  8074   8159     "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
  8075   8160     "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
  8076   8161     "   -quote               set output mode to 'quote'\n"
  8077   8162     "   -readonly            open the database read-only\n"
  8078   8163     "   -separator SEP       set output column separator. Default: '|'\n"
         8164  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         8165  +  "   -sorterref SIZE      sorter references threshold size\n"
         8166  +#endif
  8079   8167     "   -stats               print memory stats before each finalize\n"
  8080   8168     "   -version             show SQLite version\n"
  8081   8169     "   -vfs NAME            use NAME as the default VFS\n"
  8082   8170   #ifdef SQLITE_ENABLE_VFSTRACE
  8083   8171     "   -vfstrace            enable tracing of all VFS calls\n"
  8084   8172   #endif
  8085   8173   #ifdef SQLITE_HAVE_ZLIB
................................................................................
  8094   8182     if( showDetail ){
  8095   8183       utf8_printf(stderr, "OPTIONS include:\n%s", zOptions);
  8096   8184     }else{
  8097   8185       raw_printf(stderr, "Use the -help option for additional information\n");
  8098   8186     }
  8099   8187     exit(1);
  8100   8188   }
         8189  +
         8190  +/*
         8191  +** Internal check:  Verify that the SQLite is uninitialized.  Print a
         8192  +** error message if it is initialized.
         8193  +*/
         8194  +static void verify_uninitialized(void){
         8195  +  if( sqlite3_config(-1)==SQLITE_MISUSE ){
         8196  +    utf8_printf(stdout, "WARNING: attempt to configuration SQLite after"
         8197  +                        " initialization.\n");
         8198  +  }
         8199  +}
  8101   8200   
  8102   8201   /*
  8103   8202   ** Initialize the state information in data
  8104   8203   */
  8105   8204   static void main_init(ShellState *data) {
  8106   8205     memset(data, 0, sizeof(*data));
  8107   8206     data->normalMode = data->cMode = data->mode = MODE_List;
  8108   8207     data->autoExplain = 1;
  8109   8208     memcpy(data->colSeparator,SEP_Column, 2);
  8110   8209     memcpy(data->rowSeparator,SEP_Row, 2);
  8111   8210     data->showHeader = 0;
  8112   8211     data->shellFlgs = SHFLG_Lookaside;
         8212  +  verify_uninitialized();
  8113   8213     sqlite3_config(SQLITE_CONFIG_URI, 1);
  8114   8214     sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data);
  8115   8215     sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  8116   8216     sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> ");
  8117   8217     sqlite3_snprintf(sizeof(continuePrompt), continuePrompt,"   ...> ");
  8118   8218   }
  8119   8219   
................................................................................
  8169   8269     const char *zInitFile = 0;
  8170   8270     int i;
  8171   8271     int rc = 0;
  8172   8272     int warnInmemoryDb = 0;
  8173   8273     int readStdin = 1;
  8174   8274     int nCmd = 0;
  8175   8275     char **azCmd = 0;
         8276  +  const char *zVfs = 0;           /* Value of -vfs command-line option */
  8176   8277   
  8177   8278     setBinaryMode(stdin, 0);
  8178   8279     setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
  8179   8280     stdin_is_interactive = isatty(0);
  8180   8281     stdout_is_console = isatty(1);
  8181   8282   
  8182   8283   #if USE_SYSTEM_SQLITE+0!=1
................................................................................
  8193   8294     ** do this.  But we want to run an sqlite3_shutdown() afterwards so that
  8194   8295     ** subsequent sqlite3_config() calls will work.  So copy all results into
  8195   8296     ** memory that does not come from the SQLite memory allocator.
  8196   8297     */
  8197   8298   #if !SQLITE_SHELL_IS_UTF8
  8198   8299     sqlite3_initialize();
  8199   8300     argv = malloc(sizeof(argv[0])*argc);
  8200         -  if( argv==0 ){
  8201         -    raw_printf(stderr, "out of memory\n");
  8202         -    exit(1);
  8203         -  }
         8301  +  if( argv==0 ) shell_out_of_memory();
  8204   8302     for(i=0; i<argc; i++){
  8205   8303       char *z = sqlite3_win32_unicode_to_utf8(wargv[i]);
  8206   8304       int n;
  8207         -    if( z==0 ){
  8208         -      raw_printf(stderr, "out of memory\n");
  8209         -      exit(1);
  8210         -    }
         8305  +    if( z==0 ) shell_out_of_memory();
  8211   8306       n = (int)strlen(z);
  8212   8307       argv[i] = malloc( n+1 );
  8213         -    if( argv[i]==0 ){
  8214         -      raw_printf(stderr, "out of memory\n");
  8215         -      exit(1);
  8216         -    }
         8308  +    if( argv[i]==0 ) shell_out_of_memory();
  8217   8309       memcpy(argv[i], z, n+1);
  8218   8310       sqlite3_free(z);
  8219   8311     }
  8220   8312     sqlite3_shutdown();
  8221   8313   #endif
  8222   8314   
  8223   8315     assert( argc>=1 && argv && argv[0] );
................................................................................
  8245   8337   #endif
  8246   8338   
  8247   8339     /* Do an initial pass through the command-line argument to locate
  8248   8340     ** the name of the database file, the name of the initialization file,
  8249   8341     ** the size of the alternative malloc heap,
  8250   8342     ** and the first command to execute.
  8251   8343     */
         8344  +  verify_uninitialized();
  8252   8345     for(i=1; i<argc; i++){
  8253   8346       char *z;
  8254   8347       z = argv[i];
  8255   8348       if( z[0]!='-' ){
  8256   8349         if( data.zDbFilename==0 ){
  8257   8350           data.zDbFilename = z;
  8258   8351         }else{
  8259   8352           /* Excesss arguments are interpreted as SQL (or dot-commands) and
  8260   8353           ** mean that nothing is read from stdin */
  8261   8354           readStdin = 0;
  8262   8355           nCmd++;
  8263   8356           azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd);
  8264         -        if( azCmd==0 ){
  8265         -          raw_printf(stderr, "out of memory\n");
  8266         -          exit(1);
  8267         -        }
         8357  +        if( azCmd==0 ) shell_out_of_memory();
  8268   8358           azCmd[nCmd-1] = z;
  8269   8359         }
  8270   8360       }
  8271   8361       if( z[1]=='-' ) z++;
  8272   8362       if( strcmp(z,"-separator")==0
  8273   8363        || strcmp(z,"-nullvalue")==0
  8274   8364        || strcmp(z,"-newline")==0
................................................................................
  8327   8417       }else if( strcmp(z,"-multiplex")==0 ){
  8328   8418         extern int sqlite3_multiple_initialize(const char*,int);
  8329   8419         sqlite3_multiplex_initialize(0, 1);
  8330   8420   #endif
  8331   8421       }else if( strcmp(z,"-mmap")==0 ){
  8332   8422         sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
  8333   8423         sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz);
         8424  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         8425  +    }else if( strcmp(z,"-sorterref")==0 ){
         8426  +      sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
         8427  +      sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz);
         8428  +#endif
  8334   8429       }else if( strcmp(z,"-vfs")==0 ){
  8335         -      sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i));
  8336         -      if( pVfs ){
  8337         -        sqlite3_vfs_register(pVfs, 1);
  8338         -      }else{
  8339         -        utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
  8340         -        exit(1);
  8341         -      }
         8430  +      zVfs = cmdline_option_value(argc, argv, ++i);
  8342   8431   #ifdef SQLITE_HAVE_ZLIB
  8343   8432       }else if( strcmp(z,"-zip")==0 ){
  8344   8433         data.openMode = SHELL_OPEN_ZIPFILE;
  8345   8434   #endif
  8346   8435       }else if( strcmp(z,"-append")==0 ){
  8347   8436         data.openMode = SHELL_OPEN_APPENDVFS;
  8348   8437       }else if( strcmp(z,"-readonly")==0 ){
................................................................................
  8351   8440       }else if( strncmp(z, "-A",2)==0 ){
  8352   8441         /* All remaining command-line arguments are passed to the ".archive"
  8353   8442         ** command, so ignore them */
  8354   8443         break;
  8355   8444   #endif
  8356   8445       }
  8357   8446     }
         8447  +  verify_uninitialized();
         8448  +
         8449  +
         8450  +#ifdef SQLITE_SHELL_INIT_PROC
         8451  +  {
         8452  +    /* If the SQLITE_SHELL_INIT_PROC macro is defined, then it is the name
         8453  +    ** of a C-function that will perform initialization actions on SQLite that
         8454  +    ** occur just before or after sqlite3_initialize(). Use this compile-time
         8455  +    ** option to embed this shell program in larger applications. */
         8456  +    extern void SQLITE_SHELL_INIT_PROC(void);
         8457  +    SQLITE_SHELL_INIT_PROC();
         8458  +  }
         8459  +#else
         8460  +  /* All the sqlite3_config() calls have now been made. So it is safe
         8461  +  ** to call sqlite3_initialize() and process any command line -vfs option. */
         8462  +  sqlite3_initialize();
         8463  +#endif
         8464  +
         8465  +  if( zVfs ){
         8466  +    sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs);
         8467  +    if( pVfs ){
         8468  +      sqlite3_vfs_register(pVfs, 1);
         8469  +    }else{
         8470  +      utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
         8471  +      exit(1);
         8472  +    }
         8473  +  }
         8474  +
  8358   8475     if( data.zDbFilename==0 ){
  8359   8476   #ifndef SQLITE_OMIT_MEMORYDB
  8360   8477       data.zDbFilename = ":memory:";
  8361   8478       warnInmemoryDb = argc==1;
  8362   8479   #else
  8363   8480       utf8_printf(stderr,"%s: Error: no database filename specified\n", Argv0);
  8364   8481       return 1;
................................................................................
  8463   8580         i++;
  8464   8581       }else if( strcmp(z,"-pagecache")==0 ){
  8465   8582         i+=2;
  8466   8583       }else if( strcmp(z,"-lookaside")==0 ){
  8467   8584         i+=2;
  8468   8585       }else if( strcmp(z,"-mmap")==0 ){
  8469   8586         i++;
         8587  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         8588  +    }else if( strcmp(z,"-sorterref")==0 ){
         8589  +      i++;
         8590  +#endif
  8470   8591       }else if( strcmp(z,"-vfs")==0 ){
  8471   8592         i++;
  8472   8593   #ifdef SQLITE_ENABLE_VFSTRACE
  8473   8594       }else if( strcmp(z,"-vfstrace")==0 ){
  8474   8595         i++;
  8475   8596   #endif
  8476   8597   #ifdef SQLITE_ENABLE_MULTIPLEX

Changes to src/sqlite.h.in.

  1927   1927   ** Or if the threshold is -1, statement journals are always held
  1928   1928   ** exclusively in memory.
  1929   1929   ** Since many statement journals never become large, setting the spill
  1930   1930   ** threshold to a value such as 64KiB can greatly reduce the amount of
  1931   1931   ** I/O required to support statement rollback.
  1932   1932   ** The default value for this setting is controlled by the
  1933   1933   ** [SQLITE_STMTJRNL_SPILL] compile-time option.
         1934  +**
         1935  +** [[SQLITE_CONFIG_SORTERREF_SIZE]]
         1936  +** <dt>SQLITE_CONFIG_SORTERREF_SIZE
         1937  +** <dd>The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter
         1938  +** of type (int) - the new value of the sorter-reference size threshold.
         1939  +** Usually, when SQLite uses an external sort to order records according
         1940  +** to an ORDER BY clause, all fields required by the caller are present in the
         1941  +** sorted records. However, if SQLite determines based on the declared type
         1942  +** of a table column that its values are likely to be very large - larger
         1943  +** than the configured sorter-reference size threshold - then a reference
         1944  +** is stored in each sorted record and the required column values loaded
         1945  +** from the database as records are returned in sorted order. The default
         1946  +** value for this option is to never use this optimization. Specifying a 
         1947  +** negative value for this option restores the default behaviour.
         1948  +** This option is only available if SQLite is compiled with the
         1949  +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option.
  1934   1950   ** </dl>
  1935   1951   */
  1936   1952   #define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
  1937   1953   #define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
  1938   1954   #define SQLITE_CONFIG_SERIALIZED    3  /* nil */
  1939   1955   #define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
  1940   1956   #define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
................................................................................
  1956   1972   #define SQLITE_CONFIG_SQLLOG       21  /* xSqllog, void* */
  1957   1973   #define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
  1958   1974   #define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
  1959   1975   #define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
  1960   1976   #define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
  1961   1977   #define SQLITE_CONFIG_STMTJRNL_SPILL      26  /* int nByte */
  1962   1978   #define SQLITE_CONFIG_SMALL_MALLOC        27  /* boolean */
         1979  +#define SQLITE_CONFIG_SORTERREF_SIZE      28  /* int nByte */
  1963   1980   
  1964   1981   /*
  1965   1982   ** CAPI3REF: Database Connection Configuration Options
  1966   1983   **
  1967   1984   ** These constants are the available integer configuration options that
  1968   1985   ** can be passed as the second argument to the [sqlite3_db_config()] interface.
  1969   1986   **
................................................................................
  2092   2109   ** behavior. The first parameter passed to this operation is an integer -
  2093   2110   ** positive to enable output for trigger programs, or zero to disable it,
  2094   2111   ** or negative to leave the setting unchanged.
  2095   2112   ** The second parameter is a pointer to an integer into which is written 
  2096   2113   ** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if 
  2097   2114   ** it is not disabled, 1 if it is.  
  2098   2115   ** </dd>
         2116  +**
         2117  +** <dt>SQLITE_DBCONFIG_RESET_DATABASE</dt>
         2118  +** <dd> Set the SQLITE_DBCONFIG_RESET_DATABASE flag and then run
         2119  +** [VACUUM] in order to reset a database back to an empty database
         2120  +** with no schema and no content. The following process works even for
         2121  +** a badly corrupted database file:
         2122  +** <ol>
         2123  +** <li> sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
         2124  +** <li> [sqlite3_exec](db, "[VACUUM]", 0, 0, 0);
         2125  +** <li> sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0);
         2126  +** </ol>
         2127  +** Because resetting a database is destructive and irreversible, the
         2128  +** process requires the use of this obscure API and multiple steps to help
         2129  +** ensure that it does not happen by accident.
         2130  +** </dd>
  2099   2131   ** </dl>
  2100   2132   */
  2101   2133   #define SQLITE_DBCONFIG_MAINDBNAME            1000 /* const char* */
  2102   2134   #define SQLITE_DBCONFIG_LOOKASIDE             1001 /* void* int int */
  2103   2135   #define SQLITE_DBCONFIG_ENABLE_FKEY           1002 /* int int* */
  2104   2136   #define SQLITE_DBCONFIG_ENABLE_TRIGGER        1003 /* int int* */
  2105   2137   #define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */
  2106   2138   #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */
  2107   2139   #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      1006 /* int int* */
  2108   2140   #define SQLITE_DBCONFIG_ENABLE_QPSG           1007 /* int int* */
  2109   2141   #define SQLITE_DBCONFIG_TRIGGER_EQP           1008 /* int int* */
  2110         -#define SQLITE_DBCONFIG_MAX                   1008 /* Largest DBCONFIG */
         2142  +#define SQLITE_DBCONFIG_RESET_DATABASE        1009 /* int int* */
         2143  +#define SQLITE_DBCONFIG_MAX                   1009 /* Largest DBCONFIG */
  2111   2144   
  2112   2145   /*
  2113   2146   ** CAPI3REF: Enable Or Disable Extended Result Codes
  2114   2147   ** METHOD: sqlite3
  2115   2148   **
  2116   2149   ** ^The sqlite3_extended_result_codes() routine enables or disables the
  2117   2150   ** [extended result codes] feature of SQLite. ^The extended result
................................................................................
  5489   5522   ** using [sqlite3_free].
  5490   5523   ** Hence, if this variable is modified directly, either it should be
  5491   5524   ** made NULL or made to point to memory obtained from [sqlite3_malloc]
  5492   5525   ** or else the use of the [data_store_directory pragma] should be avoided.
  5493   5526   */
  5494   5527   SQLITE_EXTERN char *sqlite3_data_directory;
  5495   5528   
         5529  +/*
         5530  +** CAPI3REF: Win32 Specific Interface
         5531  +**
         5532  +** These interfaces are available only on Windows.  The
         5533  +** [sqlite3_win32_set_directory] interface is used to set the value associated
         5534  +** with the [sqlite3_temp_directory] or [sqlite3_data_directory] variable, to
         5535  +** zValue, depending on the value of the type parameter.  The zValue parameter
         5536  +** should be NULL to cause the previous value to be freed via [sqlite3_free];
         5537  +** a non-NULL value will be copied into memory obtained from [sqlite3_malloc]
         5538  +** prior to being used.  The [sqlite3_win32_set_directory] interface returns
         5539  +** [SQLITE_OK] to indicate success, [SQLITE_ERROR] if the type is unsupported,
         5540  +** or [SQLITE_NOMEM] if memory could not be allocated.  The value of the
         5541  +** [sqlite3_data_directory] variable is intended to act as a replacement for
         5542  +** the current directory on the sub-platforms of Win32 where that concept is
         5543  +** not present, e.g. WinRT and UWP.  The [sqlite3_win32_set_directory8] and
         5544  +** [sqlite3_win32_set_directory16] interfaces behave exactly the same as the
         5545  +** sqlite3_win32_set_directory interface except the string parameter must be
         5546  +** UTF-8 or UTF-16, respectively.
         5547  +*/
         5548  +int sqlite3_win32_set_directory(
         5549  +  unsigned long type, /* Identifier for directory being set or reset */
         5550  +  void *zValue        /* New value for directory being set or reset */
         5551  +);
         5552  +int sqlite3_win32_set_directory8(unsigned long type, const char *zValue);
         5553  +int sqlite3_win32_set_directory16(unsigned long type, const void *zValue);
         5554  +
         5555  +/*
         5556  +** CAPI3REF: Win32 Directory Types
         5557  +**
         5558  +** These macros are only available on Windows.  They define the allowed values
         5559  +** for the type argument to the [sqlite3_win32_set_directory] interface.
         5560  +*/
         5561  +#define SQLITE_WIN32_DATA_DIRECTORY_TYPE  1
         5562  +#define SQLITE_WIN32_TEMP_DIRECTORY_TYPE  2
         5563  +
  5496   5564   /*
  5497   5565   ** CAPI3REF: Test For Auto-Commit Mode
  5498   5566   ** KEYWORDS: {autocommit mode}
  5499   5567   ** METHOD: sqlite3
  5500   5568   **
  5501   5569   ** ^The sqlite3_get_autocommit() interface returns non-zero or
  5502   5570   ** zero if the given database connection is or is not in autocommit mode,
................................................................................
  6996   7064   #define SQLITE_TESTCTRL_FAULT_INSTALL            9
  6997   7065   #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS     10
  6998   7066   #define SQLITE_TESTCTRL_PENDING_BYTE            11
  6999   7067   #define SQLITE_TESTCTRL_ASSERT                  12
  7000   7068   #define SQLITE_TESTCTRL_ALWAYS                  13
  7001   7069   #define SQLITE_TESTCTRL_RESERVE                 14
  7002   7070   #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
  7003         -#define SQLITE_TESTCTRL_ISKEYWORD               16
         7071  +#define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
  7004   7072   #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
  7005   7073   #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
  7006   7074   #define SQLITE_TESTCTRL_EXPLAIN_STMT            19  /* NOT USED */
  7007   7075   #define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD    19
  7008   7076   #define SQLITE_TESTCTRL_NEVER_CORRUPT           20
  7009   7077   #define SQLITE_TESTCTRL_VDBE_COVERAGE           21
  7010   7078   #define SQLITE_TESTCTRL_BYTEORDER               22
  7011   7079   #define SQLITE_TESTCTRL_ISINIT                  23
  7012   7080   #define SQLITE_TESTCTRL_SORTER_MMAP             24
  7013   7081   #define SQLITE_TESTCTRL_IMPOSTER                25
  7014   7082   #define SQLITE_TESTCTRL_PARSER_COVERAGE         26
  7015   7083   #define SQLITE_TESTCTRL_LAST                    26  /* Largest TESTCTRL */
  7016   7084   
         7085  +/*
         7086  +** CAPI3REF: SQL Keyword Checking
         7087  +**
         7088  +** These routines provide access to the set of SQL language keywords 
         7089  +** recognized by SQLite.  Applications can uses these routines to determine
         7090  +** whether or not a specific identifier needs to be escaped (for example,
         7091  +** by enclosing in double-quotes) so as not to confuse the parser.
         7092  +**
         7093  +** The sqlite3_keyword_count() interface returns the number of distinct
         7094  +** keywords understood by SQLite.
         7095  +**
         7096  +** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and
         7097  +** makes *Z point to that keyword expressed as UTF8 and writes the number
         7098  +** of bytes in the keyword into *L.  The string that *Z points to is not
         7099  +** zero-terminated.  The sqlite3_keyword_name(N,Z,L) routine returns
         7100  +** SQLITE_OK if N is within bounds and SQLITE_ERROR if not. If either Z
         7101  +** or L are NULL or invalid pointers then calls to
         7102  +** sqlite3_keyword_name(N,Z,L) result in undefined behavior.
         7103  +**
         7104  +** The sqlite3_keyword_check(Z,L) interface checks to see whether or not
         7105  +** the L-byte UTF8 identifier that Z points to is a keyword, returning non-zero
         7106  +** if it is and zero if not.
         7107  +**
         7108  +** The parser used by SQLite is forgiving.  It is often possible to use
         7109  +** a keyword as an identifier as long as such use does not result in a
         7110  +** parsing ambiguity.  For example, the statement
         7111  +** "CREATE TABLE BEGIN(REPLACE,PRAGMA,END);" is accepted by SQLite, and
         7112  +** creates a new table named "BEGIN" with three columns named
         7113  +** "REPLACE", "PRAGMA", and "END".  Nevertheless, best practice is to avoid
         7114  +** using keywords as identifiers.  Common techniques used to avoid keyword
         7115  +** name collisions include:
         7116  +** <ul>
         7117  +** <li> Put all indentifier names inside double-quotes.  This is the official
         7118  +**      SQL way to escape identifier names.
         7119  +** <li> Put identifier names inside &#91;...&#93;.  This is not standard SQL,
         7120  +**      but it is what SQL Server does and so lots of programmers use this
         7121  +**      technique.
         7122  +** <li> Begin every identifier with the letter "Z" as no SQL keywords start
         7123  +**      with "Z".
         7124  +** <li> Include a digit somewhere in every identifier name.
         7125  +** </ul>
         7126  +**
         7127  +** Note that the number of keywords understood by SQLite can depend on
         7128  +** compile-time options.  For example, "VACUUM" is not a keyword if
         7129  +** SQLite is compiled with the [-DSQLITE_OMIT_VACUUM] option.  Also,
         7130  +** new keywords may be added to future releases of SQLite.
         7131  +*/
         7132  +int sqlite3_keyword_count(void);
         7133  +int sqlite3_keyword_name(int,const char**,int*);
         7134  +int sqlite3_keyword_check(const char*,int);
         7135  +
  7017   7136   /*
  7018   7137   ** CAPI3REF: SQLite Runtime Status
  7019   7138   **
  7020   7139   ** ^These interfaces are used to retrieve runtime status information
  7021   7140   ** about the performance of SQLite, and optionally to reset various
  7022   7141   ** highwater marks.  ^The first argument is an integer code for
  7023   7142   ** the specific parameter to measure.  ^(Recognized integer codes

Changes to src/sqliteInt.h.

   633    633   ** The default value of "20" was choosen to minimize the run-time of the
   634    634   ** speedtest1 test program with options: --shrink-memory --reprepare
   635    635   */
   636    636   #ifndef SQLITE_DEFAULT_PCACHE_INITSZ
   637    637   # define SQLITE_DEFAULT_PCACHE_INITSZ 20
   638    638   #endif
   639    639   
          640  +/*
          641  +** Default value for the SQLITE_CONFIG_SORTERREF_SIZE option.
          642  +*/
          643  +#ifndef SQLITE_DEFAULT_SORTERREF_SIZE
          644  +# define SQLITE_DEFAULT_SORTERREF_SIZE 0x7fffffff
          645  +#endif
          646  +
   640    647   /*
   641    648   ** The compile-time options SQLITE_MMAP_READWRITE and 
   642    649   ** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another.
   643    650   ** You must choose one or the other (or neither) but not both.
   644    651   */
   645    652   #if defined(SQLITE_MMAP_READWRITE) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE)
   646    653   #error Cannot use both SQLITE_MMAP_READWRITE and SQLITE_ENABLE_BATCH_ATOMIC_WRITE
................................................................................
  1091   1098   typedef struct TableLock TableLock;
  1092   1099   typedef struct Token Token;
  1093   1100   typedef struct TreeView TreeView;
  1094   1101   typedef struct Trigger Trigger;
  1095   1102   typedef struct TriggerPrg TriggerPrg;
  1096   1103   typedef struct TriggerStep TriggerStep;
  1097   1104   typedef struct UnpackedRecord UnpackedRecord;
         1105  +typedef struct Upsert Upsert;
  1098   1106   typedef struct VTable VTable;
  1099   1107   typedef struct VtabCtx VtabCtx;
  1100   1108   typedef struct Walker Walker;
  1101   1109   typedef struct WhereInfo WhereInfo;
  1102   1110   typedef struct With With;
  1103   1111   
  1104   1112   /* A VList object records a mapping between parameters/variables/wildcards
................................................................................
  1350   1358     u8 bBenignMalloc;             /* Do not require OOMs if true */
  1351   1359     u8 dfltLockMode;              /* Default locking-mode for attached dbs */
  1352   1360     signed char nextAutovac;      /* Autovac setting after VACUUM if >=0 */
  1353   1361     u8 suppressErr;               /* Do not issue error messages if true */
  1354   1362     u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
  1355   1363     u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
  1356   1364     u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
  1357         -  u8 skipBtreeMutex;            /* True if no shared-cache backends */
         1365  +  u8 noSharedCache;             /* True if no shared-cache backends */
  1358   1366     u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
  1359   1367     int nextPagesize;             /* Pagesize after VACUUM if >0 */
  1360   1368     u32 magic;                    /* Magic number for detect library misuse */
  1361   1369     int nChange;                  /* Value returned by sqlite3_changes() */
  1362   1370     int nTotalChange;             /* Value returned by sqlite3_total_changes() */
  1363   1371     int aLimit[SQLITE_N_LIMIT];   /* Limits */
  1364   1372     int nMaxSorterMmap;           /* Maximum size of regions mapped by sorter */
................................................................................
  1494   1502   #define SQLITE_EnableTrigger  0x00040000  /* True to enable triggers */
  1495   1503   #define SQLITE_DeferFKs       0x00080000  /* Defer all FK constraints */
  1496   1504   #define SQLITE_QueryOnly      0x00100000  /* Disable database changes */
  1497   1505   #define SQLITE_CellSizeCk     0x00200000  /* Check btree cell sizes on load */
  1498   1506   #define SQLITE_Fts3Tokenizer  0x00400000  /* Enable fts3_tokenizer(2) */
  1499   1507   #define SQLITE_EnableQPSG     0x00800000  /* Query Planner Stability Guarantee*/
  1500   1508   #define SQLITE_TriggerEQP     0x01000000  /* Show trigger EXPLAIN QUERY PLAN */
         1509  +#define SQLITE_ResetDatabase  0x02000000  /* Reset the database */
  1501   1510   
  1502   1511   /* Flags used only if debugging */
  1503   1512   #ifdef SQLITE_DEBUG
  1504   1513   #define SQLITE_SqlTrace       0x08000000  /* Debug print SQL as it executes */
  1505   1514   #define SQLITE_VdbeListing    0x10000000  /* Debug listings of VDBE programs */
  1506   1515   #define SQLITE_VdbeTrace      0x20000000  /* True to trace VDBE execution */
  1507   1516   #define SQLITE_VdbeAddopTrace 0x40000000  /* Trace sqlite3VdbeAddOp() calls */
................................................................................
  1510   1519   
  1511   1520   /*
  1512   1521   ** Allowed values for sqlite3.mDbFlags
  1513   1522   */
  1514   1523   #define DBFLAG_SchemaChange   0x0001  /* Uncommitted Hash table changes */
  1515   1524   #define DBFLAG_PreferBuiltin  0x0002  /* Preference to built-in funcs */
  1516   1525   #define DBFLAG_Vacuum         0x0004  /* Currently in a VACUUM */
         1526  +#define DBFLAG_SchemaKnownOk  0x0008  /* Schema is known to be valid */
  1517   1527   
  1518   1528   /*
  1519   1529   ** Bits of the sqlite3.dbOptFlags field that are used by the
  1520   1530   ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to
  1521   1531   ** selectively disable various optimizations.
  1522   1532   */
  1523   1533   #define SQLITE_QueryFlattener 0x0001   /* Query flattening */
................................................................................
  1755   1765   
  1756   1766   /* Allowed values for Column.colFlags:
  1757   1767   */
  1758   1768   #define COLFLAG_PRIMKEY  0x0001    /* Column is part of the primary key */
  1759   1769   #define COLFLAG_HIDDEN   0x0002    /* A hidden column in a virtual table */
  1760   1770   #define COLFLAG_HASTYPE  0x0004    /* Type name follows column name */
  1761   1771   #define COLFLAG_UNIQUE   0x0008    /* Column def contains "UNIQUE" or "PK" */
         1772  +#define COLFLAG_SORTERREF 0x0010   /* Use sorter-refs with this column */
  1762   1773   
  1763   1774   /*
  1764   1775   ** A "Collating Sequence" is defined by an instance of the following
  1765   1776   ** structure. Conceptually, a collating sequence consists of a name and
  1766   1777   ** a comparison routine that defines the order of that sequence.
  1767   1778   **
  1768   1779   ** If CollSeq.xCmp is NULL, it means that the
................................................................................
  2042   2053   */
  2043   2054   #define OE_None     0   /* There is no constraint to check */
  2044   2055   #define OE_Rollback 1   /* Fail the operation and rollback the transaction */
  2045   2056   #define OE_Abort    2   /* Back out changes but do no rollback transaction */
  2046   2057   #define OE_Fail     3   /* Stop the operation but leave all prior changes */
  2047   2058   #define OE_Ignore   4   /* Ignore the error. Do not do the INSERT or UPDATE */
  2048   2059   #define OE_Replace  5   /* Delete existing record, then do INSERT or UPDATE */
  2049         -
  2050         -#define OE_Restrict 6   /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */
  2051         -#define OE_SetNull  7   /* Set the foreign key value to NULL */
  2052         -#define OE_SetDflt  8   /* Set the foreign key value to its default */
  2053         -#define OE_Cascade  9   /* Cascade the changes */
  2054         -
  2055         -#define OE_Default  10  /* Do whatever the default action is */
         2060  +#define OE_Update   6   /* Process as a DO UPDATE in an upsert */
         2061  +#define OE_Restrict 7   /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */
         2062  +#define OE_SetNull  8   /* Set the foreign key value to NULL */
         2063  +#define OE_SetDflt  9   /* Set the foreign key value to its default */
         2064  +#define OE_Cascade  10  /* Cascade the changes */
         2065  +#define OE_Default  11  /* Do whatever the default action is */
  2056   2066   
  2057   2067   
  2058   2068   /*
  2059   2069   ** An instance of the following structure is passed as the first
  2060   2070   ** argument to sqlite3VdbeKeyCompare and is used to control the
  2061   2071   ** comparison of the two index keys.
  2062   2072   **
................................................................................
  2495   2505       Expr *pExpr;            /* The parse tree for this expression */
  2496   2506       char *zName;            /* Token associated with this expression */
  2497   2507       char *zSpan;            /* Original text of the expression */
  2498   2508       u8 sortOrder;           /* 1 for DESC or 0 for ASC */
  2499   2509       unsigned done :1;       /* A flag to indicate when processing is finished */
  2500   2510       unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */
  2501   2511       unsigned reusable :1;   /* Constant expression is reusable */
         2512  +    unsigned bSorterRef :1; /* Defer evaluation until after sorting */
  2502   2513       union {
  2503   2514         struct {
  2504   2515           u16 iOrderByCol;      /* For ORDER BY, column number in result set */
  2505   2516           u16 iAlias;           /* Index into Parse.aAlias[] for zName */
  2506   2517         } x;
  2507   2518         int iConstExprReg;      /* Register in which Expr value is cached */
  2508   2519       } u;
................................................................................
  2594   2605         unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
  2595   2606         unsigned isIndexedBy :1;   /* True if there is an INDEXED BY clause */
  2596   2607         unsigned isTabFunc :1;     /* True if table-valued-function syntax */
  2597   2608         unsigned isCorrelated :1;  /* True if sub-query is correlated */
  2598   2609         unsigned viaCoroutine :1;  /* Implemented as a co-routine */
  2599   2610         unsigned isRecursive :1;   /* True for recursive reference in WITH */
  2600   2611       } fg;
  2601         -#ifndef SQLITE_OMIT_EXPLAIN
  2602         -    u8 iSelectId;     /* If pSelect!=0, the id of the sub-select in EQP */
  2603         -#endif
  2604   2612       int iCursor;      /* The VDBE cursor number used to access this table */
  2605   2613       Expr *pOn;        /* The ON clause of a join */
  2606   2614       IdList *pUsing;   /* The USING clause of a join */
  2607   2615       Bitmask colUsed;  /* Bit N (1<<N) set if column N of pTab is used */
  2608   2616       union {
  2609   2617         char *zIndexedBy;    /* Identifier from "INDEXED BY <zIndex>" clause */
  2610   2618         ExprList *pFuncArg;  /* Arguments to table-valued-function */
................................................................................
  2678   2686   ** NameContext in the parent query.  Thus the process of scanning the
  2679   2687   ** NameContext list corresponds to searching through successively outer
  2680   2688   ** subqueries looking for a match.
  2681   2689   */
  2682   2690   struct NameContext {
  2683   2691     Parse *pParse;       /* The parser */
  2684   2692     SrcList *pSrcList;   /* One or more tables used to resolve names */
  2685         -  ExprList *pEList;    /* Optional list of result-set columns */
  2686         -  AggInfo *pAggInfo;   /* Information about aggregates at this level */
         2693  +  union {
         2694  +    ExprList *pEList;    /* Optional list of result-set columns */
         2695  +    AggInfo *pAggInfo;   /* Information about aggregates at this level */
         2696  +    Upsert *pUpsert;     /* ON CONFLICT clause information from an upsert */
         2697  +  } uNC;
  2687   2698     NameContext *pNext;  /* Next outer name context.  NULL for outermost */
  2688   2699     int nRef;            /* Number of names resolved by this context */
  2689   2700     int nErr;            /* Number of errors encountered while resolving names */
  2690   2701     u16 ncFlags;         /* Zero or more NC_* flags defined below */
  2691   2702   };
  2692   2703   
  2693   2704   /*
................................................................................
  2701   2712   #define NC_AllowAgg  0x0001  /* Aggregate functions are allowed here */
  2702   2713   #define NC_PartIdx   0x0002  /* True if resolving a partial index WHERE */
  2703   2714   #define NC_IsCheck   0x0004  /* True if resolving names in a CHECK constraint */
  2704   2715   #define NC_InAggFunc 0x0008  /* True if analyzing arguments to an agg func */
  2705   2716   #define NC_HasAgg    0x0010  /* One or more aggregate functions seen */
  2706   2717   #define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
  2707   2718   #define NC_VarSelect 0x0040  /* A correlated subquery has been seen */
         2719  +#define NC_UEList    0x0080  /* True if uNC.pEList is used */
         2720  +#define NC_UAggInfo  0x0100  /* True if uNC.pAggInfo is used */
         2721  +#define NC_UUpsert   0x0200  /* True if uNC.pUpsert is used */
  2708   2722   #define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */
  2709   2723   #define NC_Complex   0x2000  /* True if a function or subquery seen */
         2724  +
         2725  +/*
         2726  +** An instance of the following object describes a single ON CONFLICT
         2727  +** clause in an upsert.
         2728  +**
         2729  +** The pUpsertTarget field is only set if the ON CONFLICT clause includes
         2730  +** conflict-target clause.  (In "ON CONFLICT(a,b)" the "(a,b)" is the
         2731  +** conflict-target clause.)  The pUpsertTargetWhere is the optional
         2732  +** WHERE clause used to identify partial unique indexes.
         2733  +**
         2734  +** pUpsertSet is the list of column=expr terms of the UPDATE statement. 
         2735  +** The pUpsertSet field is NULL for a ON CONFLICT DO NOTHING.  The
         2736  +** pUpsertWhere is the WHERE clause for the UPDATE and is NULL if the
         2737  +** WHERE clause is omitted.
         2738  +*/
         2739  +struct Upsert {
         2740  +  ExprList *pUpsertTarget;  /* Optional description of conflicting index */
         2741  +  Expr *pUpsertTargetWhere; /* WHERE clause for partial index targets */
         2742  +  ExprList *pUpsertSet;     /* The SET clause from an ON CONFLICT UPDATE */
         2743  +  Expr *pUpsertWhere;       /* WHERE clause for the ON CONFLICT UPDATE */
         2744  +  /* The fields above comprise the parse tree for the upsert clause.
         2745  +  ** The fields below are used to transfer information from the INSERT
         2746  +  ** processing down into the UPDATE processing while generating code.
         2747  +  ** Upsert owns the memory allocated above, but not the memory below. */
         2748  +  Index *pUpsertIdx;        /* Constraint that pUpsertTarget identifies */
         2749  +  SrcList *pUpsertSrc;      /* Table to be updated */
         2750  +  int regData;              /* First register holding array of VALUES */
         2751  +  int iDataCur;             /* Index of the data cursor */
         2752  +  int iIdxCur;              /* Index of the first index cursor */
         2753  +};
  2710   2754   
  2711   2755   /*
  2712   2756   ** An instance of the following structure contains all information
  2713   2757   ** needed to generate code for a single SELECT statement.
  2714   2758   **
  2715         -** nLimit is set to -1 if there is no LIMIT clause.  nOffset is set to 0.
  2716         -** If there is a LIMIT clause, the parser sets nLimit to the value of the
  2717         -** limit and nOffset to the value of the offset (or 0 if there is not
  2718         -** offset).  But later on, nLimit and nOffset become the memory locations
  2719         -** in the VDBE that record the limit and offset counters.
         2759  +** See the header comment on the computeLimitRegisters() routine for a
         2760  +** detailed description of the meaning of the iLimit and iOffset fields.
  2720   2761   **
  2721   2762   ** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes.
  2722   2763   ** These addresses must be stored so that we can go back and fill in
  2723   2764   ** the P4_KEYINFO and P2 parameters later.  Neither the KeyInfo nor
  2724   2765   ** the number of columns in P2 can be computed at the same time
  2725   2766   ** as the OP_OpenEphm instruction is coded because not
  2726   2767   ** enough information about the compound query is known at that point.
................................................................................
  2772   2813   #define SF_NestedFrom     0x00800  /* Part of a parenthesized FROM clause */
  2773   2814   #define SF_MinMaxAgg      0x01000  /* Aggregate containing min() or max() */
  2774   2815   #define SF_Recursive      0x02000  /* The recursive part of a recursive CTE */
  2775   2816   #define SF_FixedLimit     0x04000  /* nSelectRow set by a constant LIMIT */
  2776   2817   #define SF_MaybeConvert   0x08000  /* Need convertCompoundSelectToSubquery() */
  2777   2818   #define SF_Converted      0x10000  /* By convertCompoundSelectToSubquery() */
  2778   2819   #define SF_IncludeHidden  0x20000  /* Include hidden columns in output */
  2779         -#define SF_ComplexResult  0x40000  /* Result set contains subquery or function */
  2780         -
         2820  +#define SF_ComplexResult  0x40000  /* Result contains subquery or function */
  2781   2821   
  2782   2822   /*
  2783   2823   ** The results of a SELECT can be distributed in several ways, as defined
  2784   2824   ** by one of the following macros.  The "SRT" prefix means "SELECT Result
  2785   2825   ** Type".
  2786   2826   **
  2787   2827   **     SRT_Union       Store results as a key in a temporary index
................................................................................
  3043   3083     u8 explain;               /* True if the EXPLAIN flag is found on the query */
  3044   3084   #ifndef SQLITE_OMIT_VIRTUALTABLE
  3045   3085     u8 declareVtab;           /* True if inside sqlite3_declare_vtab() */
  3046   3086     int nVtabLock;            /* Number of virtual tables to lock */
  3047   3087   #endif
  3048   3088     int nHeight;              /* Expression tree height of current sub-select */
  3049   3089   #ifndef SQLITE_OMIT_EXPLAIN
  3050         -  int iSelectId;            /* ID of current select for EXPLAIN output */
  3051         -  int iNextSelectId;        /* Next available select ID for EXPLAIN output */
         3090  +  int addrExplain;          /* Address of current OP_Explain opcode */
  3052   3091   #endif
  3053   3092     VList *pVList;            /* Mapping between variable names and numbers */
  3054   3093     Vdbe *pReprepare;         /* VM being reprepared (sqlite3Reprepare()) */
  3055   3094     const char *zTail;        /* All SQL text past the last semicolon parsed */
  3056   3095     Table *pNewTable;         /* A table being constructed by CREATE TABLE */
  3057   3096     Trigger *pNewTrigger;     /* Trigger under construct by a CREATE TRIGGER */
  3058   3097     const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */
................................................................................
  3203   3242   struct TriggerStep {
  3204   3243     u8 op;               /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */
  3205   3244     u8 orconf;           /* OE_Rollback etc. */
  3206   3245     Trigger *pTrig;      /* The trigger that this step is a part of */
  3207   3246     Select *pSelect;     /* SELECT statement or RHS of INSERT INTO SELECT ... */
  3208   3247     char *zTarget;       /* Target table for DELETE, UPDATE, INSERT */
  3209   3248     Expr *pWhere;        /* The WHERE clause for DELETE or UPDATE steps */
  3210         -  ExprList *pExprList; /* SET clause for UPDATE. */
         3249  +  ExprList *pExprList; /* SET clause for UPDATE */
  3211   3250     IdList *pIdList;     /* Column names for INSERT */
         3251  +  Upsert *pUpsert;     /* Upsert clauses on an INSERT */
  3212   3252     char *zSpan;         /* Original SQL text of this command */
  3213   3253     TriggerStep *pNext;  /* Next in the link-list */
  3214   3254     TriggerStep *pLast;  /* Last element in link-list. Valid for 1st elem only */
  3215   3255   };
  3216   3256   
  3217   3257   /*
  3218   3258   ** The following structure contains information used by the sqliteFix...
................................................................................
  3316   3356     void *pVdbeBranchArg;                                     /* 1st argument */
  3317   3357   #endif
  3318   3358   #ifndef SQLITE_UNTESTABLE
  3319   3359     int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
  3320   3360   #endif
  3321   3361     int bLocaltimeFault;              /* True to fail localtime() calls */
  3322   3362     int iOnceResetThreshold;          /* When to reset OP_Once counters */
         3363  +  u32 szSorterRef;                  /* Min size in bytes to use sorter-refs */
  3323   3364   };
  3324   3365   
  3325   3366   /*
  3326   3367   ** This macro is used inside of assert() statements to indicate that
  3327   3368   ** the assert is only valid on a well-formed database.  Instead of:
  3328   3369   **
  3329   3370   **     assert( X );
................................................................................
  3736   3777   #ifndef SQLITE_OMIT_AUTOINCREMENT
  3737   3778     void sqlite3AutoincrementBegin(Parse *pParse);
  3738   3779     void sqlite3AutoincrementEnd(Parse *pParse);
  3739   3780   #else
  3740   3781   # define sqlite3AutoincrementBegin(X)
  3741   3782   # define sqlite3AutoincrementEnd(X)
  3742   3783   #endif
  3743         -void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int);
         3784  +void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*);
  3744   3785   void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*);
  3745   3786   IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*);
  3746   3787   int sqlite3IdListIndex(IdList*,const char*);
  3747   3788   SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int);
  3748   3789   SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*);
  3749   3790   SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
  3750   3791                                         Token*, Select*, Expr*, IdList*);
................................................................................
  3766   3807   Table *sqlite3SrcListLookup(Parse*, SrcList*);
  3767   3808   int sqlite3IsReadOnly(Parse*, Table*, int);
  3768   3809   void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
  3769   3810   #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
  3770   3811   Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*);
  3771   3812   #endif
  3772   3813   void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*);
  3773         -void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*);
         3814  +void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,
         3815  +                   Upsert*);
  3774   3816   WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
  3775   3817   void sqlite3WhereEnd(WhereInfo*);
  3776   3818   LogEst sqlite3WhereOutputRowCount(WhereInfo*);
  3777   3819   int sqlite3WhereIsDistinct(WhereInfo*);
  3778   3820   int sqlite3WhereIsOrdered(WhereInfo*);
  3779   3821   int sqlite3WhereOrderedInnerLoop(WhereInfo*);
  3780   3822   int sqlite3WhereIsSorted(WhereInfo*);
................................................................................
  3859   3901   int sqlite3IsRowid(const char*);
  3860   3902   void sqlite3GenerateRowDelete(
  3861   3903       Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
  3862   3904   void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
  3863   3905   int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
  3864   3906   void sqlite3ResolvePartIdxLabel(Parse*,int);
  3865   3907   void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
  3866         -                                     u8,u8,int,int*,int*);
         3908  +                                     u8,u8,int,int*,int*,Upsert*);
  3867   3909   #ifdef SQLITE_ENABLE_NULL_TRIM
  3868   3910     void sqlite3SetMakeRecordP5(Vdbe*,Table*);
  3869   3911   #else
  3870   3912   # define sqlite3SetMakeRecordP5(A,B)
  3871   3913   #endif
  3872   3914   void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int);
  3873   3915   int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*);
................................................................................
  3912   3954                               int, int, int);
  3913   3955     void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, int, int, int);
  3914   3956     void sqliteViewTriggers(Parse*, Table*, Expr*, int, ExprList*);
  3915   3957     void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*);
  3916   3958     TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*,
  3917   3959                                           const char*,const char*);
  3918   3960     TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*,
  3919         -                                        Select*,u8,const char*,const char*);
         3961  +                                        Select*,u8,Upsert*,
         3962  +                                        const char*,const char*);
  3920   3963     TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8,
  3921   3964                                           const char*,const char*);
  3922   3965     TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*,
  3923   3966                                           const char*,const char*);
  3924   3967     void sqlite3DeleteTrigger(sqlite3*, Trigger*);
  3925   3968     void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*);
  3926   3969     u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int);
................................................................................
  4098   4141   void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*);
  4099   4142   void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*);
  4100   4143   int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*);
  4101   4144   void sqlite3ColumnDefault(Vdbe *, Table *, int, int);
  4102   4145   void sqlite3AlterFinishAddColumn(Parse *, Token *);
  4103   4146   void sqlite3AlterBeginAddColumn(Parse *, SrcList *);
  4104   4147   CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*);
  4105         -char sqlite3AffinityType(const char*, u8*);
         4148  +char sqlite3AffinityType(const char*, Column*);
  4106   4149   void sqlite3Analyze(Parse*, Token*, Token*);
  4107   4150   int sqlite3InvokeBusyHandler(BusyHandler*, sqlite3_file*);
  4108   4151   int sqlite3FindDb(sqlite3*, Token*);
  4109   4152   int sqlite3FindDbName(sqlite3 *, const char *);
  4110   4153   int sqlite3AnalysisLoad(sqlite3*,int iDB);
  4111   4154   void sqlite3DeleteIndexSamples(sqlite3*,Index*);
  4112   4155   void sqlite3DefaultRowEst(Index*);
................................................................................
  4160   4203   char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
  4161   4204   #endif
  4162   4205   
  4163   4206   /*
  4164   4207   ** The interface to the LEMON-generated parser
  4165   4208   */
  4166   4209   #ifndef SQLITE_AMALGAMATION
  4167         -  void *sqlite3ParserAlloc(void*(*)(u64));
         4210  +  void *sqlite3ParserAlloc(void*(*)(u64), Parse*);
  4168   4211     void sqlite3ParserFree(void*, void(*)(void*));
  4169   4212   #endif
  4170         -void sqlite3Parser(void*, int, Token, Parse*);
         4213  +void sqlite3Parser(void*, int, Token);
  4171   4214   #ifdef YYTRACKMAXSTACKDEPTH
  4172   4215     int sqlite3ParserStackPeak(void*);
  4173   4216   #endif
  4174   4217   
  4175   4218   void sqlite3AutoLoadExtensions(sqlite3*);
  4176   4219   #ifndef SQLITE_OMIT_LOAD_EXTENSION
  4177   4220     void sqlite3CloseExtensions(sqlite3*);
................................................................................
  4251   4294     With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*);
  4252   4295     void sqlite3WithDelete(sqlite3*,With*);
  4253   4296     void sqlite3WithPush(Parse*, With*, u8);
  4254   4297   #else
  4255   4298   #define sqlite3WithPush(x,y,z)
  4256   4299   #define sqlite3WithDelete(x,y)
  4257   4300   #endif
         4301  +#ifndef SQLITE_OMIT_UPSERT
         4302  +  Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*);
         4303  +  void sqlite3UpsertDelete(sqlite3*,Upsert*);
         4304  +  Upsert *sqlite3UpsertDup(sqlite3*,Upsert*);
         4305  +  int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*);
         4306  +  void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int);
         4307  +#else
         4308  +#define sqlite3UpsertNew(v,w,x,y,z) ((Upsert*)0)
         4309  +#define sqlite3UpsertDelete(x,y)
         4310  +#define sqlite3UpsertDup(x,y)       ((Upsert*)0)
         4311  +#endif
         4312  +
  4258   4313   
  4259   4314   /* Declarations for functions in fkey.c. All of these are replaced by
  4260   4315   ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign
  4261   4316   ** key functionality is available. If OMIT_TRIGGER is defined but
  4262   4317   ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In
  4263   4318   ** this case foreign keys are parsed, but no other functionality is
  4264   4319   ** provided (enforcement of FK constraints requires the triggers sub-system).

Changes to src/test1.c.

  2251   2251       Tcl_WrongNumArgs(interp, 1, objv, "");
  2252   2252       return TCL_ERROR;
  2253   2253     }
  2254   2254     sqlite3_config(SQLITE_CONFIG_SQLLOG, 0, 0);
  2255   2255     return TCL_OK;
  2256   2256   }
  2257   2257   #endif
         2258  +
         2259  +/*
         2260  +** Usage:  sqlite3_config_sorterref
         2261  +**
         2262  +** Set the SQLITE_CONFIG_SORTERREF_SIZE configuration option
         2263  +*/
         2264  +static int SQLITE_TCLAPI test_config_sorterref(
         2265  +  void * clientData,
         2266  +  Tcl_Interp *interp,
         2267  +  int objc,
         2268  +  Tcl_Obj *CONST objv[]
         2269  +){
         2270  +  int iVal;
         2271  +  if( objc!=2 ){
         2272  +    Tcl_WrongNumArgs(interp, 1, objv, "NBYTE");
         2273  +    return TCL_ERROR;
         2274  +  }
         2275  +  if( Tcl_GetIntFromObj(interp, objv[1], &iVal) ) return TCL_ERROR;
         2276  +  sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, iVal);
         2277  +  return TCL_OK;
         2278  +}
  2258   2279   
  2259   2280   /*
  2260   2281   ** Usage: vfs_current_time_int64
  2261   2282   **
  2262   2283   ** Return the value returned by the default VFS's xCurrentTimeInt64 method.
  2263   2284   */
  2264   2285   static int SQLITE_TCLAPI vfsCurrentTimeInt64(
................................................................................
  7584   7605     } aSetting[] = {
  7585   7606       { "FKEY",            SQLITE_DBCONFIG_ENABLE_FKEY },
  7586   7607       { "TRIGGER",         SQLITE_DBCONFIG_ENABLE_TRIGGER },
  7587   7608       { "FTS3_TOKENIZER",  SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER },
  7588   7609       { "LOAD_EXTENSION",  SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION },
  7589   7610       { "NO_CKPT_ON_CLOSE",SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE },
  7590   7611       { "QPSG",            SQLITE_DBCONFIG_ENABLE_QPSG },
         7612  +    { "TRIGGER_EQP",     SQLITE_DBCONFIG_TRIGGER_EQP },
         7613  +    { "RESET_DB",        SQLITE_DBCONFIG_RESET_DATABASE },
  7591   7614     };
  7592   7615     int i;
  7593   7616     int v;
  7594   7617     const char *zSetting;
  7595   7618     sqlite3 *db;
  7596   7619   
  7597   7620     if( objc!=4 ){
................................................................................
  7945   7968        { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 },
  7946   7969        { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
  7947   7970        { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
  7948   7971   #endif
  7949   7972        { "sqlite3_delete_database", test_delete_database,    0 },
  7950   7973        { "atomic_batch_write",      test_atomic_batch_write, 0 },
  7951   7974        { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
         7975  +     { "sqlite3_config_sorterref", test_config_sorterref,   0 },
  7952   7976     };
  7953   7977     static int bitmask_size = sizeof(Bitmask)*8;
  7954   7978     static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
  7955   7979     int i;
  7956   7980     extern int sqlite3_sync_count, sqlite3_fullsync_count;
  7957   7981     extern int sqlite3_opentemp_count;
  7958   7982     extern int sqlite3_like_count;

Changes to src/tokenize.c.

   492    492     }
   493    493     pParse->rc = SQLITE_OK;
   494    494     pParse->zTail = zSql;
   495    495     assert( pzErrMsg!=0 );
   496    496     /* sqlite3ParserTrace(stdout, "parser: "); */
   497    497   #ifdef sqlite3Parser_ENGINEALWAYSONSTACK
   498    498     pEngine = &sEngine;
   499         -  sqlite3ParserInit(pEngine);
          499  +  sqlite3ParserInit(pEngine, pParse);
   500    500   #else
   501         -  pEngine = sqlite3ParserAlloc(sqlite3Malloc);
          501  +  pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse);
   502    502     if( pEngine==0 ){
   503    503       sqlite3OomFault(db);
   504    504       return SQLITE_NOMEM_BKPT;
   505    505     }
   506    506   #endif
   507    507     assert( pParse->pNewTable==0 );
   508    508     assert( pParse->pNewTrigger==0 );
................................................................................
   538    538           sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
   539    539           break;
   540    540         }
   541    541         zSql += n;
   542    542       }else{
   543    543         pParse->sLastToken.z = zSql;
   544    544         pParse->sLastToken.n = n;
   545         -      sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
          545  +      sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
   546    546         lastTokenParsed = tokenType;
   547    547         zSql += n;
   548    548         if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
   549    549       }
   550    550     }
   551    551     assert( nErr==0 );
   552    552     pParse->zTail = zSql;

Changes to src/treeview.c.

    58     58     sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0);
    59     59     if( p ){
    60     60       for(i=0; i<p->iLevel && i<sizeof(p->bLine)-1; i++){
    61     61         sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|   " : "    ", 4);
    62     62       }
    63     63       sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4);
    64     64     }
    65         -  va_start(ap, zFormat);
    66         -  sqlite3VXPrintf(&acc, zFormat, ap);
    67         -  va_end(ap);
    68         -  assert( acc.nChar>0 );
    69         -  if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1);
           65  +  if( zFormat!=0 ){
           66  +    va_start(ap, zFormat);
           67  +    sqlite3VXPrintf(&acc, zFormat, ap);
           68  +    va_end(ap);
           69  +    assert( acc.nChar>0 );
           70  +    sqlite3StrAccumAppend(&acc, "\n", 1);
           71  +  }
    70     72     sqlite3StrAccumFinish(&acc);
    71     73     fprintf(stdout,"%s", zBuf);
    72     74     fflush(stdout);
    73     75   }
    74     76   
    75     77   /*
    76     78   ** Shorthand for starting a new tree item that consists of a single label
................................................................................
   532    534       sqlite3TreeViewLine(pView, "%s (empty)", zLabel);
   533    535     }else{
   534    536       int i;
   535    537       sqlite3TreeViewLine(pView, "%s", zLabel);
   536    538       for(i=0; i<pList->nExpr; i++){
   537    539         int j = pList->a[i].u.x.iOrderByCol;
   538    540         char *zName = pList->a[i].zName;
          541  +      int moreToFollow = i<pList->nExpr - 1;
   539    542         if( j || zName ){
   540         -        sqlite3TreeViewPush(pView, 0);
   541         -      }
   542         -      if( zName ){
   543         -        sqlite3TreeViewLine(pView, "AS %s", zName);
          543  +        sqlite3TreeViewPush(pView, moreToFollow);
          544  +        moreToFollow = 0;
          545  +        sqlite3TreeViewLine(pView, 0);
          546  +        if( zName ){
          547  +          fprintf(stdout, "AS %s ", zName);
          548  +        }
          549  +        if( j ){
          550  +          fprintf(stdout, "iOrderByCol=%d", j);
          551  +        }
          552  +        fprintf(stdout, "\n");
          553  +        fflush(stdout);
   544    554         }
   545         -      if( j ){
   546         -        sqlite3TreeViewLine(pView, "iOrderByCol=%d", j);
   547         -      }
   548         -      sqlite3TreeViewExpr(pView, pList->a[i].pExpr, i<pList->nExpr-1);
          555  +      sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow);
   549    556         if( j || zName ){
   550    557           sqlite3TreeViewPop(pView);
   551    558         }
   552    559       }
   553    560     }
   554    561   }
   555    562   void sqlite3TreeViewExprList(

Changes to src/trigger.c.

    21     21       TriggerStep * pTmp = pTriggerStep;
    22     22       pTriggerStep = pTriggerStep->pNext;
    23     23   
    24     24       sqlite3ExprDelete(db, pTmp->pWhere);
    25     25       sqlite3ExprListDelete(db, pTmp->pExprList);
    26     26       sqlite3SelectDelete(db, pTmp->pSelect);
    27     27       sqlite3IdListDelete(db, pTmp->pIdList);
           28  +    sqlite3UpsertDelete(db, pTmp->pUpsert);
    28     29       sqlite3DbFree(db, pTmp->zSpan);
    29     30   
    30     31       sqlite3DbFree(db, pTmp);
    31     32     }
    32     33   }
    33     34   
    34     35   /*
................................................................................
   412    413   */
   413    414   TriggerStep *sqlite3TriggerInsertStep(
   414    415     sqlite3 *db,        /* The database connection */
   415    416     Token *pTableName,  /* Name of the table into which we insert */
   416    417     IdList *pColumn,    /* List of columns in pTableName to insert into */
   417    418     Select *pSelect,    /* A SELECT statement that supplies values */
   418    419     u8 orconf,          /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */
          420  +  Upsert *pUpsert,    /* ON CONFLICT clauses for upsert */
   419    421     const char *zStart, /* Start of SQL text */
   420    422     const char *zEnd    /* End of SQL text */
   421    423   ){
   422    424     TriggerStep *pTriggerStep;
   423    425   
   424    426     assert(pSelect != 0 || db->mallocFailed);
   425    427   
   426    428     pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName, zStart, zEnd);
   427    429     if( pTriggerStep ){
   428    430       pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE);
   429    431       pTriggerStep->pIdList = pColumn;
          432  +    pTriggerStep->pUpsert = pUpsert;
   430    433       pTriggerStep->orconf = orconf;
   431    434     }else{
          435  +    testcase( pColumn );
   432    436       sqlite3IdListDelete(db, pColumn);
          437  +    testcase( pUpsert );
          438  +    sqlite3UpsertDelete(db, pUpsert);
   433    439     }
   434    440     sqlite3SelectDelete(db, pSelect);
   435    441   
   436    442     return pTriggerStep;
   437    443   }
   438    444   
   439    445   /*
................................................................................
   742    748   
   743    749       switch( pStep->op ){
   744    750         case TK_UPDATE: {
   745    751           sqlite3Update(pParse, 
   746    752             targetSrcList(pParse, pStep),
   747    753             sqlite3ExprListDup(db, pStep->pExprList, 0), 
   748    754             sqlite3ExprDup(db, pStep->pWhere, 0), 
   749         -          pParse->eOrconf, 0, 0
          755  +          pParse->eOrconf, 0, 0, 0
   750    756           );
   751    757           break;
   752    758         }
   753    759         case TK_INSERT: {
   754    760           sqlite3Insert(pParse, 
   755    761             targetSrcList(pParse, pStep),
   756    762             sqlite3SelectDup(db, pStep->pSelect, 0), 
   757    763             sqlite3IdListDup(db, pStep->pIdList), 
   758         -          pParse->eOrconf
          764  +          pParse->eOrconf,
          765  +          sqlite3UpsertDup(db, pStep->pUpsert)
   759    766           );
   760    767           break;
   761    768         }
   762    769         case TK_DELETE: {
   763    770           sqlite3DeleteFrom(pParse, 
   764    771             targetSrcList(pParse, pStep),
   765    772             sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0

Changes to src/update.c.

    89     89   void sqlite3Update(
    90     90     Parse *pParse,         /* The parser context */
    91     91     SrcList *pTabList,     /* The table in which we should change things */
    92     92     ExprList *pChanges,    /* Things to be changed */
    93     93     Expr *pWhere,          /* The WHERE clause.  May be null */
    94     94     int onError,           /* How to handle constraint errors */
    95     95     ExprList *pOrderBy,    /* ORDER BY clause. May be null */
    96         -  Expr *pLimit           /* LIMIT clause. May be null */
           96  +  Expr *pLimit,          /* LIMIT clause. May be null */
           97  +  Upsert *pUpsert        /* ON CONFLICT clause, or null */
    97     98   ){
    98     99     int i, j;              /* Loop counters */
    99    100     Table *pTab;           /* The table to be updated */
   100    101     int addrTop = 0;       /* VDBE instruction address of the start of the loop */
   101    102     WhereInfo *pWInfo;     /* Information about the WHERE clause */
   102    103     Vdbe *v;               /* The virtual database engine */
   103    104     Index *pIdx;           /* For looping over indices */
................................................................................
   196    197     }
   197    198   
   198    199     /* Allocate a cursors for the main database table and for all indices.
   199    200     ** The index cursors might not be used, but if they are used they
   200    201     ** need to occur right after the database cursor.  So go ahead and
   201    202     ** allocate enough space, just in case.
   202    203     */
   203         -  pTabList->a[0].iCursor = iBaseCur = iDataCur = pParse->nTab++;
          204  +  iBaseCur = iDataCur = pParse->nTab++;
   204    205     iIdxCur = iDataCur+1;
   205    206     pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab);
          207  +  testcase( pPk!=0 && pPk!=pTab->pIndex );
   206    208     for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){
   207         -    if( IsPrimaryKeyIndex(pIdx) && pPk!=0 ){
          209  +    if( pPk==pIdx ){
   208    210         iDataCur = pParse->nTab;
   209         -      pTabList->a[0].iCursor = iDataCur;
   210    211       }
   211    212       pParse->nTab++;
   212    213     }
          214  +  if( pUpsert ){
          215  +    /* On an UPSERT, reuse the same cursors already opened by INSERT */
          216  +    iDataCur = pUpsert->iDataCur;
          217  +    iIdxCur = pUpsert->iIdxCur;
          218  +    pParse->nTab = iBaseCur;
          219  +  }
          220  +  pTabList->a[0].iCursor = iDataCur;
   213    221   
   214    222     /* Allocate space for aXRef[], aRegIdx[], and aToOpen[].  
   215    223     ** Initialize aXRef[] and aToOpen[] to their default values.
   216    224     */
   217    225     aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 );
   218    226     if( aXRef==0 ) goto update_cleanup;
   219    227     aRegIdx = aXRef+pTab->nCol;
................................................................................
   222    230     aToOpen[nIdx+1] = 0;
   223    231     for(i=0; i<pTab->nCol; i++) aXRef[i] = -1;
   224    232   
   225    233     /* Initialize the name-context */
   226    234     memset(&sNC, 0, sizeof(sNC));
   227    235     sNC.pParse = pParse;
   228    236     sNC.pSrcList = pTabList;
          237  +  sNC.uNC.pUpsert = pUpsert;
          238  +  sNC.ncFlags = NC_UUpsert;
   229    239   
   230    240     /* Resolve the column names in all the expressions of the
   231    241     ** of the UPDATE statement.  Also find the column index
   232    242     ** for each column to be updated in the pChanges array.  For each
   233    243     ** column to be updated, make sure we have authorization to change
   234    244     ** that column.
   235    245     */
................................................................................
   325    335       memset(aToOpen, 1, nIdx+1);
   326    336     }
   327    337   
   328    338     /* Begin generating code. */
   329    339     v = sqlite3GetVdbe(pParse);
   330    340     if( v==0 ) goto update_cleanup;
   331    341     if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
   332         -  sqlite3BeginWriteOperation(pParse, 1, iDb);
          342  +  sqlite3BeginWriteOperation(pParse, pTrigger || hasFK, iDb);
   333    343   
   334    344     /* Allocate required registers. */
   335    345     if( !IsVirtual(pTab) ){
   336    346       regRowSet = ++pParse->nMem;
   337    347       regOldRowid = regNewRowid = ++pParse->nMem;
   338    348       if( chngPk || pTrigger || hasFK ){
   339    349         regOld = pParse->nMem + 1;
................................................................................
   376    386     if( IsVirtual(pTab) ){
   377    387       updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef,
   378    388                          pWhere, onError);
   379    389       goto update_cleanup;
   380    390     }
   381    391   #endif
   382    392   
   383         -  /* Initialize the count of updated rows */
   384         -  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){
          393  +  /* Jump to labelBreak to abandon further processing of this UPDATE */
          394  +  labelContinue = labelBreak = sqlite3VdbeMakeLabel(v);
          395  +
          396  +  /* Not an UPSERT.  Normal processing.  Begin by
          397  +  ** initialize the count of updated rows */
          398  +  if( (db->flags&SQLITE_CountRows)!=0
          399  +   && !pParse->pTriggerTab
          400  +   && !pParse->nested
          401  +   && pUpsert==0
          402  +  ){
   385    403       regRowCount = ++pParse->nMem;
   386    404       sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
   387    405     }
   388    406   
   389    407     if( HasRowid(pTab) ){
   390    408       sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid);
   391    409     }else{
   392    410       assert( pPk!=0 );
   393    411       nPk = pPk->nKeyCol;
   394    412       iPk = pParse->nMem+1;
   395    413       pParse->nMem += nPk;
   396    414       regKey = ++pParse->nMem;
   397         -    iEph = pParse->nTab++;
   398         -
   399         -    sqlite3VdbeAddOp3(v, OP_Null, 0, iPk, iPk+nPk-1);
   400         -    addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
   401         -    sqlite3VdbeSetP4KeyInfo(pParse, pPk);
   402         -  }
   403         -
   404         -  /* Begin the database scan. 
   405         -  **
   406         -  ** Do not consider a single-pass strategy for a multi-row update if
   407         -  ** there are any triggers or foreign keys to process, or rows may
   408         -  ** be deleted as a result of REPLACE conflict handling. Any of these
   409         -  ** things might disturb a cursor being used to scan through the table
   410         -  ** or index, causing a single-pass approach to malfunction.  */
   411         -  flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
   412         -  if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
   413         -    flags |= WHERE_ONEPASS_MULTIROW;
   414         -  }
   415         -  pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);
   416         -  if( pWInfo==0 ) goto update_cleanup;
   417         -
   418         -  /* A one-pass strategy that might update more than one row may not
   419         -  ** be used if any column of the index used for the scan is being
   420         -  ** updated. Otherwise, if there is an index on "b", statements like
   421         -  ** the following could create an infinite loop:
   422         -  **
   423         -  **   UPDATE t1 SET b=b+1 WHERE b>?
   424         -  **
   425         -  ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
   426         -  ** strategy that uses an index for which one or more columns are being
   427         -  ** updated.  */
   428         -  eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
   429         -  if( eOnePass==ONEPASS_MULTI ){
   430         -    int iCur = aiCurOnePass[1];
   431         -    if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
   432         -      eOnePass = ONEPASS_OFF;
   433         -    }
   434         -    assert( iCur!=iDataCur || !HasRowid(pTab) );
   435         -  }
          415  +    if( pUpsert==0 ){
          416  +      iEph = pParse->nTab++;
          417  +        sqlite3VdbeAddOp3(v, OP_Null, 0, iPk, iPk+nPk-1);
          418  +      addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
          419  +      sqlite3VdbeSetP4KeyInfo(pParse, pPk);
          420  +    }
          421  +  }
          422  +  
          423  +  if( pUpsert ){
          424  +    /* If this is an UPSERT, then all cursors have already been opened by
          425  +    ** the outer INSERT and the data cursor should be pointing at the row
          426  +    ** that is to be updated.  So bypass the code that searches for the
          427  +    ** row(s) to be updated.
          428  +    */
          429  +    pWInfo = 0;
          430  +    eOnePass = ONEPASS_SINGLE;
          431  +    sqlite3ExprIfFalse(pParse, pWhere, labelBreak, SQLITE_JUMPIFNULL);
          432  +  }else{
          433  +    /* Begin the database scan. 
          434  +    **
          435  +    ** Do not consider a single-pass strategy for a multi-row update if
          436  +    ** there are any triggers or foreign keys to process, or rows may
          437  +    ** be deleted as a result of REPLACE conflict handling. Any of these
          438  +    ** things might disturb a cursor being used to scan through the table
          439  +    ** or index, causing a single-pass approach to malfunction.  */
          440  +    flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
          441  +    if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
          442  +      flags |= WHERE_ONEPASS_MULTIROW;
          443  +    }
          444  +    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);
          445  +    if( pWInfo==0 ) goto update_cleanup;
   436    446     
          447  +    /* A one-pass strategy that might update more than one row may not
          448  +    ** be used if any column of the index used for the scan is being
          449  +    ** updated. Otherwise, if there is an index on "b", statements like
          450  +    ** the following could create an infinite loop:
          451  +    **
          452  +    **   UPDATE t1 SET b=b+1 WHERE b>?
          453  +    **
          454  +    ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
          455  +    ** strategy that uses an index for which one or more columns are being
          456  +    ** updated.  */
          457  +    eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
          458  +    if( eOnePass!=ONEPASS_SINGLE ){
          459  +      sqlite3MultiWrite(pParse);
          460  +      if( eOnePass==ONEPASS_MULTI ){
          461  +        int iCur = aiCurOnePass[1];
          462  +        if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
          463  +          eOnePass = ONEPASS_OFF;
          464  +        }
          465  +        assert( iCur!=iDataCur || !HasRowid(pTab) );
          466  +      }
          467  +    }
          468  +  }
          469  +
   437    470     if( HasRowid(pTab) ){
   438    471       /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF
   439    472       ** mode, write the rowid into the FIFO. In either of the one-pass modes,
   440    473       ** leave it in register regOldRowid.  */
   441    474       sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid);
   442    475       if( eOnePass==ONEPASS_OFF ){
   443    476         sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid);
................................................................................
   449    482       ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table 
   450    483       ** is not required) and leave the PK fields in the array of registers.  */
   451    484       for(i=0; i<nPk; i++){
   452    485         assert( pPk->aiColumn[i]>=0 );
   453    486         sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i);
   454    487       }
   455    488       if( eOnePass ){
   456         -      sqlite3VdbeChangeToNoop(v, addrOpen);
          489  +      if( addrOpen ) sqlite3VdbeChangeToNoop(v, addrOpen);
   457    490         nKey = nPk;
   458    491         regKey = iPk;
   459    492       }else{
   460    493         sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey,
   461    494                           sqlite3IndexAffinityStr(db, pPk), nPk);
   462    495         sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk);
   463    496       }
   464    497     }
   465    498   
   466         -  if( eOnePass!=ONEPASS_MULTI ){
   467         -    sqlite3WhereEnd(pWInfo);
   468         -  }
   469         -
   470         -  labelBreak = sqlite3VdbeMakeLabel(v);
   471         -  if( !isView ){
   472         -    int addrOnce = 0;
   473         -
   474         -    /* Open every index that needs updating. */
          499  +  if( pUpsert==0 ){
          500  +    if( eOnePass!=ONEPASS_MULTI ){
          501  +      sqlite3WhereEnd(pWInfo);
          502  +    }
          503  +  
          504  +    if( !isView ){
          505  +      int addrOnce = 0;
          506  +  
          507  +      /* Open every index that needs updating. */
          508  +      if( eOnePass!=ONEPASS_OFF ){
          509  +        if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
          510  +        if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
          511  +      }
          512  +  
          513  +      if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
          514  +        addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
          515  +      }
          516  +      sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur,
          517  +                                 aToOpen, 0, 0);
          518  +      if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
          519  +    }
          520  +  
          521  +    /* Top of the update loop */
   475    522       if( eOnePass!=ONEPASS_OFF ){
   476         -      if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
   477         -      if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
   478         -    }
   479         -
   480         -    if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
   481         -      addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   482         -    }
   483         -    sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen,
   484         -                               0, 0);
   485         -    if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
   486         -  }
   487         -
   488         -  /* Top of the update loop */
   489         -  if( eOnePass!=ONEPASS_OFF ){
   490         -    if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
   491         -      assert( pPk );
   492         -      sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey);
   493         -      VdbeCoverageNeverTaken(v);
   494         -    }
   495         -    if( eOnePass==ONEPASS_SINGLE ){
   496         -      labelContinue = labelBreak;
   497         -    }else{
          523  +      if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
          524  +        assert( pPk );
          525  +        sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey,nKey);
          526  +        VdbeCoverageNeverTaken(v);
          527  +      }
          528  +      if( eOnePass!=ONEPASS_SINGLE ){
          529  +        labelContinue = sqlite3VdbeMakeLabel(v);
          530  +      }
          531  +      sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
          532  +      VdbeCoverageIf(v, pPk==0);
          533  +      VdbeCoverageIf(v, pPk!=0);
          534  +    }else if( pPk ){
   498    535         labelContinue = sqlite3VdbeMakeLabel(v);
          536  +      sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
          537  +      addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);
          538  +      sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0);
          539  +      VdbeCoverage(v);
          540  +    }else{
          541  +      labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet,labelBreak,
          542  +                               regOldRowid);
          543  +      VdbeCoverage(v);
          544  +      sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
          545  +      VdbeCoverage(v);
   499    546       }
   500         -    sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
   501         -    VdbeCoverageIf(v, pPk==0);
   502         -    VdbeCoverageIf(v, pPk!=0);
   503         -  }else if( pPk ){
   504         -    labelContinue = sqlite3VdbeMakeLabel(v);
   505         -    sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
   506         -    addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);
   507         -    sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0);
   508         -    VdbeCoverage(v);
   509         -  }else{
   510         -    labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, labelBreak,
   511         -                             regOldRowid);
   512         -    VdbeCoverage(v);
   513         -    sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
   514         -    VdbeCoverage(v);
   515    547     }
   516    548   
   517         -  /* If the record number will change, set register regNewRowid to
   518         -  ** contain the new value. If the record number is not being modified,
          549  +  /* If the rowid value will change, set register regNewRowid to
          550  +  ** contain the new value. If the rowid is not being modified,
   519    551     ** then regNewRowid is the same register as regOldRowid, which is
   520    552     ** already populated.  */
   521    553     assert( chngKey || pTrigger || hasFK || regOldRowid==regNewRowid );
   522    554     if( chngRowid ){
   523    555       sqlite3ExprCode(pParse, pRowidExpr, regNewRowid);
   524    556       sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v);
   525    557     }
................................................................................
   575    607           ** into a register. This is done if there are no BEFORE triggers, or
   576    608           ** if there are one or more BEFORE triggers that use this value via
   577    609           ** a new.* reference in a trigger program.
   578    610           */
   579    611           testcase( i==31 );
   580    612           testcase( i==32 );
   581    613           sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i);
          614  +        if( tmask & TRIGGER_BEFORE ){
          615  +          /* This value will be recomputed in After-BEFORE-trigger-reload-loop
          616  +          ** below, so make sure that it is not cached and reused.
          617  +          ** Ticket d85fffd6ffe856092ed8daefa811b1e399706b28. */
          618  +          sqlite3ExprCacheRemove(pParse, regNew+i, 1);
          619  +        }
   582    620         }else{
   583    621           sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i);
   584    622         }
   585    623       }
   586    624     }
   587    625   
   588    626     /* Fire any BEFORE UPDATE triggers. This happens before constraints are
................................................................................
   603    641         sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue,regKey,nKey);
   604    642         VdbeCoverage(v);
   605    643       }else{
   606    644         sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
   607    645         VdbeCoverage(v);
   608    646       }
   609    647   
   610         -    /* If it did not delete it, the row-trigger may still have modified 
          648  +    /* After-BEFORE-trigger-reload-loop:
          649  +    ** If it did not delete it, the BEFORE trigger may still have modified 
   611    650       ** some of the columns of the row being updated. Load the values for 
   612         -    ** all columns not modified by the update statement into their 
   613         -    ** registers in case this has happened.
          651  +    ** all columns not modified by the update statement into their registers
          652  +    ** in case this has happened. Only unmodified columns are reloaded.
          653  +    ** The values computed for modified columns use the values before the
          654  +    ** BEFORE trigger runs.  See test case trigger1-18.0 (added 2018-04-26)
          655  +    ** for an example.
   614    656       */
   615    657       for(i=0; i<pTab->nCol; i++){
   616    658         if( aXRef[i]<0 && i!=pTab->iPKey ){
   617    659           sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i);
   618    660         }
   619    661       }
   620    662     }
................................................................................
   622    664     if( !isView ){
   623    665       int addr1 = 0;        /* Address of jump instruction */
   624    666   
   625    667       /* Do constraint checks. */
   626    668       assert( regOldRowid>0 );
   627    669       sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
   628    670           regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace,
   629         -        aXRef);
          671  +        aXRef, 0);
   630    672   
   631    673       /* Do FK constraint checks. */
   632    674       if( hasFK ){
   633    675         sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey);
   634    676       }
   635    677   
   636    678       /* Delete the index entries associated with the current record.  */
................................................................................
   692    734       if( hasFK ){
   693    735         sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey);
   694    736       }
   695    737     }
   696    738   
   697    739     /* Increment the row counter 
   698    740     */
   699         -  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab){
          741  +  if( regRowCount ){
   700    742       sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1);
   701    743     }
   702    744   
   703    745     sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, 
   704    746         TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue);
   705    747   
   706    748     /* Repeat the above with the next record to be updated, until
................................................................................
   719    761     }
   720    762     sqlite3VdbeResolveLabel(v, labelBreak);
   721    763   
   722    764     /* Update the sqlite_sequence table by storing the content of the
   723    765     ** maximum rowid counter values recorded while inserting into
   724    766     ** autoincrement tables.
   725    767     */
   726         -  if( pParse->nested==0 && pParse->pTriggerTab==0 ){
          768  +  if( pParse->nested==0 && pParse->pTriggerTab==0 && pUpsert==0 ){
   727    769       sqlite3AutoincrementEnd(pParse);
   728    770     }
   729    771   
   730    772     /*
   731         -  ** Return the number of rows that were changed. If this routine is 
   732         -  ** generating code because of a call to sqlite3NestedParse(), do not
   733         -  ** invoke the callback function.
          773  +  ** Return the number of rows that were changed, if we are tracking
          774  +  ** that information.
   734    775     */
   735         -  if( (db->flags&SQLITE_CountRows) && !pParse->pTriggerTab && !pParse->nested ){
          776  +  if( regRowCount ){
   736    777       sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1);
   737    778       sqlite3VdbeSetNumCols(v, 1);
   738    779       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC);
   739    780     }
   740    781   
   741    782   update_cleanup:
   742    783     sqlite3AuthContextPop(&sContext);
................................................................................
   849    890       sqlite3VdbeAddOp2(v, OP_SCopy, regArg+2+iPk, regArg+1);
   850    891     }
   851    892   
   852    893     bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy);
   853    894   
   854    895     if( bOnePass ){
   855    896       /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded
   856         -    ** above. Also, if this is a top-level parse (not a trigger), clear the
   857         -    ** multi-write flag so that the VM does not open a statement journal */
          897  +    ** above. */
   858    898       sqlite3VdbeChangeToNoop(v, addr);
   859         -    if( sqlite3IsToplevel(pParse) ){
   860         -      pParse->isMultiWrite = 0;
   861         -    }
   862    899     }else{
   863    900       /* Create a record from the argument register contents and insert it into
   864    901       ** the ephemeral table. */
          902  +    sqlite3MultiWrite(pParse);
   865    903       sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec);
   866    904   #ifdef SQLITE_DEBUG
   867    905       /* Signal an assert() within OP_MakeRecord that it is allowed to
   868    906       ** accept no-change records with serial_type 10 */
   869    907       sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC);
   870    908   #endif
   871    909       sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid);

Added src/upsert.c.

            1  +/*
            2  +** 2018-04-12
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** This file contains code to implement various aspects of UPSERT
           13  +** processing and handling of the Upsert object.
           14  +*/
           15  +#include "sqliteInt.h"
           16  +
           17  +#ifndef SQLITE_OMIT_UPSERT
           18  +/*
           19  +** Free a list of Upsert objects
           20  +*/
           21  +void sqlite3UpsertDelete(sqlite3 *db, Upsert *p){
           22  +  if( p ){
           23  +    sqlite3ExprListDelete(db, p->pUpsertTarget);
           24  +    sqlite3ExprDelete(db, p->pUpsertTargetWhere);
           25  +    sqlite3ExprListDelete(db, p->pUpsertSet);
           26  +    sqlite3ExprDelete(db, p->pUpsertWhere);
           27  +    sqlite3DbFree(db, p);
           28  +  }
           29  +}
           30  +
           31  +/*
           32  +** Duplicate an Upsert object.
           33  +*/
           34  +Upsert *sqlite3UpsertDup(sqlite3 *db, Upsert *p){
           35  +  if( p==0 ) return 0;
           36  +  return sqlite3UpsertNew(db,
           37  +           sqlite3ExprListDup(db, p->pUpsertTarget, 0),
           38  +           sqlite3ExprDup(db, p->pUpsertTargetWhere, 0),
           39  +           sqlite3ExprListDup(db, p->pUpsertSet, 0),
           40  +           sqlite3ExprDup(db, p->pUpsertWhere, 0)
           41  +         );
           42  +}
           43  +
           44  +/*
           45  +** Create a new Upsert object.
           46  +*/
           47  +Upsert *sqlite3UpsertNew(
           48  +  sqlite3 *db,           /* Determines which memory allocator to use */
           49  +  ExprList *pTarget,     /* Target argument to ON CONFLICT, or NULL */
           50  +  Expr *pTargetWhere,    /* Optional WHERE clause on the target */
           51  +  ExprList *pSet,        /* UPDATE columns, or NULL for a DO NOTHING */
           52  +  Expr *pWhere           /* WHERE clause for the ON CONFLICT UPDATE */
           53  +){
           54  +  Upsert *pNew;
           55  +  pNew = sqlite3DbMallocRaw(db, sizeof(Upsert));
           56  +  if( pNew==0 ){
           57  +    sqlite3ExprListDelete(db, pTarget);
           58  +    sqlite3ExprDelete(db, pTargetWhere);
           59  +    sqlite3ExprListDelete(db, pSet);
           60  +    sqlite3ExprDelete(db, pWhere);
           61  +    return 0;
           62  +  }else{
           63  +    pNew->pUpsertTarget = pTarget;
           64  +    pNew->pUpsertTargetWhere = pTargetWhere;
           65  +    pNew->pUpsertSet = pSet;
           66  +    pNew->pUpsertWhere = pWhere;
           67  +    pNew->pUpsertIdx = 0;
           68  +  }
           69  +  return pNew;
           70  +}
           71  +
           72  +/*
           73  +** Analyze the ON CONFLICT clause described by pUpsert.  Resolve all
           74  +** symbols in the conflict-target.
           75  +**
           76  +** Return SQLITE_OK if everything works, or an error code is something
           77  +** is wrong.
           78  +*/
           79  +int sqlite3UpsertAnalyzeTarget(
           80  +  Parse *pParse,     /* The parsing context */
           81  +  SrcList *pTabList, /* Table into which we are inserting */
           82  +  Upsert *pUpsert    /* The ON CONFLICT clauses */
           83  +){
           84  +  Table *pTab;            /* That table into which we are inserting */
           85  +  int rc;                 /* Result code */
           86  +  int iCursor;            /* Cursor used by pTab */
           87  +  Index *pIdx;            /* One of the indexes of pTab */
           88  +  ExprList *pTarget;      /* The conflict-target clause */
           89  +  Expr *pTerm;            /* One term of the conflict-target clause */
           90  +  NameContext sNC;        /* Context for resolving symbolic names */
           91  +  Expr sCol[2];           /* Index column converted into an Expr */
           92  +
           93  +  assert( pTabList->nSrc==1 );
           94  +  assert( pTabList->a[0].pTab!=0 );
           95  +  assert( pUpsert!=0 );
           96  +  assert( pUpsert->pUpsertTarget!=0 );
           97  +
           98  +  /* Resolve all symbolic names in the conflict-target clause, which
           99  +  ** includes both the list of columns and the optional partial-index
          100  +  ** WHERE clause.
          101  +  */
          102  +  memset(&sNC, 0, sizeof(sNC));
          103  +  sNC.pParse = pParse;
          104  +  sNC.pSrcList = pTabList;
          105  +  rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget);
          106  +  if( rc ) return rc;
          107  +  rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere);
          108  +  if( rc ) return rc;
          109  +
          110  +  /* Check to see if the conflict target matches the rowid. */  
          111  +  pTab = pTabList->a[0].pTab;
          112  +  pTarget = pUpsert->pUpsertTarget;
          113  +  iCursor = pTabList->a[0].iCursor;
          114  +  if( HasRowid(pTab) 
          115  +   && pTarget->nExpr==1
          116  +   && (pTerm = pTarget->a[0].pExpr)->op==TK_COLUMN
          117  +   && pTerm->iColumn==XN_ROWID
          118  +  ){
          119  +    /* The conflict-target is the rowid of the primary table */
          120  +    assert( pUpsert->pUpsertIdx==0 );
          121  +    return SQLITE_OK;
          122  +  }
          123  +
          124  +  /* Initialize sCol[0..1] to be an expression parse tree for a
          125  +  ** single column of an index.  The sCol[0] node will be the TK_COLLATE
          126  +  ** operator and sCol[1] will be the TK_COLUMN operator.  Code below
          127  +  ** will populate the specific collation and column number values
          128  +  ** prior to comparing against the conflict-target expression.
          129  +  */
          130  +  memset(sCol, 0, sizeof(sCol));
          131  +  sCol[0].op = TK_COLLATE;
          132  +  sCol[0].pLeft = &sCol[1];
          133  +  sCol[1].op = TK_COLUMN;
          134  +  sCol[1].iTable = pTabList->a[0].iCursor;
          135  +
          136  +  /* Check for matches against other indexes */
          137  +  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
          138  +    int ii, jj, nn;
          139  +    if( !IsUniqueIndex(pIdx) ) continue;
          140  +    if( pTarget->nExpr!=pIdx->nKeyCol ) continue;
          141  +    if( pIdx->pPartIdxWhere ){
          142  +      if( pUpsert->pUpsertTargetWhere==0 ) continue;
          143  +      if( sqlite3ExprCompare(pParse, pUpsert->pUpsertTargetWhere,
          144  +                             pIdx->pPartIdxWhere, iCursor)!=0 ){
          145  +        continue;
          146  +      }
          147  +    }
          148  +    nn = pIdx->nKeyCol;
          149  +    for(ii=0; ii<nn; ii++){
          150  +      Expr *pExpr;
          151  +      sCol[0].u.zToken = (char*)pIdx->azColl[ii];
          152  +      if( pIdx->aiColumn[ii]==XN_EXPR ){
          153  +        assert( pIdx->aColExpr!=0 );
          154  +        assert( pIdx->aColExpr->nExpr>ii );
          155  +        pExpr = pIdx->aColExpr->a[ii].pExpr;
          156  +        if( pExpr->op!=TK_COLLATE ){
          157  +          sCol[0].pLeft = pExpr;
          158  +          pExpr = &sCol[0];
          159  +        }
          160  +      }else{
          161  +        sCol[0].pLeft = &sCol[1];
          162  +        sCol[1].iColumn = pIdx->aiColumn[ii];
          163  +        pExpr = &sCol[0];
          164  +      }
          165  +      for(jj=0; jj<nn; jj++){
          166  +        if( sqlite3ExprCompare(pParse, pTarget->a[jj].pExpr, pExpr,iCursor)<2 ){
          167  +          break;  /* Column ii of the index matches column jj of target */
          168  +        }
          169  +      }
          170  +      if( jj>=nn ){
          171  +        /* The target contains no match for column jj of the index */
          172  +        break;
          173  +      }
          174  +    }
          175  +    if( ii<nn ){
          176  +      /* Column ii of the index did not match any term of the conflict target.
          177  +      ** Continue the search with the next index. */
          178  +      continue;
          179  +    }
          180  +    pUpsert->pUpsertIdx = pIdx;
          181  +    return SQLITE_OK;
          182  +  }
          183  +  sqlite3ErrorMsg(pParse, "ON CONFLICT clause does not match any "
          184  +                          "PRIMARY KEY or UNIQUE constraint");
          185  +  return SQLITE_ERROR;
          186  +}
          187  +
          188  +/*
          189  +** Generate bytecode that does an UPDATE as part of an upsert.
          190  +**
          191  +** If pIdx is NULL, then the UNIQUE constraint that failed was the IPK.
          192  +** In this case parameter iCur is a cursor open on the table b-tree that
          193  +** currently points to the conflicting table row. Otherwise, if pIdx
          194  +** is not NULL, then pIdx is the constraint that failed and iCur is a
          195  +** cursor points to the conflicting row.
          196  +*/
          197  +void sqlite3UpsertDoUpdate(
          198  +  Parse *pParse,        /* The parsing and code-generating context */
          199  +  Upsert *pUpsert,      /* The ON CONFLICT clause for the upsert */
          200  +  Table *pTab,          /* The table being updated */
          201  +  Index *pIdx,          /* The UNIQUE constraint that failed */
          202  +  int iCur              /* Cursor for pIdx (or pTab if pIdx==NULL) */
          203  +){
          204  +  Vdbe *v = pParse->pVdbe;
          205  +  sqlite3 *db = pParse->db;
          206  +  SrcList *pSrc;            /* FROM clause for the UPDATE */
          207  +  int iDataCur = pUpsert->iDataCur;
          208  +
          209  +  assert( v!=0 );
          210  +  VdbeNoopComment((v, "Begin DO UPDATE of UPSERT"));
          211  +  if( pIdx && iCur!=iDataCur ){
          212  +    if( HasRowid(pTab) ){
          213  +      int regRowid = sqlite3GetTempReg(pParse);
          214  +      sqlite3VdbeAddOp2(v, OP_IdxRowid, iCur, regRowid);
          215  +      sqlite3VdbeAddOp3(v, OP_SeekRowid, iDataCur, 0, regRowid);
          216  +      VdbeCoverage(v);
          217  +      sqlite3ReleaseTempReg(pParse, regRowid);
          218  +    }else{
          219  +      Index *pPk = sqlite3PrimaryKeyIndex(pTab);
          220  +      int nPk = pPk->nKeyCol;
          221  +      int iPk = pParse->nMem+1;
          222  +      int i;
          223  +      pParse->nMem += nPk;
          224  +      for(i=0; i<nPk; i++){
          225  +        int k;
          226  +        assert( pPk->aiColumn[i]>=0 );
          227  +        k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]);
          228  +        sqlite3VdbeAddOp3(v, OP_Column, iCur, k, iPk+i);
          229  +        VdbeComment((v, "%s.%s", pIdx->zName,
          230  +                    pTab->aCol[pPk->aiColumn[i]].zName));
          231  +      }
          232  +      i = sqlite3VdbeAddOp4Int(v, OP_Found, iDataCur, 0, iPk, nPk);
          233  +      VdbeCoverage(v);
          234  +      sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CORRUPT, OE_Abort, 0, 
          235  +            "corrupt database", P4_STATIC);
          236  +      sqlite3VdbeJumpHere(v, i);
          237  +    }
          238  +  }
          239  +  /* pUpsert does not own pUpsertSrc - the outer INSERT statement does.  So
          240  +  ** we have to make a copy before passing it down into sqlite3Update() */
          241  +  pSrc = sqlite3SrcListDup(db, pUpsert->pUpsertSrc, 0);
          242  +  sqlite3Update(pParse, pSrc, pUpsert->pUpsertSet,
          243  +      pUpsert->pUpsertWhere, OE_Abort, 0, 0, pUpsert);
          244  +  pUpsert->pUpsertSet = 0;    /* Will have been deleted by sqlite3Update() */
          245  +  pUpsert->pUpsertWhere = 0;  /* Will have been deleted by sqlite3Update() */
          246  +  VdbeNoopComment((v, "End DO UPDATE of UPSERT"));
          247  +}
          248  +
          249  +#endif /* SQLITE_OMIT_UPSERT */

Changes to src/vacuum.c.

    35     35   
    36     36     /* printf("SQL: [%s]\n", zSql); fflush(stdout); */
    37     37     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
    38     38     if( rc!=SQLITE_OK ) return rc;
    39     39     while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
    40     40       const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0);
    41     41       assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 );
    42         -    assert( sqlite3_strnicmp(zSubSql,"SELECT",6)!=0 || CORRUPT_DB );
    43         -    if( zSubSql && zSubSql[0]!='S' ){
           42  +    /* The secondary SQL must be one of CREATE TABLE, CREATE INDEX,
           43  +    ** or INSERT.  Historically there have been attacks that first
           44  +    ** corrupt the sqlite_master.sql field with other kinds of statements
           45  +    ** then run VACUUM to get those statements to execute at inappropriate
           46  +    ** times. */
           47  +    if( zSubSql
           48  +     && (strncmp(zSubSql,"CRE",3)==0 || strncmp(zSubSql,"INS",3)==0)
           49  +    ){
    44     50         rc = execSql(db, pzErrMsg, zSubSql);
    45     51         if( rc!=SQLITE_OK ) break;
    46     52       }
    47     53     }
    48     54     assert( rc!=SQLITE_ROW );
    49     55     if( rc==SQLITE_DONE ) rc = SQLITE_OK;
    50     56     if( rc ){
................................................................................
   249    255         " WHERE type='table'AND name<>'sqlite_sequence'"
   250    256         " AND coalesce(rootpage,1)>0",
   251    257         zDbMain
   252    258     );
   253    259     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   254    260     rc = execSqlF(db, pzErrMsg,
   255    261         "SELECT sql FROM \"%w\".sqlite_master"
   256         -      " WHERE type='index' AND length(sql)>10",
          262  +      " WHERE type='index'",
   257    263         zDbMain
   258    264     );
   259    265     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   260    266     db->init.iDb = 0;
   261    267   
   262    268     /* Loop through the tables in the main database. For each, do
   263    269     ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy

Changes to src/vdbe.h.

   193    193   #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
   194    194     void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N);
   195    195     void sqlite3VdbeVerifyNoResultRow(Vdbe *p);
   196    196   #else
   197    197   # define sqlite3VdbeVerifyNoMallocRequired(A,B)
   198    198   # define sqlite3VdbeVerifyNoResultRow(A)
   199    199   #endif
   200         -VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
          200  +VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno);
          201  +#ifndef SQLITE_OMIT_EXPLAIN
          202  +  void sqlite3VdbeExplain(Parse*,u8,const char*,...);
          203  +  void sqlite3VdbeExplainPop(Parse*);
          204  +  int sqlite3VdbeExplainParent(Parse*);
          205  +# define ExplainQueryPlan(P)        sqlite3VdbeExplain P
          206  +# define ExplainQueryPlanPop(P)     sqlite3VdbeExplainPop(P)
          207  +# define ExplainQueryPlanParent(P)  sqlite3VdbeExplainParent(P)
          208  +#else
          209  +# define ExplainQueryPlan(P)
          210  +# define ExplainQueryPlanPop(P)
          211  +# define ExplainQueryPlanParent(P) 0
          212  +#endif
   201    213   void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
   202    214   void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);
   203    215   void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1);
   204    216   void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2);
   205    217   void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3);
   206    218   void sqlite3VdbeChangeP5(Vdbe*, u16 P5);
   207    219   void sqlite3VdbeJumpHere(Vdbe*, int addr);
................................................................................
   216    228   void sqlite3VdbeRunOnlyOnce(Vdbe*);
   217    229   void sqlite3VdbeReusable(Vdbe*);
   218    230   void sqlite3VdbeDelete(Vdbe*);
   219    231   void sqlite3VdbeClearObject(sqlite3*,Vdbe*);
   220    232   void sqlite3VdbeMakeReady(Vdbe*,Parse*);
   221    233   int sqlite3VdbeFinalize(Vdbe*);
   222    234   void sqlite3VdbeResolveLabel(Vdbe*, int);
          235  +#ifdef SQLITE_COVERAGE_TEST
          236  +  int sqlite3VdbeLabelHasBeenResolved(Vdbe*,int);
          237  +#endif
   223    238   int sqlite3VdbeCurrentAddr(Vdbe*);
   224    239   #ifdef SQLITE_DEBUG
   225    240     int sqlite3VdbeAssertMayAbort(Vdbe *, int);
   226    241   #endif
   227    242   void sqlite3VdbeResetStepResult(Vdbe*);
   228    243   void sqlite3VdbeRewind(Vdbe*);
   229    244   int sqlite3VdbeReset(Vdbe*);

Changes to src/vdbeaux.c.

   299    299     int p4type          /* P4 operand type */
   300    300   ){
   301    301     char *p4copy = sqlite3DbMallocRawNN(sqlite3VdbeDb(p), 8);
   302    302     if( p4copy ) memcpy(p4copy, zP4, 8);
   303    303     return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type);
   304    304   }
   305    305   
          306  +#ifndef SQLITE_OMIT_EXPLAIN
          307  +/*
          308  +** Return the address of the current EXPLAIN QUERY PLAN baseline.
          309  +** 0 means "none".
          310  +*/
          311  +int sqlite3VdbeExplainParent(Parse *pParse){
          312  +  VdbeOp *pOp;
          313  +  if( pParse->addrExplain==0 ) return 0;
          314  +  pOp = sqlite3VdbeGetOp(pParse->pVdbe, pParse->addrExplain);
          315  +  return pOp->p2;
          316  +}
          317  +
          318  +/*
          319  +** Add a new OP_Explain opcode.
          320  +**
          321  +** If the bPush flag is true, then make this opcode the parent for
          322  +** subsequent Explains until sqlite3VdbeExplainPop() is called.
          323  +*/
          324  +void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){
          325  +  if( pParse->explain==2 ){
          326  +    char *zMsg;
          327  +    Vdbe *v = pParse->pVdbe;
          328  +    va_list ap;
          329  +    int iThis;
          330  +    va_start(ap, zFmt);
          331  +    zMsg = sqlite3VMPrintf(pParse->db, zFmt, ap);
          332  +    va_end(ap);
          333  +    v = pParse->pVdbe;
          334  +    iThis = v->nOp;
          335  +    sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0,
          336  +                      zMsg, P4_DYNAMIC);
          337  +    if( bPush) pParse->addrExplain = iThis;
          338  +  }
          339  +}
          340  +
          341  +/*
          342  +** Pop the EXPLAIN QUERY PLAN stack one level.
          343  +*/
          344  +void sqlite3VdbeExplainPop(Parse *pParse){
          345  +  pParse->addrExplain = sqlite3VdbeExplainParent(pParse);
          346  +}
          347  +#endif /* SQLITE_OMIT_EXPLAIN */
          348  +
   306    349   /*
   307    350   ** Add an OP_ParseSchema opcode.  This routine is broken out from
   308    351   ** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees
   309    352   ** as having been used.
   310    353   **
   311    354   ** The zWhere string must have been obtained from sqlite3_malloc().
   312    355   ** This routine will take ownership of the allocated memory.
................................................................................
   388    431   void sqlite3VdbeResolveLabel(Vdbe *v, int x){
   389    432     Parse *p = v->pParse;
   390    433     int j = ADDR(x);
   391    434     assert( v->magic==VDBE_MAGIC_INIT );
   392    435     assert( j<p->nLabel );
   393    436     assert( j>=0 );
   394    437     if( p->aLabel ){
          438  +#ifdef SQLITE_DEBUG
          439  +    if( p->db->flags & SQLITE_VdbeAddopTrace ){
          440  +      printf("RESOLVE LABEL %d to %d\n", x, v->nOp);
          441  +    }
          442  +#endif
          443  +    assert( p->aLabel[j]==(-1) ); /* Labels may only be resolved once */
   395    444       p->aLabel[j] = v->nOp;
   396    445     }
   397    446   }
   398    447   
          448  +#ifdef SQLITE_COVERAGE_TEST
          449  +/*
          450  +** Return TRUE if and only if the label x has already been resolved.
          451  +** Return FALSE (zero) if label x is still unresolved.
          452  +**
          453  +** This routine is only used inside of testcase() macros, and so it
          454  +** only exists when measuring test coverage.
          455  +*/
          456  +int sqlite3VdbeLabelHasBeenResolved(Vdbe *v, int x){
          457  +  return v->pParse->aLabel && v->pParse->aLabel[ADDR(x)]>=0;
          458  +}
          459  +#endif /* SQLITE_COVERAGE_TEST */
          460  +
   399    461   /*
   400    462   ** Mark the VDBE as one that can only be run one time.
   401    463   */
   402    464   void sqlite3VdbeRunOnlyOnce(Vdbe *p){
   403    465     p->runOnlyOnce = 1;
   404    466   }
   405    467   
................................................................................
  1622   1684   ** running the code, it invokes the callback once for each instruction.
  1623   1685   ** This feature is used to implement "EXPLAIN".
  1624   1686   **
  1625   1687   ** When p->explain==1, each instruction is listed.  When
  1626   1688   ** p->explain==2, only OP_Explain instructions are listed and these
  1627   1689   ** are shown in a different format.  p->explain==2 is used to implement
  1628   1690   ** EXPLAIN QUERY PLAN.
         1691  +** 2018-04-24:  In p->explain==2 mode, the OP_Init opcodes of triggers
         1692  +** are also shown, so that the boundaries between the main program and
         1693  +** each trigger are clear.
  1629   1694   **
  1630   1695   ** When p->explain==1, first the main program is listed, then each of
  1631   1696   ** the trigger subprograms are listed one by one.
  1632   1697   */
  1633   1698   int sqlite3VdbeList(
  1634   1699     Vdbe *p                   /* The VDBE */
  1635   1700   ){
................................................................................
  1684   1749         apSub = (SubProgram **)pSub->z;
  1685   1750       }
  1686   1751       for(i=0; i<nSub; i++){
  1687   1752         nRow += apSub[i]->nOp;
  1688   1753       }
  1689   1754     }
  1690   1755   
  1691         -  do{
         1756  +  while(1){  /* Loop exits via break */
  1692   1757       i = p->pc++;
  1693   1758       if( i>=nRow ){
  1694   1759         p->rc = SQLITE_OK;
  1695   1760         rc = SQLITE_DONE;
  1696   1761         break;
  1697   1762       }
  1698   1763       if( i<p->nOp ){
................................................................................
  1730   1795           apSub = (SubProgram **)pSub->z;
  1731   1796           apSub[nSub++] = pOp->p4.pProgram;
  1732   1797           pSub->flags |= MEM_Blob;
  1733   1798           pSub->n = nSub*sizeof(SubProgram*);
  1734   1799           nRow += pOp->p4.pProgram->nOp;
  1735   1800         }
  1736   1801       }
  1737         -  }while( p->explain==2 && pOp->opcode!=OP_Explain );
         1802  +    if( p->explain<2 ) break;
         1803  +    if( pOp->opcode==OP_Explain ) break;
         1804  +    if( pOp->opcode==OP_Init && p->pc>1 ) break;
         1805  +  }
  1738   1806   
  1739   1807     if( rc==SQLITE_OK ){
  1740   1808       if( db->u1.isInterrupted ){
  1741   1809         p->rc = SQLITE_INTERRUPT;
  1742   1810         rc = SQLITE_ERROR;
  1743   1811         sqlite3VdbeError(p, sqlite3ErrStr(p->rc));
  1744   1812       }else{

Changes to src/vdbemem.c.

  1474   1474       zVal = &pExpr->u.zToken[2];
  1475   1475       nVal = sqlite3Strlen30(zVal)-1;
  1476   1476       assert( zVal[nVal]=='\'' );
  1477   1477       sqlite3VdbeMemSetStr(pVal, sqlite3HexToBlob(db, zVal, nVal), nVal/2,
  1478   1478                            0, SQLITE_DYNAMIC);
  1479   1479     }
  1480   1480   #endif
  1481         -
  1482   1481   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1483   1482     else if( op==TK_FUNCTION && pCtx!=0 ){
  1484   1483       rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx);
  1485   1484     }
  1486   1485   #endif
         1486  +  else if( op==TK_TRUEFALSE ){
         1487  +     pVal = valueNew(db, pCtx);
         1488  +     pVal->flags = MEM_Int;
         1489  +     pVal->u.i = pExpr->u.zToken[4]==0;
         1490  +  }
  1487   1491   
  1488   1492     *ppVal = pVal;
  1489   1493     return rc;
  1490   1494   
  1491   1495   no_mem:
  1492   1496   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1493   1497     if( pCtx==0 || pCtx->pParse->nErr==0 )

Changes to src/where.c.

  2416   2416       }
  2417   2417       if( pTerm->prereqRight & pNew->maskSelf ) continue;
  2418   2418   
  2419   2419       /* Do not allow the upper bound of a LIKE optimization range constraint
  2420   2420       ** to mix with a lower range bound from some other source */
  2421   2421       if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
  2422   2422   
  2423         -    /* Do not allow IS constraints from the WHERE clause to be used by the
         2423  +    /* Do not allow constraints from the WHERE clause to be used by the
  2424   2424       ** right table of a LEFT JOIN.  Only constraints in the ON clause are
  2425   2425       ** allowed */
  2426   2426       if( (pSrc->fg.jointype & JT_LEFT)!=0
  2427   2427        && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
  2428         -     && (eOp & (WO_IS|WO_ISNULL))!=0
  2429   2428       ){
  2430         -      testcase( eOp & WO_IS );
  2431         -      testcase( eOp & WO_ISNULL );
  2432   2429         continue;
  2433   2430       }
  2434   2431   
  2435   2432       if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
  2436   2433         pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE;
  2437   2434       }else{
  2438   2435         pBuilder->bldFlags |= SQLITE_BLDF_INDEXED;
................................................................................
  4591   4588     /* Special case: No FROM clause
  4592   4589     */
  4593   4590     if( nTabList==0 ){
  4594   4591       if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
  4595   4592       if( wctrlFlags & WHERE_WANT_DISTINCT ){
  4596   4593         pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
  4597   4594       }
         4595  +    ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
  4598   4596     }else{
  4599   4597       /* Assign a bit from the bitmask to every term in the FROM clause.
  4600   4598       **
  4601   4599       ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
  4602   4600       **
  4603   4601       ** The rule of the previous sentence ensures thta if X is the bitmask for
  4604   4602       ** a table T, then X-1 is the bitmask for all other tables to the left of T.

Changes to src/wherecode.c.

   130    130   #if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
   131    131     if( sqlite3ParseToplevel(pParse)->explain==2 )
   132    132   #endif
   133    133     {
   134    134       struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
   135    135       Vdbe *v = pParse->pVdbe;      /* VM being constructed */
   136    136       sqlite3 *db = pParse->db;     /* Database handle */
   137         -    int iId = pParse->iSelectId;  /* Select id (left-most output column) */
   138    137       int isSearch;                 /* True for a SEARCH. False for SCAN. */
   139    138       WhereLoop *pLoop;             /* The controlling WhereLoop object */
   140    139       u32 flags;                    /* Flags that describe this loop */
   141    140       char *zMsg;                   /* Text to add to EQP output */
   142    141       StrAccum str;                 /* EQP output string */
   143    142       char zBuf[100];               /* Initial space for EQP output string */
   144    143   
................................................................................
   149    148       isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
   150    149               || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
   151    150               || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
   152    151   
   153    152       sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
   154    153       sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN");
   155    154       if( pItem->pSelect ){
   156         -      sqlite3XPrintf(&str, " SUBQUERY %d", pItem->iSelectId);
          155  +      sqlite3XPrintf(&str, " SUBQUERY 0x%p", pItem->pSelect);
   157    156       }else{
   158    157         sqlite3XPrintf(&str, " TABLE %s", pItem->zName);
   159    158       }
   160    159   
   161    160       if( pItem->zAlias ){
   162    161         sqlite3XPrintf(&str, " AS %s", pItem->zAlias);
   163    162       }
................................................................................
   210    209       if( pLoop->nOut>=10 ){
   211    210         sqlite3XPrintf(&str, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut));
   212    211       }else{
   213    212         sqlite3StrAccumAppend(&str, " (~1 row)", 9);
   214    213       }
   215    214   #endif
   216    215       zMsg = sqlite3StrAccumFinish(&str);
   217         -    ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC);
          216  +    ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
          217  +                            pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
   218    218     }
   219    219     return ret;
   220    220   }
   221    221   #endif /* SQLITE_OMIT_EXPLAIN */
   222    222   
   223    223   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   224    224   /*
................................................................................
  1211   1211     addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
  1212   1212     addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v);
  1213   1213   
  1214   1214     /* If this is the right table of a LEFT OUTER JOIN, allocate and
  1215   1215     ** initialize a memory cell that records if this table matches any
  1216   1216     ** row of the left table of the join.
  1217   1217     */
         1218  +  assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
         1219  +       || pLevel->iFrom>0 || (pTabItem[0].fg.jointype & JT_LEFT)==0
         1220  +  );
  1218   1221     if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){
  1219   1222       pLevel->iLeftJoin = ++pParse->nMem;
  1220   1223       sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin);
  1221   1224       VdbeComment((v, "init LEFT JOIN no-match flag"));
  1222   1225     }
  1223   1226   
  1224   1227     /* Compute a safe address to jump to if we discover that the table for
................................................................................
  1744   1747         sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont,
  1745   1748                              iRowidReg, pPk->nKeyCol); VdbeCoverage(v);
  1746   1749       }
  1747   1750   
  1748   1751       /* If pIdx is an index on one or more expressions, then look through
  1749   1752       ** all the expressions in pWInfo and try to transform matching expressions
  1750   1753       ** into reference to index columns.
         1754  +    **
         1755  +    ** Do not do this for the RHS of a LEFT JOIN. This is because the 
         1756  +    ** expression may be evaluated after OP_NullRow has been executed on
         1757  +    ** the cursor. In this case it is important to do the full evaluation,
         1758  +    ** as the result of the expression may not be NULL, even if all table
         1759  +    ** column values are.  https://www.sqlite.org/src/info/7fa8049685b50b5a
  1751   1760       */
  1752         -    whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
  1753         -
         1761  +    if( pLevel->iLeftJoin==0 ){
         1762  +      whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
         1763  +    }
  1754   1764   
  1755   1765       /* Record the instruction used to terminate the loop. */
  1756   1766       if( pLoop->wsFlags & WHERE_ONEROW ){
  1757   1767         pLevel->op = OP_Noop;
  1758   1768       }else if( bRev ){
  1759   1769         pLevel->op = OP_Prev;
  1760   1770       }else{
................................................................................
  1902   1912       ** See ticket http://www.sqlite.org/src/info/f2369304e4
  1903   1913       */
  1904   1914       if( pWC->nTerm>1 ){
  1905   1915         int iTerm;
  1906   1916         for(iTerm=0; iTerm<pWC->nTerm; iTerm++){
  1907   1917           Expr *pExpr = pWC->a[iTerm].pExpr;
  1908   1918           if( &pWC->a[iTerm] == pTerm ) continue;
  1909         -        if( ExprHasProperty(pExpr, EP_FromJoin) ) continue;
  1910   1919           testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL );
  1911   1920           testcase( pWC->a[iTerm].wtFlags & TERM_CODED );
  1912   1921           if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue;
  1913   1922           if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue;
  1914   1923           testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO );
  1915   1924           pExpr = sqlite3ExprDup(db, pExpr, 0);
  1916   1925           pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr);
................................................................................
  1921   1930       }
  1922   1931   
  1923   1932       /* Run a separate WHERE clause for each term of the OR clause.  After
  1924   1933       ** eliminating duplicates from other WHERE clauses, the action for each
  1925   1934       ** sub-WHERE clause is to to invoke the main loop body as a subroutine.
  1926   1935       */
  1927   1936       wctrlFlags =  WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
         1937  +    ExplainQueryPlan((pParse, 1, "MULTI-INDEX OR"));
  1928   1938       for(ii=0; ii<pOrWc->nTerm; ii++){
  1929   1939         WhereTerm *pOrTerm = &pOrWc->a[ii];
  1930   1940         if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){
  1931   1941           WhereInfo *pSubWInfo;           /* Info for single OR-term scan */
  1932   1942           Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */
  1933   1943           int jmp1 = 0;                   /* Address of jump operation */
  1934         -        if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){
         1944  +        assert( (pTabItem[0].fg.jointype & JT_LEFT)==0 
         1945  +             || ExprHasProperty(pOrExpr, EP_FromJoin) 
         1946  +        );
         1947  +        if( pAndExpr ){
  1935   1948             pAndExpr->pLeft = pOrExpr;
  1936   1949             pOrExpr = pAndExpr;
  1937   1950           }
  1938   1951           /* Loop through table entries that match term pOrTerm. */
  1939   1952           WHERETRACE(0xffff, ("Subplan for OR-clause:\n"));
  1940   1953           pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0,
  1941   1954                                         wctrlFlags, iCovCur);
................................................................................
  2038   2051             }
  2039   2052   
  2040   2053             /* Finish the loop through table entries that match term pOrTerm. */
  2041   2054             sqlite3WhereEnd(pSubWInfo);
  2042   2055           }
  2043   2056         }
  2044   2057       }
         2058  +    ExplainQueryPlanPop(pParse);
  2045   2059       pLevel->u.pCovidx = pCov;
  2046   2060       if( pCov ) pLevel->iIdxCur = iCovCur;
  2047   2061       if( pAndExpr ){
  2048   2062         pAndExpr->pLeft = 0;
  2049   2063         sqlite3ExprDelete(db, pAndExpr);
  2050   2064       }
  2051   2065       sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v));
................................................................................
  2110   2124           testcase( pWInfo->untestedTerms==0
  2111   2125               && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
  2112   2126           pWInfo->untestedTerms = 1;
  2113   2127           continue;
  2114   2128         }
  2115   2129         pE = pTerm->pExpr;
  2116   2130         assert( pE!=0 );
  2117         -      if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){
         2131  +      if( (pTabItem->fg.jointype&JT_LEFT) && !ExprHasProperty(pE,EP_FromJoin) ){
  2118   2132           continue;
  2119   2133         }
  2120   2134         
  2121   2135         if( iLoop==1 && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){
  2122   2136           iNext = 2;
  2123   2137           continue;
  2124   2138         }

Changes to src/whereexpr.c.

   814    814           testcase( idxNew==0 );
   815    815           exprAnalyze(pSrc, pWC, idxNew);
   816    816           pTerm = &pWC->a[idxTerm];
   817    817           markTermAsChild(pWC, idxNew, idxTerm);
   818    818         }else{
   819    819           sqlite3ExprListDelete(db, pList);
   820    820         }
   821         -      pTerm->eOperator = WO_NOOP;  /* case 1 trumps case 3 */
   822    821       }
   823    822     }
   824    823   }
   825    824   #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */
   826    825   
   827    826   /*
   828    827   ** We already know that pExpr is a binary operator where both operands are

Changes to test/analyze3.test.

   114    114   
   115    115   # The first of the following two SELECT statements visits 99 rows. So
   116    116   # it is better to use the index. But the second visits every row in 
   117    117   # the table (1000 in total) so it is better to do a full-table scan.
   118    118   #
   119    119   do_eqp_test analyze3-1.1.2 {
   120    120     SELECT sum(y) FROM t1 WHERE x>200 AND x<300
   121         -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}}
          121  +} {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}
   122    122   do_eqp_test analyze3-1.1.3 {
   123    123     SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 
   124         -} {0 0 0 {SCAN TABLE t1}}
          124  +} {SCAN TABLE t1}
   125    125   
   126    126   # 2017-06-26:  Verify that the SQLITE_DBCONFIG_ENABLE_QPSG setting disables
   127    127   # the use of bound parameters by STAT4
   128    128   #
   129    129   db cache flush
   130    130   unset -nocomplain l
   131    131   unset -nocomplain u
   132    132   do_eqp_test analyze3-1.1.3.100 {
   133    133     SELECT sum(y) FROM t1 WHERE x>$l AND x<$u
   134         -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}}
          134  +} {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}
   135    135   set l 200
   136    136   set u 300
   137    137   do_eqp_test analyze3-1.1.3.101 {
   138    138     SELECT sum(y) FROM t1 WHERE x>$l AND x<$u
   139         -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}}
          139  +} {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}
   140    140   set l 0
   141    141   set u 1100
   142    142   do_eqp_test analyze3-1.1.3.102 {
   143    143     SELECT sum(y) FROM t1 WHERE x>$l AND x<$u
   144         -} {0 0 0 {SCAN TABLE t1}}
          144  +} {SCAN TABLE t1}
   145    145   db cache flush
   146    146   sqlite3_db_config db ENABLE_QPSG 1
   147    147   do_eqp_test analyze3-1.1.3.103 {
   148    148     SELECT sum(y) FROM t1 WHERE x>$l AND x<$u
   149         -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}}
          149  +} {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?)}
   150    150   db cache flush
   151    151   sqlite3_db_config db ENABLE_QPSG 0
   152    152   do_eqp_test analyze3-1.1.3.104 {
   153    153     SELECT sum(y) FROM t1 WHERE x>$l AND x<$u
   154         -} {0 0 0 {SCAN TABLE t1}}
          154  +} {SCAN TABLE t1}
   155    155   
   156    156   do_test analyze3-1.1.4 {
   157    157     sf_execsql { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 }
   158    158   } {199 0 14850}
   159    159   do_test analyze3-1.1.5 {
   160    160     set l [string range "200" 0 end]
   161    161     set u [string range "300" 0 end]
................................................................................
   197    197   } {}
   198    198   do_execsql_test analyze3-2.1.x {
   199    199     SELECT count(*) FROM t2 WHERE x>1 AND x<2;
   200    200     SELECT count(*) FROM t2 WHERE x>0 AND x<99;
   201    201   } {200 990}
   202    202   do_eqp_test analyze3-1.2.2 {
   203    203     SELECT sum(y) FROM t2 WHERE x>1 AND x<2
   204         -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x<?)}}
          204  +} {SEARCH TABLE t2 USING INDEX i2 (x>? AND x<?)}
   205    205   do_eqp_test analyze3-1.2.3 {
   206    206     SELECT sum(y) FROM t2 WHERE x>0 AND x<99
   207         -} {0 0 0 {SCAN TABLE t2}}
          207  +} {SCAN TABLE t2}
   208    208   
   209    209   do_test analyze3-1.2.4 {
   210    210     sf_execsql { SELECT sum(y) FROM t2 WHERE x>12 AND x<20 }
   211    211   } {161 0 4760}
   212    212   do_test analyze3-1.2.5 {
   213    213     set l [string range "12" 0 end]
   214    214     set u [string range "20" 0 end]
................................................................................
   249    249   } {}
   250    250   do_execsql_test analyze3-1.3.x {
   251    251     SELECT count(*) FROM t3 WHERE x>200 AND x<300;
   252    252     SELECT count(*) FROM t3 WHERE x>0 AND x<1100
   253    253   } {99 1000}
   254    254   do_eqp_test analyze3-1.3.2 {
   255    255     SELECT sum(y) FROM t3 WHERE x>200 AND x<300
   256         -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x<?)}}
          256  +} {SEARCH TABLE t3 USING INDEX i3 (x>? AND x<?)}
   257    257   do_eqp_test analyze3-1.3.3 {
   258    258     SELECT sum(y) FROM t3 WHERE x>0 AND x<1100
   259         -} {0 0 0 {SCAN TABLE t3}}
          259  +} {SCAN TABLE t3}
   260    260   
   261    261   do_test analyze3-1.3.4 {
   262    262     sf_execsql { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 }
   263    263   } {199 0 14850}
   264    264   do_test analyze3-1.3.5 {
   265    265     set l [string range "200" 0 end]
   266    266     set u [string range "300" 0 end]
................................................................................
   304    304       append t [lindex {a b c d e f g h i j} [expr ($i%10)]]
   305    305       execsql { INSERT INTO t1 VALUES($i, $t) }
   306    306     }
   307    307     execsql COMMIT
   308    308   } {}
   309    309   do_eqp_test analyze3-2.2 {
   310    310     SELECT count(a) FROM t1 WHERE b LIKE 'a%'
   311         -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (b>? AND b<?)}}
          311  +} {SEARCH TABLE t1 USING INDEX i1 (b>? AND b<?)}
   312    312   do_eqp_test analyze3-2.3 {
   313    313     SELECT count(a) FROM t1 WHERE b LIKE '%a'
   314         -} {0 0 0 {SCAN TABLE t1}}
          314  +} {SCAN TABLE t1}
   315    315   
   316    316   # Return the first argument if like_match_blobs is true (the default)
   317    317   # or the second argument if not
   318    318   #
   319    319   proc ilmb {a b} {
   320    320     ifcapable like_match_blobs {return $a}
   321    321     return $b
................................................................................
   694    694     }
   695    695     execsql COMMIT
   696    696     execsql ANALYZE
   697    697   } {}
   698    698   
   699    699   do_eqp_test analyze3-6-3 {
   700    700     SELECT * FROM t1 WHERE a = 5 AND c = 13;
   701         -} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (c=?)}}
          701  +} {SEARCH TABLE t1 USING INDEX i2 (c=?)}
   702    702   
   703    703   do_eqp_test analyze3-6-2 {
   704    704     SELECT * FROM t1 WHERE a = 5 AND b > 'w' AND c = 13;
   705         -} {0 0 0 {SEARCH TABLE t1 USING INDEX i2 (c=?)}}
          705  +} {SEARCH TABLE t1 USING INDEX i2 (c=?)}
   706    706   
   707    707   #-----------------------------------------------------------------------------
   708    708   # 2015-04-20.
   709    709   # Memory leak in sqlite3Stat4ProbeFree().  (Discovered while fuzzing.)
   710    710   #
   711    711   do_execsql_test analyze-7.1 {
   712    712     DROP TABLE IF EXISTS t1;

Changes to test/analyze4.test.

    34     34       INSERT INTO t1 SELECT a+32, b FROM t1;
    35     35       INSERT INTO t1 SELECT a+64, b FROM t1;
    36     36       ANALYZE;
    37     37     }
    38     38   
    39     39     # Should choose the t1a index since it is more specific than t1b.
    40     40     db eval {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=5 AND b IS NULL}
    41         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           41  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    42     42   
    43     43   # Verify that the t1b index shows that it does not narrow down the
    44     44   # search any at all.
    45     45   #
    46     46   do_test analyze4-1.1 {
    47     47     db eval {
    48     48       SELECT idx, stat FROM sqlite_stat1 WHERE tbl='t1' ORDER BY idx;

Changes to test/analyze6.test.

    57     57   # The lowest cost plan is to scan CAT and for each integer there, do a single
    58     58   # lookup of the first corresponding entry in EV then read off the equal values
    59     59   # in EV.  (Prior to the 2011-03-04 enhancement to where.c, this query would
    60     60   # have used EV for the outer loop instead of CAT - which was about 3x slower.)
    61     61   #
    62     62   do_test analyze6-1.1 {
    63     63     eqp {SELECT count(*) FROM ev, cat WHERE x=y}
    64         -} {0 0 1 {SCAN TABLE cat USING COVERING INDEX catx} 0 1 0 {SEARCH TABLE ev USING COVERING INDEX evy (y=?)}}
           64  +} {/*SCAN TABLE cat USING COVERING INDEX catx*SEARCH TABLE ev USING COVERING INDEX evy (y=?)*/}
    65     65   
    66     66   # The same plan is chosen regardless of the order of the tables in the
    67     67   # FROM clause.
    68     68   #
    69         -do_test analyze6-1.2 {
    70         -  eqp {SELECT count(*) FROM cat, ev WHERE x=y}
    71         -} {0 0 0 {SCAN TABLE cat USING COVERING INDEX catx} 0 1 1 {SEARCH TABLE ev USING COVERING INDEX evy (y=?)}}
           69  +do_eqp_test analyze6-1.2 {
           70  +  SELECT count(*) FROM cat, ev WHERE x=y
           71  +} {
           72  +  QUERY PLAN
           73  +  |--SCAN TABLE cat USING COVERING INDEX catx
           74  +  `--SEARCH TABLE ev USING COVERING INDEX evy (y=?)
           75  +}
    72     76   
    73     77   
    74     78   # Ticket [83ea97620bd3101645138b7b0e71c12c5498fe3d] 2011-03-30
    75     79   # If ANALYZE is run on an empty table, make sure indices are used
    76     80   # on the table.
    77     81   #
    78     82   do_test analyze6-2.1 {
    79     83     execsql {
    80     84       CREATE TABLE t201(x INTEGER PRIMARY KEY, y UNIQUE, z);
    81     85       CREATE INDEX t201z ON t201(z);
    82     86       ANALYZE;
    83     87     }
    84     88     eqp {SELECT * FROM t201 WHERE z=5}
    85         -} {0 0 0 {SEARCH TABLE t201 USING INDEX t201z (z=?)}}
           89  +} {/*SEARCH TABLE t201 USING INDEX t201z (z=?)*/}
    86     90   do_test analyze6-2.2 {
    87     91     eqp {SELECT * FROM t201 WHERE y=5}
    88         -} {0 0 0 {SEARCH TABLE t201 USING INDEX sqlite_autoindex_t201_1 (y=?)}}
           92  +} {/*SEARCH TABLE t201 USING INDEX sqlite_autoindex_t201_1 (y=?)*/}
    89     93   do_test analyze6-2.3 {
    90     94     eqp {SELECT * FROM t201 WHERE x=5}
    91         -} {0 0 0 {SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)}}
           95  +} {/*SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)*/}
    92     96   do_test analyze6-2.4 {
    93     97     execsql {
    94     98       INSERT INTO t201 VALUES(1,2,3),(2,3,4),(3,4,5);
    95     99       ANALYZE t201;
    96    100     }
    97    101     eqp {SELECT * FROM t201 WHERE z=5}
    98         -} {0 0 0 {SEARCH TABLE t201 USING INDEX t201z (z=?)}}
          102  +} {/*SEARCH TABLE t201 USING INDEX t201z (z=?)*/}
    99    103   do_test analyze6-2.5 {
   100    104     eqp {SELECT * FROM t201 WHERE y=5}
   101         -} {0 0 0 {SEARCH TABLE t201 USING INDEX sqlite_autoindex_t201_1 (y=?)}}
          105  +} {/*SEARCH TABLE t201 USING INDEX sqlite_autoindex_t201_1 (y=?)*/}
   102    106   do_test analyze6-2.6 {
   103    107     eqp {SELECT * FROM t201 WHERE x=5}
   104         -} {0 0 0 {SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)}}
          108  +} {/*SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)*/}
   105    109   do_test analyze6-2.7 {
   106    110     execsql {
   107    111       INSERT INTO t201 VALUES(4,5,7);
   108    112       INSERT INTO t201 SELECT x+100, y+100, z+100 FROM t201;
   109    113       INSERT INTO t201 SELECT x+200, y+200, z+200 FROM t201;
   110    114       INSERT INTO t201 SELECT x+400, y+400, z+400 FROM t201;
   111    115       ANALYZE t201;
   112    116     }
   113    117     eqp {SELECT * FROM t201 WHERE z=5}
   114         -} {0 0 0 {SEARCH TABLE t201 USING INDEX t201z (z=?)}}
          118  +} {/*SEARCH TABLE t201 USING INDEX t201z (z=?)*/}
   115    119   do_test analyze6-2.8 {
   116    120     eqp {SELECT * FROM t201 WHERE y=5}
   117         -} {0 0 0 {SEARCH TABLE t201 USING INDEX sqlite_autoindex_t201_1 (y=?)}}
          121  +} {/*SEARCH TABLE t201 USING INDEX sqlite_autoindex_t201_1 (y=?)*/}
   118    122   do_test analyze6-2.9 {
   119    123     eqp {SELECT * FROM t201 WHERE x=5}
   120         -} {0 0 0 {SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)}}
          124  +} {/*SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)*/}
   121    125   
   122    126   finish_test

Changes to test/analyze7.test.

    33     33       CREATE INDEX t1b ON t1(b);
    34     34       CREATE INDEX t1cd ON t1(c,d);
    35     35       CREATE VIRTUAL TABLE nums USING wholenumber;
    36     36       INSERT INTO t1 SELECT value, value, value/100, value FROM nums
    37     37                       WHERE value BETWEEN 1 AND 256;
    38     38       EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123;
    39     39     }
    40         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           40  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    41     41   do_test analyze7-1.1 {
    42     42     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=123;}
    43         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}}
           43  +} {/*SEARCH TABLE t1 USING INDEX t1b (b=?)*/}
    44     44   do_test analyze7-1.2 {
    45     45     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;}
    46         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}}
           46  +} {/*SEARCH TABLE t1 USING INDEX t1cd (c=?)*/}
    47     47   
    48     48   # Run an analyze on one of the three indices.  Verify that this
    49     49   # effects the row-count estimate on the one query that uses that
    50     50   # one index.
    51     51   #
    52     52   do_test analyze7-2.0 {
    53     53     execsql {ANALYZE t1a;}
    54     54     db cache flush
    55     55     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123;}
    56         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           56  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    57     57   do_test analyze7-2.1 {
    58     58     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=123;}
    59         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}}
           59  +} {/*SEARCH TABLE t1 USING INDEX t1b (b=?)*/}
    60     60   do_test analyze7-2.2 {
    61     61     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;}
    62         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}}
           62  +} {/*SEARCH TABLE t1 USING INDEX t1cd (c=?)*/}
    63     63   
    64     64   # Verify that since the query planner now things that t1a is more
    65     65   # selective than t1b, it prefers to use t1a.
    66     66   #
    67     67   do_test analyze7-2.3 {
    68     68     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123 AND b=123}
    69         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           69  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    70     70   
    71     71   # Run an analysis on another of the three indices.  Verify  that this
    72     72   # new analysis works and does not disrupt the previous analysis.
    73     73   #
    74     74   do_test analyze7-3.0 {
    75     75     execsql {ANALYZE t1cd;}
    76     76     db cache flush;
    77     77     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123;}
    78         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           78  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    79     79   do_test analyze7-3.1 {
    80     80     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=123;}
    81         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}}
           81  +} {/*SEARCH TABLE t1 USING INDEX t1b (b=?)*/}
    82     82   do_test analyze7-3.2.1 {
    83     83     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=?;}
    84         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}}
           84  +} {/*SEARCH TABLE t1 USING INDEX t1cd (c=?)*/}
    85     85   ifcapable stat4||stat3 {
    86     86     # If ENABLE_STAT4 is defined, SQLite comes up with a different estimated
    87     87     # row count for (c=2) than it does for (c=?).
    88     88     do_test analyze7-3.2.2 {
    89     89       execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;}
    90         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}}
           90  +  } {/*SEARCH TABLE t1 USING INDEX t1cd (c=?)*/}
    91     91   } else {
    92     92     # If ENABLE_STAT4 is not defined, the expected row count for (c=2) is the
    93     93     # same as that for (c=?).
    94     94     do_test analyze7-3.2.3 {
    95     95       execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;}
    96         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}}
           96  +  } {/*SEARCH TABLE t1 USING INDEX t1cd (c=?)*/}
    97     97   }
    98     98   do_test analyze7-3.3 {
    99     99     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123 AND b=123}
   100         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
          100  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
   101    101   
   102    102   ifcapable {!stat4 && !stat3} {
   103    103     do_test analyze7-3.4 {
   104    104       execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=123 AND b=123}
   105         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}}
          105  +  } {/*SEARCH TABLE t1 USING INDEX t1b (b=?)*/}
   106    106     do_test analyze7-3.5 {
   107    107       execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123 AND c=123}
   108         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
          108  +  } {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
   109    109   }
   110    110   do_test analyze7-3.6 {
   111    111     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=123 AND d=123 AND b=123}
   112         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=? AND d=?)}}
          112  +} {/*SEARCH TABLE t1 USING INDEX t1cd (c=? AND d=?)*/}
   113    113   
   114    114   finish_test

Changes to test/analyze8.test.

    57     57   # with a==100.  And so for those cases, choose the t1b index.
    58     58   #
    59     59   # Buf ro a==99 and a==101, there are far fewer rows so choose
    60     60   # the t1a index.
    61     61   #
    62     62   do_test 1.1 {
    63     63     eqp {SELECT * FROM t1 WHERE a=100 AND b=55}
    64         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}}
           64  +} {/*SEARCH TABLE t1 USING INDEX t1b (b=?)*/}
    65     65   do_test 1.2 {
    66     66     eqp {SELECT * FROM t1 WHERE a=99 AND b=55}
    67         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           67  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    68     68   do_test 1.3 {
    69     69     eqp {SELECT * FROM t1 WHERE a=101 AND b=55}
    70         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           70  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    71     71   do_test 1.4 {
    72     72     eqp {SELECT * FROM t1 WHERE a=100 AND b=56}
    73         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}}
           73  +} {/*SEARCH TABLE t1 USING INDEX t1b (b=?)*/}
    74     74   do_test 1.5 {
    75     75     eqp {SELECT * FROM t1 WHERE a=99 AND b=56}
    76         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           76  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    77     77   do_test 1.6 {
    78     78     eqp {SELECT * FROM t1 WHERE a=101 AND b=56}
    79         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
           79  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
    80     80   do_test 2.1 {
    81     81     eqp {SELECT * FROM t1 WHERE a=100 AND b BETWEEN 50 AND 54}
    82         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}}
           82  +} {/*SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)*/}
    83     83   
    84     84   # There are many more values of c between 0 and 100000 than there are
    85     85   # between 800000 and 900000.  So t1c is more selective for the latter
    86     86   # range.
    87     87   # 
    88     88   # Test 3.2 is a little unstable. It depends on the planner estimating
    89     89   # that (b BETWEEN 30 AND 34) will match more rows than (c BETWEEN
................................................................................
    95     95   do_execsql_test 3.0 {
    96     96     SELECT count(*) FROM t1 WHERE b BETWEEN 30 AND 34;
    97     97     SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 100000;
    98     98     SELECT count(*) FROM t1 WHERE c BETWEEN 800000 AND 900000;
    99     99   } {50 376 32}
   100    100   do_test 3.1 {
   101    101     eqp {SELECT * FROM t1 WHERE b BETWEEN 30 AND 34 AND c BETWEEN 0 AND 100000}
   102         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}}
          102  +} {/*SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)*/}
   103    103   do_test 3.2 {
   104    104     eqp {SELECT * FROM t1
   105    105          WHERE b BETWEEN 30 AND 34 AND c BETWEEN 800000 AND 900000}
   106         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}}
          106  +} {/*SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)*/}
   107    107   do_test 3.3 {
   108    108     eqp {SELECT * FROM t1 WHERE a=100 AND c BETWEEN 0 AND 100000}
   109         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}}
          109  +} {/*SEARCH TABLE t1 USING INDEX t1a (a=?)*/}
   110    110   do_test 3.4 {
   111    111     eqp {SELECT * FROM t1
   112    112          WHERE a=100 AND c BETWEEN 800000 AND 900000}
   113         -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}}
          113  +} {/*SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)*/}
   114    114   
   115    115   finish_test

Changes to test/analyze9.test.

   983    983   } {/*USING INTEGER PRIMARY KEY*/}
   984    984   
   985    985   #-------------------------------------------------------------------------
   986    986   #
   987    987   reset_db
   988    988   do_execsql_test 22.0 {
   989    989     CREATE TABLE t3(a, b, c, d, PRIMARY KEY(a, b)) WITHOUT ROWID;
   990         -}
          990  +  SELECT * FROM t3;
          991  +} {}
   991    992   do_execsql_test 22.1 {
   992    993     WITH r(x) AS (
   993    994       SELECT 1
   994    995       UNION ALL
   995    996       SELECT x+1 FROM r WHERE x<=100
   996    997     )
   997    998   
................................................................................
  1051   1052   
  1052   1053   do_eqp_test 23.1 {
  1053   1054     SELECT * FROM t4 WHERE 
  1054   1055       (e=1 AND b='xyz' AND c='zyx' AND a<'AEA') AND f<300
  1055   1056     -- Formerly used index i41.  But i41 is not a covering index whereas
  1056   1057     -- the PRIMARY KEY is a covering index, and so as of 2017-10-15, the
  1057   1058     -- PRIMARY KEY is preferred.
  1058         -} {
  1059         -  0 0 0 {SEARCH TABLE t4 USING PRIMARY KEY (c=? AND b=? AND a<?)}
  1060         -}
         1059  +} {SEARCH TABLE t4 USING PRIMARY KEY (c=? AND b=? AND a<?)}
  1061   1060   do_eqp_test 23.2 {
  1062   1061     SELECT * FROM t4 WHERE 
  1063   1062       (e=1 AND b='xyz' AND c='zyx' AND a<'JJJ') AND f<300
  1064         -} {
  1065         -  0 0 0 {SEARCH TABLE t4 USING INDEX i42 (f<?)}
  1066         -}
         1063  +} {SEARCH TABLE t4 USING INDEX i42 (f<?)}
  1067   1064   
  1068   1065   do_execsql_test 24.0 {
  1069   1066     CREATE TABLE t5(c, d, b, e, a, PRIMARY KEY(a, b, c)) WITHOUT ROWID;
  1070   1067     WITH data(a, b, c, d, e) AS (
  1071   1068       SELECT 'z', 'y', 0, 0, 0
  1072   1069       UNION ALL
  1073   1070       SELECT 
................................................................................
  1104   1101       CREATE INDEX aa ON t6(a);
  1105   1102       CREATE INDEX bb ON t6(b);
  1106   1103       ANALYZE;
  1107   1104     }
  1108   1105   
  1109   1106     # Term (b<?) is estimated at 25%. Better than (a<30) but not as
  1110   1107     # good as (a<20).
  1111         -  do_eqp_test 25.2.1 { SELECT * FROM t6 WHERE a<30 AND b<? } {
  1112         -    0 0 0 {SEARCH TABLE t6 USING INDEX bb (b<?)}
  1113         -  }
  1114         -  do_eqp_test 25.2.2 { SELECT * FROM t6 WHERE a<20 AND b<? } {
  1115         -    0 0 0 {SEARCH TABLE t6 USING INDEX aa (a<?)}
  1116         -  }
         1108  +  do_eqp_test 25.2.1 { SELECT * FROM t6 WHERE a<30 AND b<? } \
         1109  +    {SEARCH TABLE t6 USING INDEX bb (b<?)}
         1110  +  do_eqp_test 25.2.2 { SELECT * FROM t6 WHERE a<20 AND b<? } \
         1111  +    {SEARCH TABLE t6 USING INDEX aa (a<?)}
  1117   1112   
  1118   1113     # Term (b BETWEEN ? AND ?) is estimated at 1/64.
  1119   1114     do_eqp_test 25.3.1 { 
  1120   1115       SELECT * FROM t6 WHERE a BETWEEN 5 AND 10 AND b BETWEEN ? AND ? 
  1121         -  } {
  1122         -    0 0 0 {SEARCH TABLE t6 USING INDEX bb (b>? AND b<?)}
  1123         -  }
         1116  +  } {SEARCH TABLE t6 USING INDEX bb (b>? AND b<?)}
  1124   1117     
  1125   1118     # Term (b BETWEEN ? AND 60) is estimated to return roughly 15 rows -
  1126   1119     # 60 from (b<=60) multiplied by 0.25 for the b>=? term. Better than
  1127   1120     # (a<20) but not as good as (a<10).
  1128   1121     do_eqp_test 25.4.1 { 
  1129   1122       SELECT * FROM t6 WHERE a < 10 AND (b BETWEEN ? AND 60)
  1130         -  } {
  1131         -    0 0 0 {SEARCH TABLE t6 USING INDEX aa (a<?)}
  1132         -  }
         1123  +  } {SEARCH TABLE t6 USING INDEX aa (a<?)}
         1124  +
  1133   1125     do_eqp_test 25.4.2 { 
  1134   1126       SELECT * FROM t6 WHERE a < 20 AND (b BETWEEN ? AND 60)
  1135         -  } {
  1136         -    0 0 0 {SEARCH TABLE t6 USING INDEX bb (b>? AND b<?)}
  1137         -  }
         1127  +  } {SEARCH TABLE t6 USING INDEX bb (b>? AND b<?)}
  1138   1128   }
  1139   1129   
  1140   1130   #-------------------------------------------------------------------------
  1141   1131   # Check that a problem in they way stat4 data is used has been 
  1142   1132   # resolved (see below).
  1143   1133   #
  1144   1134   reset_db
................................................................................
  1186   1176   # no more than that. Guessing less than 20 is therefore unreasonable.
  1187   1177   #
  1188   1178   # At one point though, due to a problem in whereKeyStats(), the planner was
  1189   1179   # estimating that (x=10000 AND y<50) would match only 2 rows.
  1190   1180   #
  1191   1181   do_eqp_test 26.1.4 {
  1192   1182     SELECT * FROM t1 WHERE x = 10000 AND y < 50 AND z = 444;
  1193         -} {
  1194         -  0 0 0 {SEARCH TABLE t1 USING INDEX t1z (z=?)}
  1195         -}
         1183  +} {SEARCH TABLE t1 USING INDEX t1z (z=?)}
  1196   1184   
  1197   1185   
  1198   1186   # This test - 26.2.* - tests that another manifestation of the same problem
  1199   1187   # is no longer present in the library. Assuming:
  1200   1188   # 
  1201   1189   #   CREATE INDEX t1xy ON t1(x, y)
  1202   1190   #
................................................................................
  1237   1225       UPDATE t1 SET z = (rowid / 95);
  1238   1226       ANALYZE;
  1239   1227     COMMIT;
  1240   1228   }
  1241   1229   
  1242   1230   do_eqp_test 26.2.2 {
  1243   1231     SELECT * FROM t1 WHERE x='B' AND y>25 AND z=?;
  1244         -} {
  1245         -  0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x=? AND y>?)}
  1246         -}
         1232  +} {SEARCH TABLE t1 USING INDEX i1 (x=? AND y>?)}
  1247   1233   
  1248   1234   
  1249   1235   finish_test

Changes to test/analyzeA.test.

   132    132     do_execsql_test 1.$tn.2.1 { SELECT count(*) FROM t1 WHERE b=31 } 1
   133    133     do_execsql_test 1.$tn.2.2 { SELECT count(*) FROM t1 WHERE c=0  } 49
   134    134     do_execsql_test 1.$tn.2.3 { SELECT count(*) FROM t1 WHERE b=125  } 49
   135    135     do_execsql_test 1.$tn.2.4 { SELECT count(*) FROM t1 WHERE c=16  } 1
   136    136   
   137    137     do_eqp_test 1.$tn.2.5 {
   138    138       SELECT * FROM t1 WHERE b = 31 AND c = 0;
   139         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}}
          139  +  } {SEARCH TABLE t1 USING INDEX t1b (b=?)}
   140    140     do_eqp_test 1.$tn.2.6 {
   141    141       SELECT * FROM t1 WHERE b = 125 AND c = 16;
   142         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c=?)}}
          142  +  } {SEARCH TABLE t1 USING INDEX t1c (c=?)}
   143    143   
   144    144     do_execsql_test 1.$tn.3.1 { 
   145    145       SELECT count(*) FROM t1 WHERE b BETWEEN 0 AND 50
   146    146     } {6}
   147    147     do_execsql_test 1.$tn.3.2 { 
   148    148       SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 50
   149    149     } {90}
................................................................................
   152    152     } {90}
   153    153     do_execsql_test 1.$tn.3.4 { 
   154    154       SELECT count(*) FROM t1 WHERE c BETWEEN 75 AND 125
   155    155     } {6}
   156    156   
   157    157     do_eqp_test 1.$tn.3.5 {
   158    158       SELECT * FROM t1 WHERE b BETWEEN 0 AND 50 AND c BETWEEN 0 AND 50
   159         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}}
          159  +  } {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}
   160    160   
   161    161     do_eqp_test 1.$tn.3.6 {
   162    162       SELECT * FROM t1 WHERE b BETWEEN 75 AND 125 AND c BETWEEN 75 AND 125
   163         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}}
          163  +  } {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}
   164    164   
   165    165     do_eqp_test 1.$tn.3.7 {
   166    166       SELECT * FROM t1 WHERE b BETWEEN +0 AND +50 AND c BETWEEN +0 AND +50
   167         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}}
          167  +  } {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}
   168    168   
   169    169     do_eqp_test 1.$tn.3.8 {
   170    170       SELECT * FROM t1
   171    171        WHERE b BETWEEN cast('0' AS int) AND cast('50.0' AS real)
   172    172          AND c BETWEEN cast('0' AS numeric) AND cast('50.0' AS real)
   173         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}}
          173  +  } {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)}
   174    174   
   175    175     do_eqp_test 1.$tn.3.9 {
   176    176       SELECT * FROM t1 WHERE b BETWEEN +75 AND +125 AND c BETWEEN +75 AND +125
   177         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}}
          177  +  } {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}
   178    178   
   179    179     do_eqp_test 1.$tn.3.10 {
   180    180       SELECT * FROM t1
   181    181        WHERE b BETWEEN cast('75' AS int) AND cast('125.0' AS real)
   182    182          AND c BETWEEN cast('75' AS numeric) AND cast('125.0' AS real)
   183         -  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}}
          183  +  } {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?)}
   184    184   }
   185    185   
   186    186   finish_test

Changes to test/analyzeD.test.

    59     59   } {}
    60     60   
    61     61   # With full ANALYZE data, SQLite sees that c=150 (5 rows) is better than
    62     62   # a=3001 (7 rows).
    63     63   #
    64     64   do_eqp_test 1.2 {
    65     65     SELECT * FROM t1 WHERE a=3001 AND c=150;
    66         -} {
    67         -  0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
    68         -}
           66  +} {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
    69     67   
    70     68   do_test 1.3 {
    71     69     execsql { DELETE FROM sqlite_stat1 }
    72     70     db close
    73     71     sqlite3 db test.db
    74     72   } {}
    75     73   
................................................................................
    76     74   # Without stat1, because 3001 is larger than all samples in the stat4
    77     75   # table, SQLite thinks that a=3001 matches just 1 row. So it (incorrectly)
    78     76   # chooses it over the c=150 index (5 rows). Even with stat1 data, things
    79     77   # worked this way before commit [e6f7f97dbc].
    80     78   #
    81     79   do_eqp_test 1.4 {
    82     80     SELECT * FROM t1 WHERE a=3001 AND c=150;
    83         -} {
    84         -  0 0 0 {SEARCH TABLE t1 USING INDEX t1_ab (a=?)}
    85         -}
           81  +} {SEARCH TABLE t1 USING INDEX t1_ab (a=?)}
    86     82   
    87     83   do_test 1.5 {
    88     84     execsql { 
    89     85       UPDATE t1 SET a=13 WHERE a = 3001;
    90     86       ANALYZE;
    91     87     }
    92     88   } {}
    93     89   
    94     90   do_eqp_test 1.6 {
    95     91     SELECT * FROM t1 WHERE a=13 AND c=150;
    96         -} {
    97         -  0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
    98         -}
           92  +} {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
    99     93   
   100     94   do_test 1.7 {
   101     95     execsql { DELETE FROM sqlite_stat1 }
   102     96     db close
   103     97     sqlite3 db test.db
   104     98   } {}
   105     99   
   106    100   # Same test as 1.4, except this time the 7 rows that match the a=? condition 
   107    101   # do not feature larger values than all rows in the stat4 table. So SQLite
   108    102   # gets this right, even without stat1 data.
   109    103   do_eqp_test 1.8 {
   110    104     SELECT * FROM t1 WHERE a=13 AND c=150;
   111         -} {
   112         -  0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
   113         -}
          105  +} {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
   114    106   
   115    107   finish_test

Changes to test/analyzeF.test.

    58     58   
    59     59     9  "x = str('19') AND y = str('4')" {t1y (y=?)}
    60     60     10 "x = str('4') AND y = str('19')" {t1y (y=?)}
    61     61   
    62     62     11 "x = nullif('19', 0) AND y = nullif('4', 0)" {t1y (y=?)}
    63     63     12 "x = nullif('4', 0) AND y = nullif('19', 0)" {t1y (y=?)}
    64     64   } {
    65         -  set res "0 0 0 {SEARCH TABLE t1 USING INDEX $idx}"
           65  +  set res "SEARCH TABLE t1 USING INDEX $idx"
    66     66     do_eqp_test 1.$tn "SELECT * FROM t1 WHERE $where" $res
    67     67   }
    68     68   
    69     69   # Test that functions that do not exist - "func()" - do not cause an error.
    70     70   #
    71     71   do_catchsql_test 2.1 {
    72     72     SELECT * FROM t1 WHERE x = substr('145', 2, 1) AND y = func(1, 2, 3)
................................................................................
    88     88   foreach {tn where idx} {
    89     89     1 "x = det4() AND y = det19()"     {t1x (x=?)}
    90     90     2 "x = det19() AND y = det4()"     {t1y (y=?)}
    91     91   
    92     92     3 "x = nondet4() AND y = nondet19()"     {t1y (y=?)}
    93     93     4 "x = nondet19() AND y = nondet4()"     {t1y (y=?)}
    94     94   } {
    95         -  set res "0 0 0 {SEARCH TABLE t1 USING INDEX $idx}"
           95  +  set res "SEARCH TABLE t1 USING INDEX $idx"
    96     96     do_eqp_test 3.$tn "SELECT * FROM t1 WHERE $where" $res
    97     97   }
    98     98   
    99     99   
   100    100   execsql { DELETE FROM t1 }
   101    101   
   102    102   proc throw_error {err} { error $err }

Changes to test/autoinc.test.

   671    671     CREATE TABLE t10a(a INTEGER PRIMARY KEY AUTOINCREMENT, b UNIQUE);
   672    672     INSERT INTO t10a VALUES(888,9999);
   673    673     CREATE TABLE t10b(x INTEGER PRIMARY KEY AUTOINCREMENT, y UNIQUE);
   674    674     INSERT INTO t10b SELECT * FROM t10a;
   675    675     SELECT * FROM sqlite_sequence;
   676    676   } {t10a 888 t10b 888}
   677    677   
   678         -
          678  +# 2018-04-21 autoincrement does not cause problems for upsert
          679  +#
          680  +do_execsql_test autoinc-11.1 {
          681  +  CREATE TABLE t11(a INTEGER PRIMARY KEY AUTOINCREMENT,b UNIQUE);
          682  +  INSERT INTO t11(a,b) VALUES(2,3),(5,6),(4,3),(1,2)
          683  +    ON CONFLICT(b) DO UPDATE SET a=a+1000;
          684  +  SELECT seq FROM sqlite_sequence WHERE name='t11';
          685  +} {5}
   679    686   
   680    687   finish_test

Changes to test/autoindex1.test.

   173    173   #
   174    174   do_execsql_test autoindex1-500 {
   175    175     CREATE TABLE t501(a INTEGER PRIMARY KEY, b);
   176    176     CREATE TABLE t502(x INTEGER PRIMARY KEY, y);
   177    177     INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES('t501',null,'1000000');
   178    178     INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES('t502',null,'1000');
   179    179     ANALYZE sqlite_master;
   180         -  EXPLAIN QUERY PLAN
          180  +}
          181  +do_eqp_test autoindex1-500.1 {
   181    182     SELECT b FROM t501
   182    183      WHERE t501.a IN (SELECT x FROM t502 WHERE y=?);
   183    184   } {
   184         -  0 0 0 {SEARCH TABLE t501 USING INTEGER PRIMARY KEY (rowid=?)} 
   185         -  0 0 0 {EXECUTE LIST SUBQUERY 1} 
   186         -  1 0 0 {SCAN TABLE t502}
          185  +  QUERY PLAN
          186  +  |--SEARCH TABLE t501 USING INTEGER PRIMARY KEY (rowid=?)
          187  +  `--LIST SUBQUERY
          188  +     `--SCAN TABLE t502
   187    189   }
   188         -do_execsql_test autoindex1-501 {
   189         -  EXPLAIN QUERY PLAN
          190  +do_eqp_test autoindex1-501 {
   190    191     SELECT b FROM t501
   191    192      WHERE t501.a IN (SELECT x FROM t502 WHERE y=t501.b);
   192    193   } {
   193         -  0 0 0 {SCAN TABLE t501} 
   194         -  0 0 0 {EXECUTE CORRELATED LIST SUBQUERY 1} 
   195         -  1 0 0 {SEARCH TABLE t502 USING AUTOMATIC COVERING INDEX (y=?)}
          194  +  QUERY PLAN
          195  +  |--SCAN TABLE t501
          196  +  `--CORRELATED LIST SUBQUERY
          197  +     `--SEARCH TABLE t502 USING AUTOMATIC COVERING INDEX (y=?)
   196    198   }
   197         -do_execsql_test autoindex1-502 {
   198         -  EXPLAIN QUERY PLAN
          199  +do_eqp_test autoindex1-502 {
   199    200     SELECT b FROM t501
   200    201      WHERE t501.a=123
   201    202        AND t501.a IN (SELECT x FROM t502 WHERE y=t501.b);
   202    203   } {
   203         -  0 0 0 {SEARCH TABLE t501 USING INTEGER PRIMARY KEY (rowid=?)} 
   204         -  0 0 0 {EXECUTE CORRELATED LIST SUBQUERY 1} 
   205         -  1 0 0 {SCAN TABLE t502}
          204  +  QUERY PLAN
          205  +  |--SEARCH TABLE t501 USING INTEGER PRIMARY KEY (rowid=?)
          206  +  `--CORRELATED LIST SUBQUERY
          207  +     `--SCAN TABLE t502
   206    208   }
   207         -
   208    209   
   209    210   # The following code checks a performance regression reported on the
   210    211   # mailing list on 2010-10-19.  The problem is that the nRowEst field
   211    212   # of ephermeral tables was not being initialized correctly and so no
   212    213   # automatic index was being created for the emphemeral table when it was
   213    214   # used as part of a join.
   214    215   #
................................................................................
   253    254                 ON flock_owner (owner_change_date);
   254    255     CREATE INDEX fo_owner_person_id_index  
   255    256                 ON flock_owner (owner_person_id);
   256    257     CREATE INDEX sheep_org_flock_index  
   257    258              ON sheep (originating_flock);
   258    259     CREATE INDEX sheep_reg_flock_index  
   259    260              ON sheep (registering_flock);
   260         -  EXPLAIN QUERY PLAN
          261  +}
          262  +do_eqp_test autoindex1-600a {
   261    263     SELECT x.sheep_no, x.registering_flock, x.date_of_registration
   262    264      FROM sheep x LEFT JOIN
   263    265          (SELECT s.sheep_no, prev.flock_no, prev.owner_person_id,
   264    266          s.date_of_registration, prev.owner_change_date
   265    267          FROM sheep s JOIN flock_owner prev ON s.registering_flock =
   266    268      prev.flock_no
   267    269          AND (prev.owner_change_date <= s.date_of_registration || ' 00:00:00')
................................................................................
   270    272              WHERE prev.flock_no = later.flock_no
   271    273              AND later.owner_change_date > prev.owner_change_date
   272    274              AND later.owner_change_date <= s.date_of_registration||' 00:00:00')
   273    275          ) y ON x.sheep_no = y.sheep_no
   274    276      WHERE y.sheep_no IS NULL
   275    277      ORDER BY x.registering_flock;
   276    278   } {
   277         -  1 0 0 {SCAN TABLE sheep AS s} 
   278         -  1 1 1 {SEARCH TABLE flock_owner AS prev USING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date<?)} 
   279         -  1 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 2} 
   280         -  2 0 0 {SEARCH TABLE flock_owner AS later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date<?)} 
   281         -  0 0 0 {SCAN TABLE sheep AS x USING INDEX sheep_reg_flock_index} 
   282         -  0 1 1 {SEARCH SUBQUERY 1 AS y USING AUTOMATIC COVERING INDEX (sheep_no=?)}
          279  +  QUERY PLAN
          280  +  |--MATERIALIZE xxxxxx
          281  +  |  |--SCAN TABLE sheep AS s
          282  +  |  |--SEARCH TABLE flock_owner AS prev USING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date<?)
          283  +  |  `--CORRELATED SCALAR SUBQUERY
          284  +  |     `--SEARCH TABLE flock_owner AS later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date<?)
          285  +  |--SCAN TABLE sheep AS x USING INDEX sheep_reg_flock_index
          286  +  `--SEARCH SUBQUERY xxxxxx AS y USING AUTOMATIC COVERING INDEX (sheep_no=?)
   283    287   }
   284    288   
   285    289   
   286    290   do_execsql_test autoindex1-700 {
   287    291     CREATE TABLE t5(a, b, c);
   288         -  EXPLAIN QUERY PLAN SELECT a FROM t5 WHERE b=10 ORDER BY c;
          292  +}
          293  +do_eqp_test autoindex1-700a {
          294  +  SELECT a FROM t5 WHERE b=10 ORDER BY c;
   289    295   } {
   290         -  0 0 0 {SCAN TABLE t5} 
   291         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          296  +  QUERY PLAN
          297  +  |--SCAN TABLE t5
          298  +  `--USE TEMP B-TREE FOR ORDER BY
   292    299   }
   293    300   
   294    301   # The following checks a performance issue reported on the sqlite-dev
   295    302   # mailing list on 2013-01-10
   296    303   #
   297    304   do_execsql_test autoindex1-800 {
   298    305     CREATE TABLE accounts(

Changes to test/autoindex3.test.

    80     80   # on the basis that the real index "uab" must be better than the automatic
    81     81   # index. This is not right - a skip-scan is not necessarily better than an
    82     82   # automatic index scan.
    83     83   #
    84     84   do_eqp_test 220 {
    85     85     select count(*) from u, v where u.b = v.b and v.e > 34;
    86     86   } {
    87         -  0 0 1 {SEARCH TABLE v USING INDEX ve (e>?)} 
    88         -  0 1 0 {SEARCH TABLE u USING AUTOMATIC COVERING INDEX (b=?)}
           87  +  QUERY PLAN
           88  +  |--SEARCH TABLE v USING INDEX ve (e>?)
           89  +  `--SEARCH TABLE u USING AUTOMATIC COVERING INDEX (b=?)
    89     90   }
    90     91   
    91     92   
    92     93   finish_test

Changes to test/autoindex5.test.

    80     80                 AND debian_bugs.note = package_notes.id
    81     81                 ORDER BY debian_bugs.bug;
    82     82   } {}
    83     83   
    84     84   # The following query should use an automatic index for the view
    85     85   # in FROM clause of the subquery of the second result column.
    86     86   #
    87         -do_execsql_test autoindex5-1.1 {
    88         -  EXPLAIN QUERY PLAN
           87  +do_eqp_test autoindex5-1.1 {
    89     88     SELECT
    90     89       st.bug_name,
    91     90       (SELECT ALL debian_cve.bug FROM debian_cve
    92     91         WHERE debian_cve.bug_name = st.bug_name
    93     92         ORDER BY debian_cve.bug),
    94     93       sp.release
    95     94     FROM
................................................................................
    99     98     WHERE
   100     99        sp.rowid = st.package
   101    100        AND st.bug_name = bugs.name
   102    101        AND ( st.bug_name LIKE 'CVE-%' OR st.bug_name LIKE 'TEMP-%' )
   103    102        AND ( sp.release = 'sid' OR sp.release = 'stretch' OR sp.release = 'jessie'
   104    103               OR sp.release = 'wheezy' OR sp.release = 'squeeze' )
   105    104     ORDER BY sp.name, st.bug_name, sp.release, sp.subrelease;
   106         -} {/SEARCH SUBQUERY 2 USING AUTOMATIC COVERING INDEX .bug_name=/}
          105  +} {SEARCH SUBQUERY * USING AUTOMATIC COVERING INDEX (bug_name=?)}
   107    106   
   108    107   #-------------------------------------------------------------------------
   109    108   # Test that ticket [8a2adec1] has been fixed.
   110    109   #
   111    110   do_execsql_test 2.1 {
   112    111     CREATE TABLE one(o);
   113    112     INSERT INTO one DEFAULT VALUES;

Changes to test/bestindex1.test.

    47     47   
    48     48   do_execsql_test 1.0 {
    49     49     CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
    50     50   } {}
    51     51   
    52     52   do_eqp_test 1.1 {
    53     53     SELECT * FROM x1 WHERE a = 'abc'
    54         -} {
    55         -  0 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
    56         -}
           54  +} {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
    57     55   
    58     56   do_eqp_test 1.2 {
    59     57     SELECT * FROM x1 WHERE a IN ('abc', 'def');
    60         -} {
    61         -  0 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
    62         -  0 0 0 {EXECUTE LIST SUBQUERY 1}
    63         -}
           58  +} {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
    64     59   
    65     60   #-------------------------------------------------------------------------
    66     61   #
    67     62   reset_db
    68     63   register_tcl_module db
    69     64   
    70     65   # Parameter $mode may be one of:
................................................................................
   141    136     do_execsql_test 2.2.$mode.4 {SELECT rowid FROM t1 WHERE a='two'} {2} 
   142    137   
   143    138     do_execsql_test 2.2.$mode.5 {
   144    139       SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
   145    140     } {1 4} 
   146    141   
   147    142     set plan(use) {
   148         -    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
   149         -    0 0 0 {EXECUTE LIST SUBQUERY 1}
   150         -    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          143  +    QUERY PLAN
          144  +    |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'
          145  +    `--USE TEMP B-TREE FOR ORDER BY
   151    146     }
   152    147     set plan(omit) {
   153         -    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
   154         -    0 0 0 {EXECUTE LIST SUBQUERY 1}
   155         -    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          148  +    QUERY PLAN
          149  +    |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'
          150  +    `--USE TEMP B-TREE FOR ORDER BY
   156    151     }
   157    152     set plan(use2) {
   158         -    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x}
   159         -    0 0 0 {EXECUTE LIST SUBQUERY 1}
   160         -    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          153  +    QUERY PLAN
          154  +    |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x
          155  +    `--USE TEMP B-TREE FOR ORDER BY
   161    156     }
   162    157   
   163    158     do_eqp_test 2.2.$mode.6 { 
   164    159       SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
   165         -  } $plan($mode)
          160  +  } [string map {"\n  " "\n"} $plan($mode)]
   166    161   }
   167    162   
   168    163   # 2016-04-09.
   169    164   # Demonstrate a register overwrite problem when using two virtual
   170    165   # tables where the outer loop uses the IN operator.
   171    166   #
   172    167   set G(collist) [list PrimaryKey flagA columnA]

Changes to test/bestindex2.test.

    85     85     CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd t1 {a b}");
    86     86     CREATE VIRTUAL TABLE t2 USING tcl("vtab_cmd t2 {c d}");
    87     87     CREATE VIRTUAL TABLE t3 USING tcl("vtab_cmd t3 {e f}");
    88     88   }
    89     89   
    90     90   do_eqp_test 1.1 {
    91     91     SELECT * FROM t1 WHERE a='abc'
    92         -} {
    93         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=?)}
    94         -}
           92  +} {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=?)}
           93  +
    95     94   do_eqp_test 1.2 {
    96     95     SELECT * FROM t1 WHERE a='abc' AND b='def'
    97         -} {
    98         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=? AND b=?)}
    99         -}
           96  +} {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=? AND b=?)}
           97  +
   100     98   do_eqp_test 1.3 {
   101     99     SELECT * FROM t1 WHERE a='abc' AND a='def'
   102         -} {
   103         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=?)}
   104         -}
          100  +} {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=?)}
          101  +
   105    102   do_eqp_test 1.4 {
   106    103     SELECT * FROM t1,t2 WHERE c=a
   107    104   } {
   108         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:} 
   109         -  0 1 1 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)}
          105  +  QUERY PLAN
          106  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:
          107  +  `--SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)
   110    108   }
   111    109   
   112    110   do_eqp_test 1.5 {
   113    111     SELECT * FROM t1, t2 CROSS JOIN t3 WHERE t2.c = +t1.b AND t3.e=t2.d
   114    112   } {
   115         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:} 
   116         -  0 1 1 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)} 
   117         -  0 2 2 {SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)}
          113  +  QUERY PLAN
          114  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:
          115  +  |--SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)
          116  +  `--SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)
   118    117   }
   119    118   
   120    119   do_eqp_test 1.6 {
   121    120     SELECT * FROM t1, t2, t3 WHERE t2.c = +t1.b AND t3.e = t2.d
   122    121   } {
   123         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:} 
   124         -  0 1 1 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)} 
   125         -  0 2 2 {SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)}
          122  +  QUERY PLAN
          123  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:
          124  +  |--SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)
          125  +  `--SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)
   126    126   }
   127    127   
   128    128   do_execsql_test 1.7.1 {
   129    129     CREATE TABLE x1(a, b);
   130    130   }
   131    131   do_eqp_test 1.7.2 {
   132    132     SELECT * FROM x1 CROSS JOIN t1, t2, t3 
   133    133       WHERE t1.a = t2.c AND t1.b = t3.e
   134    134   } {
   135         -  0 0 0 {SCAN TABLE x1} 
   136         -  0 1 1 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:}
   137         -  0 2 2 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)} 
   138         -  0 3 3 {SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)}
          135  +  QUERY PLAN
          136  +  |--SCAN TABLE x1
          137  +  |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:
          138  +  |--SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)
          139  +  `--SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)
   139    140   }
   140    141   
   141    142   finish_test

Changes to test/bestindex3.test.

    75     75   
    76     76   do_execsql_test 1.0 {
    77     77     CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd 0");
    78     78   }
    79     79   
    80     80   do_eqp_test 1.1 {
    81     81     SELECT * FROM t1 WHERE a LIKE 'abc';
    82         -} {
    83         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
    84         -}
           82  +} {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
    85     83   
    86     84   do_eqp_test 1.2 {
    87     85     SELECT * FROM t1 WHERE a = 'abc';
    88         -} {
    89         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
    90         -}
           86  +} {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
    91     87   
    92     88   do_eqp_test 1.3 {
    93     89     SELECT * FROM t1 WHERE a = 'abc' OR b = 'def';
    94     90   } {
    95         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
    96         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
           91  +  QUERY PLAN
           92  +  `--MULTI-INDEX OR
           93  +     |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?
           94  +     `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?
    97     95   }
    98     96   
    99     97   do_eqp_test 1.4 {
   100     98     SELECT * FROM t1 WHERE a LIKE 'abc%' OR b = 'def';
   101     99   } {
   102         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
   103         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
          100  +  QUERY PLAN
          101  +  `--MULTI-INDEX OR
          102  +     |--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?
          103  +     `--SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?
   104    104   }
   105    105   
   106    106   do_execsql_test 1.5 {
   107    107     CREATE TABLE ttt(a, b, c);
   108    108   
   109    109     INSERT INTO ttt VALUES(1, 'two',   'three');
   110    110     INSERT INTO ttt VALUES(2, 'one',   'two');
................................................................................
   143    143       CREATE TABLE t2(x TEXT COLLATE nocase, y TEXT);
   144    144       CREATE INDEX t2x ON t2(x COLLATE nocase);
   145    145       CREATE INDEX t2y ON t2(y);
   146    146     }
   147    147   
   148    148     do_eqp_test 2.2 {
   149    149       SELECT * FROM t2 WHERE x LIKE 'abc%' OR y = 'def'
   150         -  } {
   151         -    0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x>? AND x<?)}
   152         -    0 0 0 {SEARCH TABLE t2 USING INDEX t2y (y=?)}
   153         -  }
          150  +  } [string map {"\n  " \n} {
          151  +    QUERY PLAN
          152  +    `--MULTI-INDEX OR
          153  +       |--SEARCH TABLE t2 USING INDEX t2x (x>? AND x<?)
          154  +       `--SEARCH TABLE t2 USING INDEX t2y (y=?)
          155  +  }]
   154    156   }
   155    157   
   156    158   #-------------------------------------------------------------------------
   157    159   # Test that any PRIMARY KEY within a sqlite3_decl_vtab() CREATE TABLE 
   158    160   # statement is currently ignored.
   159    161   #
   160    162   proc vvv_command {method args} {

Changes to test/bigmmap.test.

    88     88         ORDER BY b, c;
    89     89       " {}
    90     90       
    91     91       do_eqp_test 2.$i.$t.3 "
    92     92         SELECT * FROM t$t AS o WHERE 
    93     93           NOT EXISTS( SELECT * FROM t$t AS i WHERE a=o.a AND +b=o.b AND +c=o.c )
    94     94         ORDER BY b, c;
    95         -    " "
    96         -      0 0 0 {SCAN TABLE t$t AS o USING COVERING INDEX sqlite_autoindex_t${t}_1}
    97         -      0 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 1}
    98         -      1 0 0 {SEARCH TABLE t$t AS i USING INTEGER PRIMARY KEY (rowid=?)}
    99         -    "
           95  +    " [string map {"\n    " "\n"} "
           96  +      QUERY PLAN
           97  +      |--SCAN TABLE t$t AS o USING COVERING INDEX sqlite_autoindex_t${t}_1
           98  +      `--CORRELATED SCALAR SUBQUERY
           99  +         `--SEARCH TABLE t$t AS i USING INTEGER PRIMARY KEY (rowid=?)
          100  +    "]
   100    101     }
   101    102   }
   102    103   
   103    104   finish_test
   104         -

Changes to test/closure01.test.

   268    268      WHERE root=1
   269    269        AND depth=3
   270    270        AND tablename='t1'
   271    271        AND idcolumn='x'
   272    272        AND parentcolumn='y'
   273    273     ORDER BY id;
   274    274   } {8 9 10 11 12 13 14 15}
          275  +
          276  +#-------------------------------------------------------------------------
          277  +# At one point the following join query was causing a malfunction in
          278  +# xBestIndex.
          279  +#
          280  +do_execsql_test 6.0 {
          281  +  CREATE TABLE t4 (
          282  +    id INTEGER PRIMARY KEY, 
          283  +    name TEXT NOT NULL,
          284  +    parent_id INTEGER
          285  +  );
          286  +  CREATE VIRTUAL TABLE vt4 USING transitive_closure (
          287  +    idcolumn=id, parentcolumn=parent_id, tablename=t4
          288  +  );
          289  +}
          290  +
          291  +do_execsql_test 6.1 {
          292  +  SELECT * FROM t4, vt4 WHERE t4.id = vt4.root AND vt4.id=4 AND vt4.depth=2;
          293  +}
   275    294   
   276    295   finish_test

Changes to test/cost.test.

    20     20     CREATE TABLE t4(c, d, e);
    21     21     CREATE UNIQUE INDEX i3 ON t3(b);
    22     22     CREATE UNIQUE INDEX i4 ON t4(c, d);
    23     23   }
    24     24   do_eqp_test 1.2 {
    25     25     SELECT e FROM t3, t4 WHERE b=c ORDER BY b, d;
    26     26   } {
    27         -  0 0 0 {SCAN TABLE t3 USING COVERING INDEX i3} 
    28         -  0 1 1 {SEARCH TABLE t4 USING INDEX i4 (c=?)}
           27  +  QUERY PLAN
           28  +  |--SCAN TABLE t3 USING COVERING INDEX i3
           29  +  `--SEARCH TABLE t4 USING INDEX i4 (c=?)
    29     30   }
    30     31   
    31     32   
    32     33   do_execsql_test 2.1 {
    33     34     CREATE TABLE t1(a, b);
    34     35     CREATE INDEX i1 ON t1(a);
    35     36   }
    36     37   
    37     38   # It is better to use an index for ORDER BY than sort externally, even 
    38     39   # if the index is a non-covering index.
    39     40   do_eqp_test 2.2 {
    40     41     SELECT * FROM t1 ORDER BY a;
    41         -} {
    42         -  0 0 0 {SCAN TABLE t1 USING INDEX i1}
    43         -}
           42  +} {SCAN TABLE t1 USING INDEX i1}
    44     43   
    45     44   do_execsql_test 3.1 {
    46     45     CREATE TABLE t5(a INTEGER PRIMARY KEY,b,c,d,e,f,g);
    47     46     CREATE INDEX t5b ON t5(b);
    48     47     CREATE INDEX t5c ON t5(c);
    49     48     CREATE INDEX t5d ON t5(d);
    50     49     CREATE INDEX t5e ON t5(e);
................................................................................
    53     52   }
    54     53   
    55     54   do_eqp_test 3.2 {
    56     55     SELECT a FROM t5 
    57     56     WHERE b IS NULL OR c IS NULL OR d IS NULL 
    58     57     ORDER BY a;
    59     58   } {
    60         -  0 0 0 {SEARCH TABLE t5 USING INDEX t5b (b=?)} 
    61         -  0 0 0 {SEARCH TABLE t5 USING INDEX t5c (c=?)} 
    62         -  0 0 0 {SEARCH TABLE t5 USING INDEX t5d (d=?)} 
    63         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
           59  +  QUERY PLAN
           60  +  |--MULTI-INDEX OR
           61  +  |  |--SEARCH TABLE t5 USING INDEX t5b (b=?)
           62  +  |  |--SEARCH TABLE t5 USING INDEX t5c (c=?)
           63  +  |  `--SEARCH TABLE t5 USING INDEX t5d (d=?)
           64  +  `--USE TEMP B-TREE FOR ORDER BY
    64     65   }
    65     66   
    66     67   #-------------------------------------------------------------------------
    67     68   # If there is no likelihood() or stat3 data, SQLite assumes that a closed
    68     69   # range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint)
    69     70   # visits 1/64 of the rows in a table.
    70     71   #
................................................................................
    75     76   do_execsql_test 4.1 {
    76     77     CREATE TABLE t1(a, b);
    77     78     CREATE INDEX i1 ON t1(a);
    78     79     CREATE INDEX i2 ON t1(b);
    79     80   }
    80     81   do_eqp_test 4.2 {
    81     82     SELECT * FROM t1 WHERE likelihood(a=?, 0.014) AND b BETWEEN ? AND ?;
    82         -} {
    83         -  0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}
    84         -}
           83  +} {SEARCH TABLE t1 USING INDEX i1 (a=?)}
           84  +
    85     85   do_eqp_test 4.3 {
    86     86     SELECT * FROM t1 WHERE likelihood(a=?, 0.016) AND b BETWEEN ? AND ?;
    87         -} {
    88         -  0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b<?)}
    89         -}
           87  +} {SEARCH TABLE t1 USING INDEX i2 (b>? AND b<?)}
    90     88   
    91     89   
    92     90   #-------------------------------------------------------------------------
    93     91   #
    94     92   reset_db
    95     93   do_execsql_test 5.1 {
    96     94     CREATE TABLE t2(x, y);
    97     95     CREATE INDEX t2i1 ON t2(x);
    98     96   }
    99     97   
   100     98   do_eqp_test 5.2 {
   101     99     SELECT * FROM t2 ORDER BY x, y;
   102    100   } {
   103         -  0 0 0 {SCAN TABLE t2 USING INDEX t2i1} 
   104         -  0 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY}
          101  +  QUERY PLAN
          102  +  |--SCAN TABLE t2 USING INDEX t2i1
          103  +  `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
   105    104   }
   106    105   
   107    106   do_eqp_test 5.3 {
   108    107     SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid;
   109    108   } {
   110         -  0 0 0 {SEARCH TABLE t2 USING INDEX t2i1 (x>? AND x<?)} 
   111         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          109  +  QUERY PLAN
          110  +  |--SEARCH TABLE t2 USING INDEX t2i1 (x>? AND x<?)
          111  +  `--USE TEMP B-TREE FOR ORDER BY
   112    112   }
   113    113   
   114    114   # where7.test, where8.test:
   115    115   #
   116    116   do_execsql_test 6.1 {
   117    117     CREATE TABLE t3(a INTEGER PRIMARY KEY, b, c);
   118    118     CREATE INDEX t3i1 ON t3(b);
   119    119     CREATE INDEX t3i2 ON t3(c);
   120    120   }
   121    121   
   122    122   do_eqp_test 6.2 {
   123    123     SELECT a FROM t3 WHERE (b BETWEEN 2 AND 4) OR c=100 ORDER BY a
   124    124   } {
   125         -  0 0 0 {SEARCH TABLE t3 USING INDEX t3i1 (b>? AND b<?)} 
   126         -  0 0 0 {SEARCH TABLE t3 USING INDEX t3i2 (c=?)}
   127         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          125  +  QUERY PLAN
          126  +  |--MULTI-INDEX OR
          127  +  |  |--SEARCH TABLE t3 USING INDEX t3i1 (b>? AND b<?)
          128  +  |  `--SEARCH TABLE t3 USING INDEX t3i2 (c=?)
          129  +  `--USE TEMP B-TREE FOR ORDER BY
   128    130   }
   129    131   
   130    132   #-------------------------------------------------------------------------
   131    133   #
   132    134   reset_db
   133    135   do_execsql_test 7.1 {
   134    136     CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d,e,f,g);
................................................................................
   141    143   }
   142    144   
   143    145   do_eqp_test 7.2 {
   144    146     SELECT a FROM t1
   145    147        WHERE (b>=950 AND b<=1010) OR (b IS NULL AND c NOT NULL)
   146    148     ORDER BY a
   147    149   } {
   148         -  0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)} 
   149         -  0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)} 
   150         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          150  +  QUERY PLAN
          151  +  |--MULTI-INDEX OR
          152  +  |  |--SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?)
          153  +  |  `--SEARCH TABLE t1 USING INDEX t1b (b=?)
          154  +  `--USE TEMP B-TREE FOR ORDER BY
   151    155   }
   152    156   
   153    157   do_eqp_test 7.3 {
   154    158     SELECT rowid FROM t1
   155    159     WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL)
   156    160           OR (b NOT NULL AND c IS NULL AND d NOT NULL)
   157    161           OR (b NOT NULL AND c NOT NULL AND d IS NULL)
   158         -} {
   159         -  0 0 0 {SCAN TABLE t1}
   160         -}
          162  +} {SCAN TABLE t1}
   161    163   
   162    164   do_eqp_test 7.4 {
   163    165     SELECT rowid FROM t1 WHERE (+b IS NULL AND c NOT NULL) OR c IS NULL
   164         -} {
   165         -  0 0 0 {SCAN TABLE t1}
   166         -}
          166  +} {SCAN TABLE t1}
   167    167   
   168    168   #-------------------------------------------------------------------------
   169    169   #
   170    170   reset_db
   171    171   do_execsql_test 8.1 {
   172    172     CREATE TABLE composer(
   173    173       cid INTEGER PRIMARY KEY,
................................................................................
   190    190   do_eqp_test 8.2 {
   191    191     SELECT DISTINCT aname
   192    192       FROM album, composer, track
   193    193      WHERE cname LIKE '%bach%'
   194    194        AND unlikely(composer.cid=track.cid)
   195    195        AND unlikely(album.aid=track.aid);
   196    196   } {
   197         -  0 0 2 {SCAN TABLE track} 
   198         -  0 1 0 {SEARCH TABLE album USING INTEGER PRIMARY KEY (rowid=?)}
   199         -  0 2 1 {SEARCH TABLE composer USING INTEGER PRIMARY KEY (rowid=?)}
   200         -  0 0 0 {USE TEMP B-TREE FOR DISTINCT}
          197  +  QUERY PLAN
          198  +  |--SCAN TABLE track
          199  +  |--SEARCH TABLE album USING INTEGER PRIMARY KEY (rowid=?)
          200  +  |--SEARCH TABLE composer USING INTEGER PRIMARY KEY (rowid=?)
          201  +  `--USE TEMP B-TREE FOR DISTINCT
   201    202   }
   202    203   
   203    204   #-------------------------------------------------------------------------
   204    205   #
   205    206   do_execsql_test 9.1 {
   206    207     CREATE TABLE t1(
   207    208       a,b,c,d,e, f,g,h,i,j,
................................................................................
   259    260         execsql { INSERT INTO t6 VALUES($i%4, 'xyz', $i%8) }
   260    261       }
   261    262       execsql ANALYZE
   262    263     } {}
   263    264   
   264    265     do_eqp_test 10.3 {
   265    266       SELECT rowid FROM t6 WHERE a=0 AND c=0
   266         -  } {
   267         -    0 0 0 {SEARCH TABLE t6 USING INDEX t6i2 (c=?)}
   268         -  }
          267  +  } {SEARCH TABLE t6 USING INDEX t6i2 (c=?)}
   269    268   
   270    269     do_eqp_test 10.4 {
   271    270       SELECT rowid FROM t6 WHERE a=0 AND b='xyz' AND c=0
   272         -  } {
   273         -    0 0 0 {SEARCH TABLE t6 USING INDEX t6i2 (c=?)}
   274         -  }
          271  +  } {SEARCH TABLE t6 USING INDEX t6i2 (c=?)}
   275    272   
   276    273     do_eqp_test 10.5 {
   277    274       SELECT rowid FROM t6 WHERE likelihood(a=0, 0.1) AND c=0
   278         -  } {
   279         -    0 0 0 {SEARCH TABLE t6 USING INDEX t6i1 (a=?)}
   280         -  }
          275  +  } {SEARCH TABLE t6 USING INDEX t6i1 (a=?)}
   281    276   
   282    277     do_eqp_test 10.6 {
   283    278       SELECT rowid FROM t6 WHERE likelihood(a=0, 0.1) AND b='xyz' AND c=0
   284         -  } {
   285         -    0 0 0 {SEARCH TABLE t6 USING INDEX t6i1 (a=? AND b=?)}
   286         -  }
          279  +  } {SEARCH TABLE t6 USING INDEX t6i1 (a=? AND b=?)}
   287    280   }
   288    281   
   289    282   finish_test

Changes to test/coveridxscan.test.

   105    105   
   106    106     CREATE TABLE t2(i INTEGER PRIMARY KEY, $cols);
   107    107     CREATE INDEX i2 ON t2($cols);
   108    108   "
   109    109   
   110    110   do_eqp_test 5.1.1 {
   111    111     SELECT * FROM t1 ORDER BY c1, c2;
   112         -} {
   113         -  0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1}
   114         -}
          112  +} {SCAN TABLE t1 USING COVERING INDEX i1}
   115    113   
   116    114   do_eqp_test 5.1.2 {
   117    115     SELECT * FROM t2 ORDER BY c1, c2;
   118         -} {
   119         -  0 0 0 {SCAN TABLE t2 USING COVERING INDEX i2}
   120         -}
   121         -
          116  +} {SCAN TABLE t2 USING COVERING INDEX i2}
   122    117   
   123    118   
   124    119   finish_test

Changes to test/csv01.test.

   136    136         columns=4,
   137    137         schema=
   138    138         'CREATE TABLE t3(a,b,c,d) WITHOUT ROWID',
   139    139         testflags=1
   140    140         );
   141    141   } {1 {vtable constructor failed: t5}}
   142    142   
          143  +# 2018-04-24
          144  +# Memory leak reported on the sqlite-users mailing list by Ralf Junker.
          145  +#
          146  +do_catchsql_test 4.3 {
          147  +  CREATE VIRTUAL TABLE IF NOT EXISTS temp.t1
          148  +  USING csv(filename='FileDoesNotExist.csv');
          149  +} {1 {cannot open 'FileDoesNotExist.csv' for reading}}
   143    150   
   144    151   finish_test

Changes to test/delete.test.

   398    398   do_execsql_test delete-10.1 {
   399    399     DELETE FROM t1 WHERE a='1' AND b='2';
   400    400   }
   401    401   
   402    402   do_execsql_test delete-10.2 {
   403    403     SELECT * FROM t1 WHERE a='1' AND b='2';
   404    404   }
          405  +
          406  +do_execsql_test delete-11.0 {
          407  +  CREATE TABLE t11(a INTEGER PRIMARY KEY, b INT);
          408  +  WITH RECURSIVE cnt(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM cnt WHERE x<20)
          409  +    INSERT INTO t11(a,b) SELECT x, (x*17)%100 FROM cnt;
          410  +  SELECT * FROM t11;
          411  +} {1 17 2 34 3 51 4 68 5 85 6 2 7 19 8 36 9 53 10 70 11 87 12 4 13 21 14 38 15 55 16 72 17 89 18 6 19 23 20 40}
          412  +do_execsql_test delete-11.1 {
          413  +  DELETE FROM t11 AS xyz
          414  +   WHERE EXISTS(SELECT 1 FROM t11 WHERE t11.a>xyz.a AND t11.b<=xyz.b);
          415  +  SELECT * FROM t11;
          416  +} {6 2 12 4 18 6 19 23 20 40}
   405    417   
   406    418   
   407    419   finish_test

Changes to test/e_createtable.test.

   652    652     1    "CREATE TABLE x1 AS SELECT * FROM t1"                     {a b c}
   653    653     2    "CREATE TABLE x1 AS SELECT c, b, a FROM t1"               {c b a}
   654    654     3    "CREATE TABLE x1 AS SELECT * FROM t1, t2"                 {a b c d e f}
   655    655     4    "CREATE TABLE x1 AS SELECT count(*) FROM t1"              {count(*)}
   656    656     5    "CREATE TABLE x1 AS SELECT count(a) AS a, max(b) FROM t1" {a max(b)}
   657    657   }
   658    658   
   659         -# EVIDENCE-OF: R-37111-22855 The declared type of each column is
          659  +# EVIDENCE-OF: R-55407-45319 The declared type of each column is
   660    660   # determined by the expression affinity of the corresponding expression
   661    661   # in the result set of the SELECT statement, as follows: Expression
   662    662   # Affinity Column Declared Type TEXT "TEXT" NUMERIC "NUM" INTEGER "INT"
   663         -# REAL "REAL" NONE "" (empty string)
          663  +# REAL "REAL" BLOB (a.k.a "NONE") "" (empty string)
   664    664   #
   665    665   do_createtable_tests 2.2 -tclquery {
   666    666     table_column_decltypes x1
   667    667   } -repair {
   668    668     catchsql { DROP TABLE x1 }
   669    669   } {
   670    670     1    "CREATE TABLE x1 AS SELECT a FROM t1"     {""}
................................................................................
  1381   1381   #
  1382   1382   do_execsql_test 4.10.0 {
  1383   1383     CREATE TABLE t1(a, b PRIMARY KEY);
  1384   1384     CREATE TABLE t2(a, b, c, UNIQUE(b, c));
  1385   1385   }
  1386   1386   do_createtable_tests 4.10 {
  1387   1387     1    "EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b = 5" 
  1388         -       {0 0 0 {SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (b=?)}}
         1388  +       {/*SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (b=?)*/}
  1389   1389   
  1390   1390     2    "EXPLAIN QUERY PLAN SELECT * FROM t2 ORDER BY b, c"
  1391         -       {0 0 0 {SCAN TABLE t2 USING INDEX sqlite_autoindex_t2_1}}
         1391  +       {/*SCAN TABLE t2 USING INDEX sqlite_autoindex_t2_1*/}
  1392   1392   
  1393   1393     3    "EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE b=10 AND c>10"
  1394         -       {0 0 0 {SEARCH TABLE t2 USING INDEX sqlite_autoindex_t2_1 (b=? AND c>?)}}
         1394  +       {/*SEARCH TABLE t2 USING INDEX sqlite_autoindex_t2_1 (b=? AND c>?)*/}
  1395   1395   }
  1396   1396   
  1397   1397   # EVIDENCE-OF: R-45493-35653 A CHECK constraint may be attached to a
  1398   1398   # column definition or specified as a table constraint. In practice it
  1399   1399   # makes no difference.
  1400   1400   #
  1401   1401   #   All the tests that deal with CHECK constraints below (4.11.* and 

Changes to test/eqp.test.

    39     39     CREATE TABLE t2(a INT, b INT, ex TEXT);
    40     40     CREATE TABLE t3(a INT, b INT, ex TEXT);
    41     41   }
    42     42   
    43     43   do_eqp_test 1.2 {
    44     44     SELECT * FROM t2, t1 WHERE t1.a=1 OR t1.b=2;
    45     45   } {
    46         -  0 0 1 {SEARCH TABLE t1 USING INDEX i1 (a=?)} 
    47         -  0 0 1 {SEARCH TABLE t1 USING INDEX i2 (b=?)} 
    48         -  0 1 0 {SCAN TABLE t2}
           46  +  QUERY PLAN
           47  +  |--MULTI-INDEX OR
           48  +  |  |--SEARCH TABLE t1 USING INDEX i1 (a=?)
           49  +  |  `--SEARCH TABLE t1 USING INDEX i2 (b=?)
           50  +  `--SCAN TABLE t2
    49     51   }
    50     52   do_eqp_test 1.3 {
    51     53     SELECT * FROM t2 CROSS JOIN t1 WHERE t1.a=1 OR t1.b=2;
    52     54   } {
    53         -  0 0 0 {SCAN TABLE t2}
    54         -  0 1 1 {SEARCH TABLE t1 USING INDEX i1 (a=?)} 
    55         -  0 1 1 {SEARCH TABLE t1 USING INDEX i2 (b=?)} 
           55  +  QUERY PLAN
           56  +  |--SCAN TABLE t2
           57  +  `--MULTI-INDEX OR
           58  +     |--SEARCH TABLE t1 USING INDEX i1 (a=?)
           59  +     `--SEARCH TABLE t1 USING INDEX i2 (b=?)
    56     60   }
    57     61   do_eqp_test 1.3 {
    58     62     SELECT a FROM t1 ORDER BY a
    59     63   } {
    60         -  0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1}
           64  +  QUERY PLAN
           65  +  `--SCAN TABLE t1 USING COVERING INDEX i1
    61     66   }
    62     67   do_eqp_test 1.4 {
    63     68     SELECT a FROM t1 ORDER BY +a
    64     69   } {
    65         -  0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1}
    66         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
           70  +  QUERY PLAN
           71  +  |--SCAN TABLE t1 USING COVERING INDEX i1
           72  +  `--USE TEMP B-TREE FOR ORDER BY
    67     73   }
    68     74   do_eqp_test 1.5 {
    69     75     SELECT a FROM t1 WHERE a=4
    70     76   } {
    71         -  0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}
           77  +  QUERY PLAN
           78  +  `--SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)
    72     79   }
    73     80   do_eqp_test 1.6 {
    74     81     SELECT DISTINCT count(*) FROM t3 GROUP BY a;
    75     82   } {
    76         -  0 0 0 {SCAN TABLE t3}
    77         -  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
    78         -  0 0 0 {USE TEMP B-TREE FOR DISTINCT}
           83  +  QUERY PLAN
           84  +  |--SCAN TABLE t3
           85  +  |--USE TEMP B-TREE FOR GROUP BY
           86  +  `--USE TEMP B-TREE FOR DISTINCT
    79     87   }
    80     88   
    81     89   do_eqp_test 1.7 {
    82     90     SELECT * FROM t3 JOIN (SELECT 1)
    83     91   } {
    84         -  0 0 1 {SCAN SUBQUERY 1}
    85         -  0 1 0 {SCAN TABLE t3}
           92  +  QUERY PLAN
           93  +  |--MATERIALIZE xxxxxx
           94  +  |  `--SCAN CONSTANT ROW
           95  +  |--SCAN SUBQUERY xxxxxx
           96  +  `--SCAN TABLE t3
    86     97   }
    87     98   do_eqp_test 1.8 {
    88     99     SELECT * FROM t3 JOIN (SELECT 1 UNION SELECT 2)
    89    100   } {
    90         -  1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (UNION)}
    91         -  0 0 1 {SCAN SUBQUERY 1}
    92         -  0 1 0 {SCAN TABLE t3}
          101  +  QUERY PLAN
          102  +  |--MATERIALIZE xxxxxx
          103  +  |  `--COMPOUND QUERY
          104  +  |     |--LEFT-MOST SUBQUERY
          105  +  |     |  `--SCAN CONSTANT ROW
          106  +  |     `--UNION USING TEMP B-TREE
          107  +  |        `--SCAN CONSTANT ROW
          108  +  |--SCAN SUBQUERY xxxxxx
          109  +  `--SCAN TABLE t3
    93    110   }
    94    111   do_eqp_test 1.9 {
    95    112     SELECT * FROM t3 JOIN (SELECT 1 EXCEPT SELECT a FROM t3 LIMIT 17)
    96    113   } {
    97         -  3 0 0 {SCAN TABLE t3}
    98         -  1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (EXCEPT)}
    99         -  0 0 1 {SCAN SUBQUERY 1}
   100         -  0 1 0 {SCAN TABLE t3}
          114  +  QUERY PLAN
          115  +  |--MATERIALIZE xxxxxx
          116  +  |  `--COMPOUND QUERY
          117  +  |     |--LEFT-MOST SUBQUERY
          118  +  |     |  `--SCAN CONSTANT ROW
          119  +  |     `--EXCEPT USING TEMP B-TREE
          120  +  |        `--SCAN TABLE t3
          121  +  |--SCAN SUBQUERY xxxxxx
          122  +  `--SCAN TABLE t3
   101    123   }
   102    124   do_eqp_test 1.10 {
   103    125     SELECT * FROM t3 JOIN (SELECT 1 INTERSECT SELECT a FROM t3 LIMIT 17)
   104    126   } {
   105         -  3 0 0 {SCAN TABLE t3}
   106         -  1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (INTERSECT)}
   107         -  0 0 1 {SCAN SUBQUERY 1}
   108         -  0 1 0 {SCAN TABLE t3}
          127  +  QUERY PLAN
          128  +  |--MATERIALIZE xxxxxx
          129  +  |  `--COMPOUND QUERY
          130  +  |     |--LEFT-MOST SUBQUERY
          131  +  |     |  `--SCAN CONSTANT ROW
          132  +  |     `--INTERSECT USING TEMP B-TREE
          133  +  |        `--SCAN TABLE t3
          134  +  |--SCAN SUBQUERY xxxxxx
          135  +  `--SCAN TABLE t3
   109    136   }
   110    137   
   111    138   do_eqp_test 1.11 {
   112    139     SELECT * FROM t3 JOIN (SELECT 1 UNION ALL SELECT a FROM t3 LIMIT 17)
   113    140   } {
   114         -  3 0 0 {SCAN TABLE t3}
   115         -  1 0 0 {COMPOUND SUBQUERIES 2 AND 3 (UNION ALL)}
   116         -  0 0 1 {SCAN SUBQUERY 1}
   117         -  0 1 0 {SCAN TABLE t3}
          141  +  QUERY PLAN
          142  +  |--MATERIALIZE xxxxxx
          143  +  |  `--COMPOUND QUERY
          144  +  |     |--LEFT-MOST SUBQUERY
          145  +  |     |  `--SCAN CONSTANT ROW
          146  +  |     `--UNION ALL
          147  +  |        `--SCAN TABLE t3
          148  +  |--SCAN SUBQUERY xxxxxx
          149  +  `--SCAN TABLE t3
   118    150   }
   119    151   
   120    152   #-------------------------------------------------------------------------
   121    153   # Test cases eqp-2.* - tests for single select statements.
   122    154   #
   123    155   drop_all_tables
   124    156   do_execsql_test 2.1 {
................................................................................
   125    157     CREATE TABLE t1(x INT, y INT, ex TEXT);
   126    158   
   127    159     CREATE TABLE t2(x INT, y INT, ex TEXT);
   128    160     CREATE INDEX t2i1 ON t2(x);
   129    161   }
   130    162   
   131    163   det 2.2.1 "SELECT DISTINCT min(x), max(x) FROM t1 GROUP BY x ORDER BY 1" {
   132         -  0 0 0 {SCAN TABLE t1}
   133         -  0 0 0 {USE TEMP B-TREE FOR GROUP BY}
   134         -  0 0 0 {USE TEMP B-TREE FOR DISTINCT}
   135         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          164  +  QUERY PLAN
          165  +  |--SCAN TABLE t1
          166  +  |--USE TEMP B-TREE FOR GROUP BY
          167  +  |--USE TEMP B-TREE FOR DISTINCT
          168  +  `--USE TEMP B-TREE FOR ORDER BY
   136    169   }
   137    170   det 2.2.2 "SELECT DISTINCT min(x), max(x) FROM t2 GROUP BY x ORDER BY 1" {
   138         -  0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1}
   139         -  0 0 0 {USE TEMP B-TREE FOR DISTINCT}
   140         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          171  +  QUERY PLAN
          172  +  |--SCAN TABLE t2 USING COVERING INDEX t2i1
          173  +  |--USE TEMP B-TREE FOR DISTINCT
          174  +  `--USE TEMP B-TREE FOR ORDER BY
   141    175   }
   142    176   det 2.2.3 "SELECT DISTINCT * FROM t1" {
   143         -  0 0 0 {SCAN TABLE t1}
   144         -  0 0 0 {USE TEMP B-TREE FOR DISTINCT}
          177  +  QUERY PLAN
          178  +  |--SCAN TABLE t1
          179  +  `--USE TEMP B-TREE FOR DISTINCT
   145    180   }
   146    181   det 2.2.4 "SELECT DISTINCT * FROM t1, t2" {
   147         -  0 0 0 {SCAN TABLE t1}
   148         -  0 1 1 {SCAN TABLE t2}
   149         -  0 0 0 {USE TEMP B-TREE FOR DISTINCT}
          182  +  QUERY PLAN
          183  +  |--SCAN TABLE t1
          184  +  |--SCAN TABLE t2
          185  +  `--USE TEMP B-TREE FOR DISTINCT
   150    186   }
   151    187   det 2.2.5 "SELECT DISTINCT * FROM t1, t2 ORDER BY t1.x" {
   152         -  0 0 0 {SCAN TABLE t1}
   153         -  0 1 1 {SCAN TABLE t2}
   154         -  0 0 0 {USE TEMP B-TREE FOR DISTINCT}
   155         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          188  +  QUERY PLAN
          189  +  |--SCAN TABLE t1
          190  +  |--SCAN TABLE t2
          191  +  |--USE TEMP B-TREE FOR DISTINCT
          192  +  `--USE TEMP B-TREE FOR ORDER BY
   156    193   }
   157    194   det 2.2.6 "SELECT DISTINCT t2.x FROM t1, t2 ORDER BY t2.x" {
   158         -  0 0 1 {SCAN TABLE t2 USING COVERING INDEX t2i1}
   159         -  0 1 0 {SCAN TABLE t1}
          195  +  QUERY PLAN
          196  +  |--SCAN TABLE t2 USING COVERING INDEX t2i1
          197  +  `--SCAN TABLE t1
   160    198   }
   161    199   
   162    200   det 2.3.1 "SELECT max(x) FROM t2" {
   163         -  0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2i1}
          201  +  QUERY PLAN
          202  +  `--SEARCH TABLE t2 USING COVERING INDEX t2i1
   164    203   }
   165    204   det 2.3.2 "SELECT min(x) FROM t2" {
   166         -  0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2i1}
          205  +  QUERY PLAN
          206  +  `--SEARCH TABLE t2 USING COVERING INDEX t2i1
   167    207   }
   168    208   det 2.3.3 "SELECT min(x), max(x) FROM t2" {
   169         -  0 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1}
          209  +  QUERY PLAN
          210  +  `--SCAN TABLE t2 USING COVERING INDEX t2i1
   170    211   }
   171    212   
   172    213   det 2.4.1 "SELECT * FROM t1 WHERE rowid=?" {
   173         -  0 0 0 {SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)}
          214  +  QUERY PLAN
          215  +  `--SEARCH TABLE t1 USING INTEGER PRIMARY KEY (rowid=?)
   174    216   }
   175    217   
   176    218   
   177    219   
   178    220   #-------------------------------------------------------------------------
   179    221   # Test cases eqp-3.* - tests for select statements that use sub-selects.
   180    222   #
   181    223   do_eqp_test 3.1.1 {
   182    224     SELECT (SELECT x FROM t1 AS sub) FROM t1;
   183    225   } {
   184         -  0 0 0 {SCAN TABLE t1}
   185         -  0 0 0 {EXECUTE SCALAR SUBQUERY 1}
   186         -  1 0 0 {SCAN TABLE t1 AS sub}
          226  +  QUERY PLAN
          227  +  |--SCAN TABLE t1
          228  +  `--SCALAR SUBQUERY
          229  +     `--SCAN TABLE t1 AS sub
   187    230   }
   188    231   do_eqp_test 3.1.2 {
   189    232     SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub);
   190    233   } {
   191         -  0 0 0 {SCAN TABLE t1}
   192         -  0 0 0 {EXECUTE SCALAR SUBQUERY 1}
   193         -  1 0 0 {SCAN TABLE t1 AS sub}
          234  +  QUERY PLAN
          235  +  |--SCAN TABLE t1
          236  +  `--SCALAR SUBQUERY
          237  +     `--SCAN TABLE t1 AS sub
   194    238   }
   195    239   do_eqp_test 3.1.3 {
   196    240     SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub ORDER BY y);
   197    241   } {
   198         -  0 0 0 {SCAN TABLE t1}
   199         -  0 0 0 {EXECUTE SCALAR SUBQUERY 1}
   200         -  1 0 0 {SCAN TABLE t1 AS sub}
   201         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY}
          242  +  QUERY PLAN
          243  +  |--SCAN TABLE t1
          244  +  `--SCALAR SUBQUERY
          245  +     |--SCAN TABLE t1 AS sub
          246  +     `--USE TEMP B-TREE FOR ORDER BY
   202    247   }
   203    248   do_eqp_test 3.1.4 {
   204    249     SELECT * FROM t1 WHERE (SELECT x FROM t2 ORDER BY x);
   205    250   } {
   206         -  0 0 0 {SCAN TABLE t1}
   207         -  0 0 0 {EXECUTE SCALAR SUBQUERY 1}
   208         -  1 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1}
          251  +  QUERY PLAN
          252  +  |--SCAN TABLE t1
          253  +  `--SCALAR SUBQUERY
          254  +     `--SCAN TABLE t2 USING COVERING INDEX t2i1
   209    255   }
   210    256   
   211    257   det 3.2.1 {
   212    258     SELECT * FROM (SELECT * FROM t1 ORDER BY x LIMIT 10) ORDER BY y LIMIT 5
   213    259   } {
   214         -  1 0 0 {SCAN TABLE t1} 
   215         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY} 
   216         -  0 0 0 {SCAN SUBQUERY 1} 
   217         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          260  +  QUERY PLAN
          261  +  |--CO-ROUTINE xxxxxx
          262  +  |  |--SCAN TABLE t1
          263  +  |  `--USE TEMP B-TREE FOR ORDER BY
          264  +  |--SCAN SUBQUERY xxxxxx
          265  +  `--USE TEMP B-TREE FOR ORDER BY
   218    266   }
   219    267   det 3.2.2 {
   220    268     SELECT * FROM 
   221    269       (SELECT * FROM t1 ORDER BY x LIMIT 10) AS x1,
   222    270       (SELECT * FROM t2 ORDER BY x LIMIT 10) AS x2
   223    271     ORDER BY x2.y LIMIT 5
   224    272   } {
   225         -  1 0 0 {SCAN TABLE t1} 
   226         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY} 
   227         -  2 0 0 {SCAN TABLE t2 USING INDEX t2i1} 
   228         -  0 0 0 {SCAN SUBQUERY 1 AS x1} 
   229         -  0 1 1 {SCAN SUBQUERY 2 AS x2} 
   230         -  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
          273  +  QUERY PLAN
          274  +  |--MATERIALIZE xxxxxx
          275  +  |  |--SCAN TABLE t1
          276  +  |  `--USE TEMP B-TREE FOR ORDER BY
          277  +  |--MATERIALIZE xxxxxx
          278  +  |  `--SCAN TABLE t2 USING INDEX t2i1
          279  +  |--SCAN SUBQUERY xxxxxx AS x1
          280  +  |--SCAN SUBQUERY xxxxxx AS x2
          281  +  `--USE TEMP B-TREE FOR ORDER BY
   231    282   }
   232    283   
   233    284   det 3.3.1 {
   234    285     SELECT * FROM t1 WHERE y IN (SELECT y FROM t2)
   235    286   } {
   236         -  0 0 0 {SCAN TABLE t1} 
   237         -  0 0 0 {EXECUTE LIST SUBQUERY 1} 
   238         -  1 0 0 {SCAN TABLE t2}
          287  +  QUERY PLAN
          288  +  |--SCAN TABLE t1
          289  +  `--LIST SUBQUERY
          290  +     `--SCAN TABLE t2
   239    291   }
   240    292   det 3.3.2 {
   241    293     SELECT * FROM t1 WHERE y IN (SELECT y FROM t2 WHERE t1.x!=t2.x)
   242    294   } {
   243         -  0 0 0 {SCAN TABLE t1} 
   244         -  0 0 0 {EXECUTE CORRELATED LIST SUBQUERY 1} 
   245         -  1 0 0 {SCAN TABLE t2}
          295  +  QUERY PLAN
          296  +  |--SCAN TABLE t1
          297  +  `--CORRELATED LIST SUBQUERY
          298  +     `--SCAN TABLE t2
   246    299   }
   247    300   det 3.3.3 {
   248    301     SELECT * FROM t1 WHERE EXISTS (SELECT y FROM t2 WHERE t1.x!=t2.x)
   249    302   } {
   250         -  0 0 0 {SCAN TABLE t1} 
   251         -  0 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 1} 
   252         -  1 0 0 {SCAN TABLE t2}
          303  +  QUERY PLAN
          304  +  |--SCAN TABLE t1
          305  +  `--CORRELATED SCALAR SUBQUERY
          306  +     `--SCAN TABLE t2
   253    307   }
   254    308   
   255    309   #-------------------------------------------------------------------------
   256    310   # Test cases eqp-4.* - tests for composite select statements.
   257    311   #
   258    312   do_eqp_test 4.1.1 {
   259    313     SELECT * FROM t1 UNION ALL SELECT * FROM t2
   260    314   } {
   261         -  1 0 0 {SCAN TABLE t1} 
   262         -  2 0 0 {SCAN TABLE t2} 
   263         -  0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} 
          315  +  QUERY PLAN
          316  +  `--COMPOUND QUERY
          317  +     |--LEFT-MOST SUBQUERY
          318  +     |  `--SCAN TABLE t1
          319  +     `--UNION ALL
          320  +        `--SCAN TABLE t2
   264    321   }
   265    322   do_eqp_test 4.1.2 {
   266    323     SELECT * FROM t1 UNION ALL SELECT * FROM t2 ORDER BY 2
   267    324   } {
   268         -  1 0 0 {SCAN TABLE t1} 
   269         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY}
   270         -  2 0 0 {SCAN TABLE t2} 
   271         -  2 0 0 {USE TEMP B-TREE FOR ORDER BY}
   272         -  0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} 
          325  +  QUERY PLAN
          326  +  `--MERGE (UNION ALL)
          327  +     |--LEFT
          328  +     |  |--SCAN TABLE t1
          329  +     |  `--USE TEMP B-TREE FOR ORDER BY
          330  +     `--RIGHT
          331  +        |--SCAN TABLE t2
          332  +        `--USE TEMP B-TREE FOR ORDER BY
   273    333   }
   274    334   do_eqp_test 4.1.3 {
   275    335     SELECT * FROM t1 UNION SELECT * FROM t2 ORDER BY 2
   276    336   } {
   277         -  1 0 0 {SCAN TABLE t1} 
   278         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY}
   279         -  2 0 0 {SCAN TABLE t2} 
   280         -  2 0 0 {USE TEMP B-TREE FOR ORDER BY}
   281         -  0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION)} 
          337  +  QUERY PLAN
          338  +  `--MERGE (UNION)
          339  +     |--LEFT
          340  +     |  |--SCAN TABLE t1
          341  +     |  `--USE TEMP B-TREE FOR ORDER BY
          342  +     `--RIGHT
          343  +        |--SCAN TABLE t2
          344  +        `--USE TEMP B-TREE FOR ORDER BY
   282    345   }
   283    346   do_eqp_test 4.1.4 {
   284    347     SELECT * FROM t1 INTERSECT SELECT * FROM t2 ORDER BY 2
   285    348   } {
   286         -  1 0 0 {SCAN TABLE t1} 
   287         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY}
   288         -  2 0 0 {SCAN TABLE t2} 
   289         -  2 0 0 {USE TEMP B-TREE FOR ORDER BY}
   290         -  0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (INTERSECT)} 
          349  +  QUERY PLAN
          350  +  `--MERGE (INTERSECT)
          351  +     |--LEFT
          352  +     |  |--SCAN TABLE t1
          353  +     |  `--USE TEMP B-TREE FOR ORDER BY
          354  +     `--RIGHT
          355  +        |--SCAN TABLE t2
          356  +        `--USE TEMP B-TREE FOR ORDER BY
   291    357   }
   292    358   do_eqp_test 4.1.5 {
   293    359     SELECT * FROM t1 EXCEPT SELECT * FROM t2 ORDER BY 2
   294    360   } {
   295         -  1 0 0 {SCAN TABLE t1} 
   296         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY}
   297         -  2 0 0 {SCAN TABLE t2} 
   298         -  2 0 0 {USE TEMP B-TREE FOR ORDER BY}
   299         -  0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} 
          361  +  QUERY PLAN
          362  +  `--MERGE (EXCEPT)
          363  +     |--LEFT
          364  +     |  |--SCAN TABLE t1
          365  +     |  `--USE TEMP B-TREE FOR ORDER BY
          366  +     `--RIGHT
          367  +        |--SCAN TABLE t2
          368  +        `--USE TEMP B-TREE FOR ORDER BY
   300    369   }
   301    370   
   302    371   do_eqp_test 4.2.2 {
   303    372     SELECT * FROM t1 UNION ALL SELECT * FROM t2 ORDER BY 1
   304    373   } {
   305         -  1 0 0 {SCAN TABLE t1} 
   306         -  1 0 0 {USE TEMP B-TREE FOR ORDER BY}
   307         -  2 0 0 {SCAN TABLE t2 USING INDEX t2i1} 
   308         -  0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} 
          374  +  QUERY PLAN
          375  +  `--MERGE (UNION ALL)
          376  +     |--LEFT
          377  +     |  |--SCAN TABLE t1
          378  +     |  `--USE TEMP B-TREE FOR ORDER BY
          379  +     `--RIGHT
          380  +        `--SCAN TABLE t2 USING INDEX t2i1
   309    381   }