/ Check-in [2d8d13e4]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Update this branch with latest changes from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent-pnu
Files: files | file ages | folders
SHA3-256: 2d8d13e4a169cbfa957c4e978b559817a13927a85e30fe275129bf7e3fb54920
User & Date: dan 2018-04-24 19:22:49
Context
2018-04-26
18:01
Merge latest changes from trunk. Including the "ORDER BY ... LIMIT" optimization. check-in: 83a4c9f0 user: dan tags: begin-concurrent-pnu
2018-04-24
19:22
Update this branch with latest changes from trunk. check-in: 2d8d13e4 user: dan tags: begin-concurrent-pnu
19:21
Merge latest trunk changes into this branch. check-in: b27bd799 user: dan tags: begin-concurrent
2018-03-20
14:00
Merge all recent trunk enchancements, and especially the autoincrement write reduction fix. check-in: 25790049 user: drh tags: begin-concurrent-pnu
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   183    183            memdb.lo memjournal.lo \
   184    184            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   185    185            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   186    186            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   187    187            random.lo resolve.lo rowset.lo rtree.lo \
   188    188            sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
   189    189            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   190         -         update.lo util.lo vacuum.lo \
          190  +         update.lo upsert.lo util.lo vacuum.lo \
   191    191            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   192    192            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   193    193            utf.lo vtab.lo
   194    194   
   195    195   # Object files for the amalgamation.
   196    196   #
   197    197   LIBOBJS1 = sqlite3.lo
................................................................................
   280    280     $(TOP)/src/tclsqlite.c \
   281    281     $(TOP)/src/threads.c \
   282    282     $(TOP)/src/tokenize.c \
   283    283     $(TOP)/src/treeview.c \
   284    284     $(TOP)/src/trigger.c \
   285    285     $(TOP)/src/utf.c \
   286    286     $(TOP)/src/update.c \
          287  +  $(TOP)/src/upsert.c \
   287    288     $(TOP)/src/util.c \
   288    289     $(TOP)/src/vacuum.c \
   289    290     $(TOP)/src/vdbe.c \
   290    291     $(TOP)/src/vdbe.h \
   291    292     $(TOP)/src/vdbeapi.c \
   292    293     $(TOP)/src/vdbeaux.c \
   293    294     $(TOP)/src/vdbeblob.c \
................................................................................
   910    911   
   911    912   trigger.lo:	$(TOP)/src/trigger.c $(HDR)
   912    913   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/trigger.c
   913    914   
   914    915   update.lo:	$(TOP)/src/update.c $(HDR)
   915    916   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c
   916    917   
          918  +upsert.lo:	$(TOP)/src/upsert.c $(HDR)
          919  +	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c
          920  +
   917    921   utf.lo:	$(TOP)/src/utf.c $(HDR)
   918    922   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c
   919    923   
   920    924   util.lo:	$(TOP)/src/util.c $(HDR)
   921    925   	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/util.c
   922    926   
   923    927   vacuum.lo:	$(TOP)/src/vacuum.c $(HDR)

Changes to Makefile.msc.

  1189   1189            memdb.lo memjournal.lo \
  1190   1190            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
  1191   1191            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
  1192   1192            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
  1193   1193            random.lo resolve.lo rowset.lo rtree.lo \
  1194   1194            sqlite3session.lo select.lo sqlite3rbu.lo status.lo \
  1195   1195            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
  1196         -         update.lo util.lo vacuum.lo \
         1196  +         update.lo upsert.lo util.lo vacuum.lo \
  1197   1197            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
  1198   1198            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
  1199   1199            utf.lo vtab.lo
  1200   1200   # <</mark>>
  1201   1201   
  1202   1202   # Object files for the amalgamation.
  1203   1203   #
................................................................................
  1288   1288     $(TOP)\src\threads.c \
  1289   1289     $(TOP)\src\tclsqlite.c \
  1290   1290     $(TOP)\src\tokenize.c \
  1291   1291     $(TOP)\src\treeview.c \
  1292   1292     $(TOP)\src\trigger.c \
  1293   1293     $(TOP)\src\utf.c \
  1294   1294     $(TOP)\src\update.c \
         1295  +  $(TOP)\src\upsert.c \
  1295   1296     $(TOP)\src\util.c \
  1296   1297     $(TOP)\src\vacuum.c \
  1297   1298     $(TOP)\src\vdbe.c \
  1298   1299     $(TOP)\src\vdbeapi.c \
  1299   1300     $(TOP)\src\vdbeaux.c \
  1300   1301     $(TOP)\src\vdbeblob.c \
  1301   1302     $(TOP)\src\vdbemem.c \
................................................................................
  1668   1669   $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
  1669   1670   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
  1670   1671   
  1671   1672   # <<block2>>
  1672   1673   sqlite3.def:	libsqlite3.lib
  1673   1674   	echo EXPORTS > sqlite3.def
  1674   1675   	dumpbin /all libsqlite3.lib \
  1675         -		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup)?_[^@]*)(?:@\d+)?$$" \1 \
         1676  +		| $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@]*)(?:@\d+)?$$" \1 \
  1676   1677   		| sort >> sqlite3.def
  1677   1678   # <</block2>>
  1678   1679   
  1679   1680   $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
  1680   1681   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
  1681   1682   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1682   1683   
................................................................................
  1989   1990   
  1990   1991   trigger.lo:	$(TOP)\src\trigger.c $(HDR)
  1991   1992   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\trigger.c
  1992   1993   
  1993   1994   update.lo:	$(TOP)\src\update.c $(HDR)
  1994   1995   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\update.c
  1995   1996   
         1997  +upsert.lo:	$(TOP)\src\upsert.c $(HDR)
         1998  +	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\upsert.c
         1999  +
  1996   2000   utf.lo:	$(TOP)\src\utf.c $(HDR)
  1997   2001   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\utf.c
  1998   2002   
  1999   2003   util.lo:	$(TOP)\src\util.c $(HDR)
  2000   2004   	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\util.c
  2001   2005   
  2002   2006   vacuum.lo:	$(TOP)\src\vacuum.c $(HDR)
................................................................................
  2087   2091   
  2088   2092   keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
  2089   2093   	.\mkkeywordhash.exe > keywordhash.h
  2090   2094   
  2091   2095   # Source files that go into making shell.c
  2092   2096   SHELL_SRC = \
  2093   2097   	$(TOP)\src\shell.c.in \
  2094         -        $(TOP)\ext\misc\appendvfs.c \
         2098  +	$(TOP)\ext\misc\appendvfs.c \
  2095   2099   	$(TOP)\ext\misc\shathree.c \
  2096   2100   	$(TOP)\ext\misc\fileio.c \
  2097   2101   	$(TOP)\ext\misc\completion.c \
  2098   2102   	$(TOP)\ext\expert\sqlite3expert.c \
  2099   2103   	$(TOP)\ext\expert\sqlite3expert.h \
  2100   2104   	$(TOP)\src\test_windirent.c
  2101   2105   

Changes to README.md.

     1      1   <h1 align="center">SQLite Source Repository</h1>
     2      2   
     3      3   This repository contains the complete source code for the SQLite database
     4      4   engine.  Some test scripts are also included.  However, many other test scripts
     5      5   and most of the documentation are managed separately.
     6      6   
     7         -If you are reading this on a Git mirror someplace, you are doing it wrong.
     8         -The [official repository](https://www.sqlite.org/src/) is better.  Go there
     9         -now.
            7  +SQLite [does not use Git](https://sqlite.org/whynotgit.html).
            8  +If you are reading this on GitHub, then you are looking at an
            9  +unofficial mirror. See <https://sqlite.org/src> for the official
           10  +repository.
    10     11   
    11     12   ## Obtaining The Code
    12     13   
    13     14   SQLite sources are managed using the
    14     15   [Fossil](https://www.fossil-scm.org/), a distributed version control system
    15     16   that was specifically designed to support SQLite development.
    16     17   If you do not want to use Fossil, you can download tarballs or ZIP

Changes to VERSION.

     1         -3.23.0
            1  +3.24.0

Changes to autoconf/Makefile.msc.

   962    962   
   963    963   Replace.exe:
   964    964   	$(CSC) /target:exe $(TOP)\Replace.cs
   965    965   
   966    966   sqlite3.def:	Replace.exe $(LIBOBJ)
   967    967   	echo EXPORTS > sqlite3.def
   968    968   	dumpbin /all $(LIBOBJ) \
   969         -		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
          969  +		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset|changegroup|rebaser)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \
   970    970   		| sort >> sqlite3.def
   971    971   
   972    972   $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
   973    973   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
   974    974   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
   975    975   
   976    976   

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.23.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.24.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.23.0'
   730         -PACKAGE_STRING='sqlite 3.23.0'
          729  +PACKAGE_VERSION='3.24.0'
          730  +PACKAGE_STRING='sqlite 3.24.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1461   1461   #
  1462   1462   # Report the --help message.
  1463   1463   #
  1464   1464   if test "$ac_init_help" = "long"; then
  1465   1465     # Omit some internal or obsolete options to make the list less imposing.
  1466   1466     # This message is too long to be a string in the A/UX 3.1 sh.
  1467   1467     cat <<_ACEOF
  1468         -\`configure' configures sqlite 3.23.0 to adapt to many kinds of systems.
         1468  +\`configure' configures sqlite 3.24.0 to adapt to many kinds of systems.
  1469   1469   
  1470   1470   Usage: $0 [OPTION]... [VAR=VALUE]...
  1471   1471   
  1472   1472   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1473   1473   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1474   1474   
  1475   1475   Defaults for the options are specified in brackets.
................................................................................
  1526   1526     --build=BUILD     configure for building on BUILD [guessed]
  1527   1527     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1528   1528   _ACEOF
  1529   1529   fi
  1530   1530   
  1531   1531   if test -n "$ac_init_help"; then
  1532   1532     case $ac_init_help in
  1533         -     short | recursive ) echo "Configuration of sqlite 3.23.0:";;
         1533  +     short | recursive ) echo "Configuration of sqlite 3.24.0:";;
  1534   1534      esac
  1535   1535     cat <<\_ACEOF
  1536   1536   
  1537   1537   Optional Features:
  1538   1538     --disable-option-checking  ignore unrecognized --enable/--with options
  1539   1539     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1540   1540     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1651   1651       cd "$ac_pwd" || { ac_status=$?; break; }
  1652   1652     done
  1653   1653   fi
  1654   1654   
  1655   1655   test -n "$ac_init_help" && exit $ac_status
  1656   1656   if $ac_init_version; then
  1657   1657     cat <<\_ACEOF
  1658         -sqlite configure 3.23.0
         1658  +sqlite configure 3.24.0
  1659   1659   generated by GNU Autoconf 2.69
  1660   1660   
  1661   1661   Copyright (C) 2012 Free Software Foundation, Inc.
  1662   1662   This configure script is free software; the Free Software Foundation
  1663   1663   gives unlimited permission to copy, distribute and modify it.
  1664   1664   _ACEOF
  1665   1665     exit
................................................................................
  2070   2070     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2071   2071   
  2072   2072   } # ac_fn_c_check_header_mongrel
  2073   2073   cat >config.log <<_ACEOF
  2074   2074   This file contains any messages produced by compilers while
  2075   2075   running configure, to aid debugging if configure makes a mistake.
  2076   2076   
  2077         -It was created by sqlite $as_me 3.23.0, which was
         2077  +It was created by sqlite $as_me 3.24.0, which was
  2078   2078   generated by GNU Autoconf 2.69.  Invocation command line was
  2079   2079   
  2080   2080     $ $0 $@
  2081   2081   
  2082   2082   _ACEOF
  2083   2083   exec 5>>config.log
  2084   2084   {
................................................................................
 12238  12238   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12239  12239   
 12240  12240   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12241  12241   # Save the log message, to keep $0 and so on meaningful, and to
 12242  12242   # report actual input values of CONFIG_FILES etc. instead of their
 12243  12243   # values after options handling.
 12244  12244   ac_log="
 12245         -This file was extended by sqlite $as_me 3.23.0, which was
        12245  +This file was extended by sqlite $as_me 3.24.0, which was
 12246  12246   generated by GNU Autoconf 2.69.  Invocation command line was
 12247  12247   
 12248  12248     CONFIG_FILES    = $CONFIG_FILES
 12249  12249     CONFIG_HEADERS  = $CONFIG_HEADERS
 12250  12250     CONFIG_LINKS    = $CONFIG_LINKS
 12251  12251     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12252  12252     $ $0 $@
................................................................................
 12304  12304   
 12305  12305   Report bugs to the package provider."
 12306  12306   
 12307  12307   _ACEOF
 12308  12308   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12309  12309   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12310  12310   ac_cs_version="\\
 12311         -sqlite config.status 3.23.0
        12311  +sqlite config.status 3.24.0
 12312  12312   configured by $0, generated by GNU Autoconf 2.69,
 12313  12313     with options \\"\$ac_cs_config\\"
 12314  12314   
 12315  12315   Copyright (C) 2012 Free Software Foundation, Inc.
 12316  12316   This config.status script is free software; the Free Software Foundation
 12317  12317   gives unlimited permission to copy, distribute and modify it."
 12318  12318   

Changes to doc/lemon.html.

    95     95   As of this writing, the following command-line options are supported:
    96     96   <ul>
    97     97   <li><b>-b</b>
    98     98   Show only the basis for each parser state in the report file.
    99     99   <li><b>-c</b>
   100    100   Do not compress the generated action tables.  The parser will be a
   101    101   little larger and slower, but it will detect syntax errors sooner.
          102  +<li><b>-d</b><i>directory</i>
          103  +Write all output files into <i>directory</i>.  Normally, output files
          104  +are written into the directory that contains the input grammar file.
   102    105   <li><b>-D<i>name</i></b>
   103    106   Define C preprocessor macro <i>name</i>.  This macro is usable by
   104    107   "<tt><a href='#pifdef'>%ifdef</a></tt>" and
   105    108   "<tt><a href='#pifdef'>%ifndef</a></tt>" lines
   106    109   in the grammar file.
   107    110   <li><b>-g</b>
   108    111   Do not generate a parser.  Instead write the input grammar to standard
................................................................................
   675    678   </pre></p>
   676    679   
   677    680   <p>Then the Parse() function generated will have an 4th parameter
   678    681   of type "MyStruct*" and all action routines will have access to
   679    682   a variable named "pAbc" that is the value of the 4th parameter
   680    683   in the most recent call to Parse().</p>
   681    684   
          685  +<p>The <tt>%extra_context</tt> directive works the same except that it
          686  +is passed in on the ParseAlloc() or ParseInit() routines instead of
          687  +on Parse().
          688  +
          689  +<a name='extractx'></a>
          690  +<h4>The <tt>%extra_context</tt> directive</h4>
          691  +
          692  +The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
          693  +to the parameter list of the ParseAlloc() and ParseInif() functions.  Lemon
          694  +doesn't do anything itself with these extra argument, but it does
          695  +store the value make it available to C-code action routines, destructors,
          696  +and so forth.  For example, if the grammar file contains:</p>
          697  +
          698  +<p><pre>
          699  +    %extra_context { MyStruct *pAbc }
          700  +</pre></p>
          701  +
          702  +<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
          703  +of type "MyStruct*" and all action routines will have access to
          704  +a variable named "pAbc" that is the value of that 2th parameter.</p>
          705  +
          706  +<p>The <tt>%extra_argument</tt> directive works the same except that it
          707  +is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
          708  +
   682    709   <a name='pfallback'></a>
   683    710   <h4>The <tt>%fallback</tt> directive</h4>
   684    711   
   685    712   <p>The <tt>%fallback</tt> directive specifies an alternative meaning for one
   686    713   or more tokens.  The alternative meaning is tried if the original token
   687    714   would have generated a syntax error.</p>
   688    715   

Changes to ext/expert/expert1.test.

   281    281     CREATE TRIGGER t9t AFTER INSERT ON t9 BEGIN
   282    282       UPDATE t10 SET a=new.a WHERE b = new.b;
   283    283     END;
   284    284   } {
   285    285     INSERT INTO t9 VALUES(?, ?, ?);
   286    286   } {
   287    287     CREATE INDEX t10_idx_00000062 ON t10(b); 
          288  +  0|1|0|-- TRIGGER t9t
   288    289     0|0|0|SEARCH TABLE t10 USING INDEX t10_idx_00000062 (b=?)
   289    290   }
   290    291   
   291    292   do_setup_rec_test $tn.15 {
   292    293     CREATE TABLE t1(a, b);
   293    294     CREATE TABLE t2(c, d);
   294    295   
................................................................................
   375    376     t2 t2_idx_00000063 {100 20} 
   376    377     t2 t2_idx_00000064 {100 5} 
   377    378     t2 t2_idx_0001295b {100 20 5}
   378    379   }
   379    380   
   380    381   
   381    382   finish_test
   382         -

Changes to ext/fts5/fts5_main.c.

   530    530                                       FTS5_BI_ROWID_GE, 0, 0, -1},
   531    531     };
   532    532   
   533    533     int aColMap[3];
   534    534     aColMap[0] = -1;
   535    535     aColMap[1] = nCol;
   536    536     aColMap[2] = nCol+1;
          537  +
          538  +  assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH );
          539  +  assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH );
          540  +  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
          541  +  assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH );
          542  +  assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
   537    543   
   538    544     /* Set idxFlags flags for all WHERE clause terms that will be used. */
   539    545     for(i=0; i<pInfo->nConstraint; i++){
   540    546       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   541    547       int iCol = p->iColumn;
   542    548   
   543    549       if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
................................................................................
   549    555           aConstraint[0].iConsIndex = i;
   550    556         }else{
   551    557           /* As there exists an unusable MATCH constraint this is an 
   552    558           ** unusable plan. Set a prohibitively high cost. */
   553    559           pInfo->estimatedCost = 1e50;
   554    560           return SQLITE_OK;
   555    561         }
   556         -    }else{
          562  +    }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){
   557    563         int j;
   558    564         for(j=1; j<ArraySize(aConstraint); j++){
   559    565           struct Constraint *pC = &aConstraint[j];
   560         -        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
          566  +        if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){
   561    567             pC->iConsIndex = i;
   562    568             idxFlags |= pC->fts5op;
   563    569           }
   564    570         }
   565    571       }
   566    572     }
   567    573   

Changes to ext/fts5/test/fts5aa.test.

   586    586       INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
   587    587     COMMIT;
   588    588   }
   589    589   
   590    590   do_execsql_test 22.1 {
   591    591     SELECT rowid FROM t9('a*')
   592    592   } {1}
          593  +
          594  +#-------------------------------------------------------------------------
          595  +do_execsql_test 23.0 {
          596  +  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
          597  +  CREATE TABLE t11(x);
          598  +}
          599  +do_execsql_test 23.1 {
          600  +  SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
          601  +}
          602  +do_execsql_test 23.2 {
          603  +  SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
          604  +}
   593    605   
   594    606   }
   595    607   
   596    608   expand_all_sql db
   597    609   finish_test

Changes to ext/icu/README.txt.

    35     35   
    36     36          http://www.icu-project.org/userguide/caseMappings.html
    37     37          http://www.icu-project.org/userguide/posix.html#case_mappings
    38     38   
    39     39       To utilise "general" case mapping, the upper() or lower() scalar 
    40     40       functions are invoked with one argument:
    41     41   
    42         -        upper('ABC') -> 'abc'
    43         -        lower('abc') -> 'ABC'
           42  +        upper('abc') -> 'ABC'
           43  +        lower('ABC') -> 'abc'
    44     44   
    45     45       To access ICU "language specific" case mapping, upper() or lower()
    46     46       should be invoked with two arguments. The second argument is the name
    47     47       of the locale to use. Passing an empty string ("") or SQL NULL value
    48     48       as the second argument is the same as invoking the 1 argument version
    49     49       of upper() or lower():
    50     50   

Changes to ext/misc/csv.c.

   128    128       p->zIn = sqlite3_malloc( CSV_INBUFSZ );
   129    129       if( p->zIn==0 ){
   130    130         csv_errmsg(p, "out of memory");
   131    131         return 1;
   132    132       }
   133    133       p->in = fopen(zFilename, "rb");
   134    134       if( p->in==0 ){
          135  +      sqlite3_free(p->zIn);
   135    136         csv_reader_reset(p);
   136    137         csv_errmsg(p, "cannot open '%s' for reading", zFilename);
   137    138         return 1;
   138    139       }
   139    140     }else{
   140    141       assert( p->in==0 );
   141    142       p->zIn = (char*)zData;

Changes to ext/misc/eval.c.

    30     30   
    31     31   /*
    32     32   ** Callback from sqlite_exec() for the eval() function.
    33     33   */
    34     34   static int callback(void *pCtx, int argc, char **argv, char **colnames){
    35     35     struct EvalResult *p = (struct EvalResult*)pCtx;
    36     36     int i; 
           37  +  if( argv==0 ) return 0;
    37     38     for(i=0; i<argc; i++){
    38     39       const char *z = argv[i] ? argv[i] : "";
    39     40       size_t sz = strlen(z);
    40     41       if( (sqlite3_int64)sz+p->nUsed+p->szSep+1 > p->nAlloc ){
    41     42         char *zNew;
    42     43         p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1;
    43     44         /* Using sqlite3_realloc64() would be better, but it is a recent

Changes to ext/misc/fileio.c.

    89     89   #  include <sys/time.h>
    90     90   #else
    91     91   #  include "windows.h"
    92     92   #  include <io.h>
    93     93   #  include <direct.h>
    94     94   #  include "test_windirent.h"
    95     95   #  define dirent DIRENT
           96  +#  ifndef chmod
           97  +#    define chmod _chmod
           98  +#  endif
    96     99   #  ifndef stat
    97    100   #    define stat _stat
    98    101   #  endif
    99    102   #  define mkdir(path,mode) _mkdir(path)
   100    103   #  define lstat(path,buf) stat(path,buf)
   101    104   #endif
   102    105   #include <time.h>
................................................................................
   154    157     va_list ap;
   155    158     va_start(ap, zFmt);
   156    159     zMsg = sqlite3_vmprintf(zFmt, ap);
   157    160     sqlite3_result_error(ctx, zMsg, -1);
   158    161     sqlite3_free(zMsg);
   159    162     va_end(ap);
   160    163   }
          164  +
          165  +#if defined(_WIN32)
          166  +/*
          167  +** This function is designed to convert a Win32 FILETIME structure into the
          168  +** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC).
          169  +*/
          170  +static sqlite3_uint64 fileTimeToUnixTime(
          171  +  LPFILETIME pFileTime
          172  +){
          173  +  SYSTEMTIME epochSystemTime;
          174  +  ULARGE_INTEGER epochIntervals;
          175  +  FILETIME epochFileTime;
          176  +  ULARGE_INTEGER fileIntervals;
          177  +
          178  +  memset(&epochSystemTime, 0, sizeof(SYSTEMTIME));
          179  +  epochSystemTime.wYear = 1970;
          180  +  epochSystemTime.wMonth = 1;
          181  +  epochSystemTime.wDay = 1;
          182  +  SystemTimeToFileTime(&epochSystemTime, &epochFileTime);
          183  +  epochIntervals.LowPart = epochFileTime.dwLowDateTime;
          184  +  epochIntervals.HighPart = epochFileTime.dwHighDateTime;
          185  +
          186  +  fileIntervals.LowPart = pFileTime->dwLowDateTime;
          187  +  fileIntervals.HighPart = pFileTime->dwHighDateTime;
          188  +
          189  +  return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000;
          190  +}
          191  +
          192  +/*
          193  +** This function attempts to normalize the time values found in the stat()
          194  +** buffer to UTC.  This is necessary on Win32, where the runtime library
          195  +** appears to return these values as local times.
          196  +*/
          197  +static void statTimesToUtc(
          198  +  const char *zPath,
          199  +  struct stat *pStatBuf
          200  +){
          201  +  HANDLE hFindFile;
          202  +  WIN32_FIND_DATAW fd;
          203  +  LPWSTR zUnicodeName;
          204  +  extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
          205  +  zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath);
          206  +  if( zUnicodeName ){
          207  +    memset(&fd, 0, sizeof(WIN32_FIND_DATA));
          208  +    hFindFile = FindFirstFileW(zUnicodeName, &fd);
          209  +    if( hFindFile!=NULL ){
          210  +      pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime);
          211  +      pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime);
          212  +      pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime);
          213  +      FindClose(hFindFile);
          214  +    }
          215  +    sqlite3_free(zUnicodeName);
          216  +  }
          217  +}
          218  +#endif
          219  +
          220  +/*
          221  +** This function is used in place of stat().  On Windows, special handling
          222  +** is required in order for the included time to be returned as UTC.  On all
          223  +** other systems, this function simply calls stat().
          224  +*/
          225  +static int fileStat(
          226  +  const char *zPath,
          227  +  struct stat *pStatBuf
          228  +){
          229  +#if defined(_WIN32)
          230  +  int rc = stat(zPath, pStatBuf);
          231  +  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
          232  +  return rc;
          233  +#else
          234  +  return stat(zPath, pStatBuf);
          235  +#endif
          236  +}
          237  +
          238  +/*
          239  +** This function is used in place of lstat().  On Windows, special handling
          240  +** is required in order for the included time to be returned as UTC.  On all
          241  +** other systems, this function simply calls lstat().
          242  +*/
          243  +static int fileLinkStat(
          244  +  const char *zPath,
          245  +  struct stat *pStatBuf
          246  +){
          247  +#if defined(_WIN32)
          248  +  int rc = lstat(zPath, pStatBuf);
          249  +  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
          250  +  return rc;
          251  +#else
          252  +  return lstat(zPath, pStatBuf);
          253  +#endif
          254  +}
   161    255   
   162    256   /*
   163    257   ** Argument zFile is the name of a file that will be created and/or written
   164    258   ** by SQL function writefile(). This function ensures that the directory
   165    259   ** zFile will be written to exists, creating it if required. The permissions
   166    260   ** for any path components created by this function are set to (mode&0777).
   167    261   **
................................................................................
   186    280         struct stat sStat;
   187    281         int rc2;
   188    282   
   189    283         for(; zCopy[i]!='/' && i<nCopy; i++);
   190    284         if( i==nCopy ) break;
   191    285         zCopy[i] = '\0';
   192    286   
   193         -      rc2 = stat(zCopy, &sStat);
          287  +      rc2 = fileStat(zCopy, &sStat);
   194    288         if( rc2!=0 ){
   195    289           if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
   196    290         }else{
   197    291           if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
   198    292         }
   199    293         zCopy[i] = '/';
   200    294         i++;
................................................................................
   228    322         if( mkdir(zFile, mode) ){
   229    323           /* The mkdir() call to create the directory failed. This might not
   230    324           ** be an error though - if there is already a directory at the same
   231    325           ** path and either the permissions already match or can be changed
   232    326           ** to do so using chmod(), it is not an error.  */
   233    327           struct stat sStat;
   234    328           if( errno!=EEXIST
   235         -         || 0!=stat(zFile, &sStat)
          329  +         || 0!=fileStat(zFile, &sStat)
   236    330            || !S_ISDIR(sStat.st_mode)
   237    331            || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
   238    332           ){
   239    333             return 1;
   240    334           }
   241    335         }
   242    336       }else{
................................................................................
   275    369   
   276    370       GetSystemTime(&currentTime);
   277    371       SystemTimeToFileTime(&currentTime, &lastAccess);
   278    372       intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
   279    373       lastWrite.dwLowDateTime = (DWORD)intervals;
   280    374       lastWrite.dwHighDateTime = intervals >> 32;
   281    375       zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
          376  +    if( zUnicodeName==0 ){
          377  +      return 1;
          378  +    }
   282    379       hFile = CreateFileW(
   283    380         zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
   284    381         FILE_FLAG_BACKUP_SEMANTICS, NULL
   285    382       );
   286    383       sqlite3_free(zUnicodeName);
   287    384       if( hFile!=INVALID_HANDLE_VALUE ){
   288    385         BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
   289    386         CloseHandle(hFile);
   290    387         return !bResult;
   291    388       }else{
   292    389         return 1;
   293    390       }
   294         -#elif defined(AT_FDCWD) && 0 /* utimensat() is not univerally available */
          391  +#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */
   295    392       /* Recent unix */
   296    393       struct timespec times[2];
   297    394       times[0].tv_nsec = times[1].tv_nsec = 0;
   298    395       times[0].tv_sec = time(0);
   299    396       times[1].tv_sec = mtime;
   300    397       if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
   301    398         return 1;
................................................................................
   564    661         if( pEntry->d_name[0]=='.' ){
   565    662          if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
   566    663          if( pEntry->d_name[1]=='\0' ) continue;
   567    664         }
   568    665         sqlite3_free(pCur->zPath);
   569    666         pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
   570    667         if( pCur->zPath==0 ) return SQLITE_NOMEM;
   571         -      if( lstat(pCur->zPath, &pCur->sStat) ){
          668  +      if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
   572    669           fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
   573    670           return SQLITE_ERROR;
   574    671         }
   575    672         return SQLITE_OK;
   576    673       }
   577    674       closedir(pLvl->pDir);
   578    675       sqlite3_free(pLvl->zDir);
................................................................................
   698    795     }else{
   699    796       pCur->zPath = sqlite3_mprintf("%s", zDir);
   700    797     }
   701    798   
   702    799     if( pCur->zPath==0 ){
   703    800       return SQLITE_NOMEM;
   704    801     }
   705         -  if( lstat(pCur->zPath, &pCur->sStat) ){
          802  +  if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
   706    803       fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
   707    804       return SQLITE_ERROR;
   708    805     }
   709    806   
   710    807     return SQLITE_OK;
   711    808   }
   712    809   

Changes to ext/misc/series.c.

   265    265       pCur->mxValue = 0xffffffff;
   266    266     }
   267    267     if( idxNum & 4 ){
   268    268       pCur->iStep = sqlite3_value_int64(argv[i++]);
   269    269       if( pCur->iStep<1 ) pCur->iStep = 1;
   270    270     }else{
   271    271       pCur->iStep = 1;
          272  +  }
          273  +  for(i=0; i<argc; i++){
          274  +    if( sqlite3_value_type(argv[i])==SQLITE_NULL ){
          275  +      /* If any of the constraints have a NULL value, then return no rows.
          276  +      ** See ticket https://www.sqlite.org/src/info/fac496b61722daf2 */
          277  +      pCur->mnValue = 1;
          278  +      pCur->mxValue = 0;
          279  +      break;
          280  +    }
   272    281     }
   273    282     if( idxNum & 8 ){
   274    283       pCur->isDesc = 1;
   275    284       pCur->iValue = pCur->mxValue;
   276    285       if( pCur->iStep>0 ){
   277    286         pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep;
   278    287       }

Changes to ext/misc/spellfix.c.

   760    760       int nTo = zTo ? sqlite3_column_bytes(pStmt, 2) : 0;
   761    761       int iCost = sqlite3_column_int(pStmt, 3);
   762    762   
   763    763       assert( zFrom!=0 || nFrom==0 );
   764    764       assert( zTo!=0 || nTo==0 );
   765    765       if( nFrom>100 || nTo>100 ) continue;
   766    766       if( iCost<0 ) continue;
   767         -    if( iCost>10000 ) continue;   /* Costs above 10K are considered infinite */
          767  +    if( iCost>=10000 ) continue;  /* Costs above 10K are considered infinite */
   768    768       if( pLang==0 || iLang!=iLangPrev ){
   769    769         EditDist3Lang *pNew;
   770    770         pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0]));
   771    771         if( pNew==0 ){ rc = SQLITE_NOMEM; break; }
   772    772         p->a = pNew;
   773    773         pLang = &p->a[p->nLang];
   774    774         p->nLang++;
................................................................................
   831    831   }
   832    832   
   833    833   /*
   834    834   ** Return TRUE (non-zero) if the To side of the given cost matches
   835    835   ** the given string.
   836    836   */
   837    837   static int matchTo(EditDist3Cost *p, const char *z, int n){
          838  +  assert( n>0 );
   838    839     if( p->a[p->nFrom]!=z[0] ) return 0;
   839    840     if( p->nTo>n ) return 0;
   840    841     if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0;
   841    842     return 1;
   842    843   }
   843    844   
   844    845   /*
   845    846   ** Return TRUE (non-zero) if the From side of the given cost matches
   846    847   ** the given string.
   847    848   */
   848    849   static int matchFrom(EditDist3Cost *p, const char *z, int n){
   849    850     assert( p->nFrom<=n );
   850         -  if( p->a[0]!=z[0] ) return 0;
   851         -  if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
          851  +  if( p->nFrom ){
          852  +    if( p->a[0]!=z[0] ) return 0;
          853  +    if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
          854  +  }
   852    855     return 1;
   853    856   }
   854    857   
   855    858   /*
   856    859   ** Return TRUE (non-zero) of the next FROM character and the next TO
   857    860   ** character are the same.
   858    861   */
................................................................................
   860    863     EditDist3FromString *pStr,  /* Left hand string */
   861    864     int n1,                     /* Index of comparison character on the left */
   862    865     const char *z2,             /* Right-handl comparison character */
   863    866     int n2                      /* Bytes remaining in z2[] */
   864    867   ){
   865    868     int b1 = pStr->a[n1].nByte;
   866    869     if( b1>n2 ) return 0;
          870  +  assert( b1>0 );
   867    871     if( pStr->z[n1]!=z2[0] ) return 0;
   868    872     if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0;
   869    873     return 1;
   870    874   }
   871    875   
   872    876   /*
   873    877   ** Delete an EditDist3FromString objecct

Added ext/misc/templatevtab.c.

            1  +/*
            2  +** 2018-04-19
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file implements a template virtual-table implementation.
           14  +** Developers can make a copy of this file as a baseline for writing
           15  +** new virtual tables and/or table-valued functions.
           16  +**
           17  +** Steps for writing a new virtual table implementation:
           18  +**
           19  +**     (1)  Make a copy of this file.  Prehaps "mynewvtab.c"
           20  +**
           21  +**     (2)  Replace this header comment with something appropriate for
           22  +**          the new virtual table
           23  +**
           24  +**     (3)  Change every occurrence of "templatevtab" to some other string
           25  +**          appropriate for the new virtual table.  Ideally, the new string
           26  +**          should be the basename of the source file: "mynewvtab".
           27  +**
           28  +**     (4)  Run a test compilation to make sure the unmodified virtual
           29  +**          table works.
           30  +**
           31  +**     (5)  Begin making changes to make the new virtual table do what you
           32  +**          want it to do.
           33  +**
           34  +**     (6)  Ensure that all the "FIXME" comments in the file have been dealt
           35  +**          with.
           36  +**
           37  +** This template is minimal, in the sense that it uses only the required
           38  +** methods on the sqlite3_module object.  As a result, templatevtab is
           39  +** a read-only and eponymous-only table.  Those limitation can be removed
           40  +** by adding new methods.
           41  +*/
           42  +#if !defined(SQLITEINT_H)
           43  +#include "sqlite3ext.h"
           44  +#endif
           45  +SQLITE_EXTENSION_INIT1
           46  +#include <string.h>
           47  +
           48  +/* templatevtab_vtab is a subclass of sqlite3_vtab which is
           49  +** underlying representation of the virtual table
           50  +*/
           51  +typedef struct templatevtab_vtab templatevtab_vtab;
           52  +struct templatevtab_vtab {
           53  +  sqlite3_vtab base;  /* Base class - must be first */
           54  +  /* Add new fields here, as necessary */
           55  +};
           56  +
           57  +/* templatevtab_cursor is a subclass of sqlite3_vtab_cursor which will
           58  +** serve as the underlying representation of a cursor that scans
           59  +** over rows of the result
           60  +*/
           61  +typedef struct templatevtab_cursor templatevtab_cursor;
           62  +struct templatevtab_cursor {
           63  +  sqlite3_vtab_cursor base;  /* Base class - must be first */
           64  +  /* Insert new fields here.  For this templatevtab we only keep track
           65  +  ** of the rowid */
           66  +  sqlite3_int64 iRowid;      /* The rowid */
           67  +};
           68  +
           69  +/*
           70  +** The templatevtabConnect() method is invoked to create a new
           71  +** template virtual table.
           72  +**
           73  +** Think of this routine as the constructor for templatevtab_vtab objects.
           74  +**
           75  +** All this routine needs to do is:
           76  +**
           77  +**    (1) Allocate the templatevtab_vtab object and initialize all fields.
           78  +**
           79  +**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
           80  +**        result set of queries against the virtual table will look like.
           81  +*/
           82  +static int templatevtabConnect(
           83  +  sqlite3 *db,
           84  +  void *pAux,
           85  +  int argc, const char *const*argv,
           86  +  sqlite3_vtab **ppVtab,
           87  +  char **pzErr
           88  +){
           89  +  templatevtab_vtab *pNew;
           90  +  int rc;
           91  +
           92  +  rc = sqlite3_declare_vtab(db,
           93  +           "CREATE TABLE x(a,b)"
           94  +       );
           95  +  if( rc==SQLITE_OK ){
           96  +    pNew = sqlite3_malloc( sizeof(*pNew) );
           97  +    *ppVtab = (sqlite3_vtab*)pNew;
           98  +    if( pNew==0 ) return SQLITE_NOMEM;
           99  +    memset(pNew, 0, sizeof(*pNew));
          100  +  }
          101  +  return rc;
          102  +}
          103  +
          104  +/*
          105  +** This method is the destructor for templatevtab_vtab objects.
          106  +*/
          107  +static int templatevtabDisconnect(sqlite3_vtab *pVtab){
          108  +  templatevtab_vtab *p = (templatevtab_vtab*)pVtab;
          109  +  sqlite3_free(p);
          110  +  return SQLITE_OK;
          111  +}
          112  +
          113  +/*
          114  +** Constructor for a new templatevtab_cursor object.
          115  +*/
          116  +static int templatevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
          117  +  templatevtab_cursor *pCur;
          118  +  pCur = sqlite3_malloc( sizeof(*pCur) );
          119  +  if( pCur==0 ) return SQLITE_NOMEM;
          120  +  memset(pCur, 0, sizeof(*pCur));
          121  +  *ppCursor = &pCur->base;
          122  +  return SQLITE_OK;
          123  +}
          124  +
          125  +/*
          126  +** Destructor for a templatevtab_cursor.
          127  +*/
          128  +static int templatevtabClose(sqlite3_vtab_cursor *cur){
          129  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          130  +  sqlite3_free(pCur);
          131  +  return SQLITE_OK;
          132  +}
          133  +
          134  +
          135  +/*
          136  +** Advance a templatevtab_cursor to its next row of output.
          137  +*/
          138  +static int templatevtabNext(sqlite3_vtab_cursor *cur){
          139  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          140  +  pCur->iRowid++;
          141  +  return SQLITE_OK;
          142  +}
          143  +
          144  +/*
          145  +** Return values of columns for the row at which the templatevtab_cursor
          146  +** is currently pointing.
          147  +*/
          148  +static int templatevtabColumn(
          149  +  sqlite3_vtab_cursor *cur,   /* The cursor */
          150  +  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
          151  +  int i                       /* Which column to return */
          152  +){
          153  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          154  +  sqlite3_result_int(ctx, (i+1)*1000 + pCur->iRowid);
          155  +  return SQLITE_OK;
          156  +}
          157  +
          158  +/*
          159  +** Return the rowid for the current row.  In this implementation, the
          160  +** rowid is the same as the output value.
          161  +*/
          162  +static int templatevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
          163  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          164  +  *pRowid = pCur->iRowid;
          165  +  return SQLITE_OK;
          166  +}
          167  +
          168  +/*
          169  +** Return TRUE if the cursor has been moved off of the last
          170  +** row of output.
          171  +*/
          172  +static int templatevtabEof(sqlite3_vtab_cursor *cur){
          173  +  templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
          174  +  return pCur->iRowid>=10;
          175  +}
          176  +
          177  +/*
          178  +** This method is called to "rewind" the templatevtab_cursor object back
          179  +** to the first row of output.  This method is always called at least
          180  +** once prior to any call to templatevtabColumn() or templatevtabRowid() or 
          181  +** templatevtabEof().
          182  +*/
          183  +static int templatevtabFilter(
          184  +  sqlite3_vtab_cursor *pVtabCursor, 
          185  +  int idxNum, const char *idxStr,
          186  +  int argc, sqlite3_value **argv
          187  +){
          188  +  templatevtab_cursor *pCur = (templatevtab_cursor *)pVtabCursor;
          189  +  pCur->iRowid = 1;
          190  +  return SQLITE_OK;
          191  +}
          192  +
          193  +/*
          194  +** SQLite will invoke this method one or more times while planning a query
          195  +** that uses the virtual table.  This routine needs to create
          196  +** a query plan for each invocation and compute an estimated cost for that
          197  +** plan.
          198  +*/
          199  +static int templatevtabBestIndex(
          200  +  sqlite3_vtab *tab,
          201  +  sqlite3_index_info *pIdxInfo
          202  +){
          203  +  pIdxInfo->estimatedCost = (double)10;
          204  +  pIdxInfo->estimatedRows = 10;
          205  +  return SQLITE_OK;
          206  +}
          207  +
          208  +/*
          209  +** This following structure defines all the methods for the 
          210  +** virtual table.
          211  +*/
          212  +static sqlite3_module templatevtabModule = {
          213  +  /* iVersion    */ 0,
          214  +  /* xCreate     */ 0,
          215  +  /* xConnect    */ templatevtabConnect,
          216  +  /* xBestIndex  */ templatevtabBestIndex,
          217  +  /* xDisconnect */ templatevtabDisconnect,
          218  +  /* xDestroy    */ 0,
          219  +  /* xOpen       */ templatevtabOpen,
          220  +  /* xClose      */ templatevtabClose,
          221  +  /* xFilter     */ templatevtabFilter,
          222  +  /* xNext       */ templatevtabNext,
          223  +  /* xEof        */ templatevtabEof,
          224  +  /* xColumn     */ templatevtabColumn,
          225  +  /* xRowid      */ templatevtabRowid,
          226  +  /* xUpdate     */ 0,
          227  +  /* xBegin      */ 0,
          228  +  /* xSync       */ 0,
          229  +  /* xCommit     */ 0,
          230  +  /* xRollback   */ 0,
          231  +  /* xFindMethod */ 0,
          232  +  /* xRename     */ 0,
          233  +  /* xSavepoint  */ 0,
          234  +  /* xRelease    */ 0,
          235  +  /* xRollbackTo */ 0
          236  +};
          237  +
          238  +
          239  +#ifdef _WIN32
          240  +__declspec(dllexport)
          241  +#endif
          242  +int sqlite3_templatevtab_init(
          243  +  sqlite3 *db, 
          244  +  char **pzErrMsg, 
          245  +  const sqlite3_api_routines *pApi
          246  +){
          247  +  int rc = SQLITE_OK;
          248  +  SQLITE_EXTENSION_INIT2(pApi);
          249  +  rc = sqlite3_create_module(db, "templatevtab", &templatevtabModule, 0);
          250  +  return rc;
          251  +}

Changes to ext/misc/zipfile.c.

  1495   1495   */
  1496   1496   static u32 zipfileGetTime(sqlite3_value *pVal){
  1497   1497     if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){
  1498   1498       return zipfileTime();
  1499   1499     }
  1500   1500     return (u32)sqlite3_value_int64(pVal);
  1501   1501   }
         1502  +
         1503  +/*
         1504  +** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry
         1505  +** linked list.  Remove it from the list and free the object.
         1506  +*/
         1507  +static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){
         1508  +  if( pOld ){
         1509  +    ZipfileEntry **pp;
         1510  +    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
         1511  +    *pp = (*pp)->pNext;
         1512  +    zipfileEntryFree(pOld);
         1513  +  }
         1514  +}
  1502   1515   
  1503   1516   /*
  1504   1517   ** xUpdate method.
  1505   1518   */
  1506   1519   static int zipfileUpdate(
  1507   1520     sqlite3_vtab *pVtab, 
  1508   1521     int nVal, 
................................................................................
  1520   1533     int nPath = 0;                  /* strlen(zPath) */
  1521   1534     const u8 *pData = 0;            /* Pointer to buffer containing content */
  1522   1535     int nData = 0;                  /* Size of pData buffer in bytes */
  1523   1536     int iMethod = 0;                /* Compression method for new entry */
  1524   1537     u8 *pFree = 0;                  /* Free this */
  1525   1538     char *zFree = 0;                /* Also free this */
  1526   1539     ZipfileEntry *pOld = 0;
         1540  +  ZipfileEntry *pOld2 = 0;
         1541  +  int bUpdate = 0;                /* True for an update that modifies "name" */
  1527   1542     int bIsDir = 0;
  1528   1543     u32 iCrc32 = 0;
  1529   1544   
  1530   1545     if( pTab->pWriteFd==0 ){
  1531   1546       rc = zipfileBegin(pVtab);
  1532   1547       if( rc!=SQLITE_OK ) return rc;
  1533   1548     }
  1534   1549   
  1535   1550     /* If this is a DELETE or UPDATE, find the archive entry to delete. */
  1536   1551     if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  1537   1552       const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
  1538   1553       int nDelete = (int)strlen(zDelete);
         1554  +    if( nVal>1 ){
         1555  +      const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]);
         1556  +      if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){
         1557  +        bUpdate = 1;
         1558  +      }
         1559  +    }
  1539   1560       for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
  1540   1561         if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){
  1541   1562           break;
  1542   1563         }
  1543   1564         assert( pOld->pNext );
  1544   1565       }
  1545   1566     }
................................................................................
  1609   1630           zFree = sqlite3_mprintf("%s/", zPath);
  1610   1631           if( zFree==0 ){ rc = SQLITE_NOMEM; }
  1611   1632           zPath = (const char*)zFree;
  1612   1633           nPath++;
  1613   1634         }
  1614   1635       }
  1615   1636   
  1616         -    /* Check that we're not inserting a duplicate entry */
  1617         -    if( pOld==0 && rc==SQLITE_OK ){
         1637  +    /* Check that we're not inserting a duplicate entry -OR- updating an
         1638  +    ** entry with a path, thereby making it into a duplicate. */
         1639  +    if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){
  1618   1640         ZipfileEntry *p;
  1619   1641         for(p=pTab->pFirstEntry; p; p=p->pNext){
  1620   1642           if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){
  1621   1643             switch( sqlite3_vtab_on_conflict(pTab->db) ){
  1622   1644               case SQLITE_IGNORE: {
  1623   1645                 goto zipfile_update_done;
  1624   1646               }
  1625   1647               case SQLITE_REPLACE: {
  1626         -              pOld = p;
         1648  +              pOld2 = p;
  1627   1649                 break;
  1628   1650               }
  1629   1651               default: {
  1630   1652                 zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath);
  1631   1653                 rc = SQLITE_CONSTRAINT;
  1632   1654                 break;
  1633   1655               }
................................................................................
  1657   1679           pNew->mUnixTime = (u32)mTime;
  1658   1680           rc = zipfileAppendEntry(pTab, pNew, pData, nData);
  1659   1681           zipfileAddEntry(pTab, pOld, pNew);
  1660   1682         }
  1661   1683       }
  1662   1684     }
  1663   1685   
  1664         -  if( rc==SQLITE_OK && pOld ){
  1665         -    ZipfileEntry **pp;
         1686  +  if( rc==SQLITE_OK && (pOld || pOld2) ){
  1666   1687       ZipfileCsr *pCsr;
  1667   1688       for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){
  1668         -      if( pCsr->pCurrent==pOld ){
  1669         -        pCsr->pCurrent = pOld->pNext;
         1689  +      if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){
         1690  +        pCsr->pCurrent = pCsr->pCurrent->pNext;
  1670   1691           pCsr->bNoop = 1;
  1671   1692         }
  1672   1693       }
  1673         -    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
  1674         -    *pp = (*pp)->pNext;
  1675         -    zipfileEntryFree(pOld);
         1694  +
         1695  +    zipfileRemoveEntryFromList(pTab, pOld);
         1696  +    zipfileRemoveEntryFromList(pTab, pOld2);
  1676   1697     }
  1677   1698   
  1678   1699   zipfile_update_done:
  1679   1700     sqlite3_free(pFree);
  1680   1701     sqlite3_free(zFree);
  1681   1702     return rc;
  1682   1703   }
................................................................................
  2036   2057   
  2037   2058     /* Append the LFH to the body of the new archive */
  2038   2059     nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9;
  2039   2060     if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out;
  2040   2061     p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]);
  2041   2062   
  2042   2063     /* Append the data to the body of the new archive */
  2043         -  if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out;
  2044         -  memcpy(&p->body.a[p->body.n], aData, nData);
  2045         -  p->body.n += nData;
         2064  +  if( nData>0 ){
         2065  +    if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out;
         2066  +    memcpy(&p->body.a[p->body.n], aData, nData);
         2067  +    p->body.n += nData;
         2068  +  }
  2046   2069   
  2047   2070     /* Append the CDS record to the directory of the new archive */
  2048   2071     nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9;
  2049   2072     if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out;
  2050   2073     p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]);
  2051   2074   
  2052   2075     /* Increment the count of entries in the archive */

Changes to ext/rbu/rbu.c.

    25     25   void usage(const char *zArgv0){
    26     26     fprintf(stderr, 
    27     27   "Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n"
    28     28   "\n"
    29     29   "Where options are:\n"
    30     30   "\n"
    31     31   "    -step NSTEP\n"
           32  +"    -statstep NSTATSTEP\n"
    32     33   "    -vacuum\n"
           34  +"    -presql SQL\n"
    33     35   "\n"
    34     36   "  If the -vacuum switch is not present, argument RBU-DB must be an RBU\n"
    35     37   "  database containing an update suitable for target database TARGET-DB.\n"
    36     38   "  Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n"
    37     39   "  RBU, and RBU-DB is used as the state database for the vacuum (refer to\n"
    38     40   "  API documentation for details).\n"
    39     41   "\n"
................................................................................
    75     77     int i;
    76     78     const char *zTarget;            /* Target database to apply RBU to */
    77     79     const char *zRbu;               /* Database containing RBU */
    78     80     char zBuf[200];                 /* Buffer for printf() */
    79     81     char *zErrmsg;                  /* Error message, if any */
    80     82     sqlite3rbu *pRbu;               /* RBU handle */
    81     83     int nStep = 0;                  /* Maximum number of step() calls */
           84  +  int nStatStep = 0;              /* Report stats after this many step calls */
    82     85     int bVacuum = 0;
           86  +  const char *zPreSql = 0;
    83     87     int rc;
    84     88     sqlite3_int64 nProgress = 0;
    85     89     int nArgc = argc-2;
    86     90   
    87     91     if( argc<3 ) usage(argv[0]);
    88     92     for(i=1; i<nArgc; i++){
    89     93       const char *zArg = argv[i];
    90     94       int nArg = strlen(zArg);
    91     95       if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
    92     96         bVacuum = 1;
           97  +    }else if( nArg>1 && nArg<=7 
           98  +           && 0==memcmp(zArg, "-presql", nArg) && i<nArg-1 ){
           99  +      i++;
          100  +      zPreSql = argv[i];
    93    101       }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
    94    102         i++;
    95    103         nStep = atoi(argv[i]);
          104  +    }else if( nArg>1 && nArg<=9 
          105  +           && 0==memcmp(zArg, "-statstep", nArg) && i<nArg-1 
          106  +    ){
          107  +      i++;
          108  +      nStatStep = atoi(argv[i]);
    96    109       }else{
    97    110         usage(argv[0]);
    98    111       }
    99    112     }
   100    113   
   101    114     zTarget = argv[argc-2];
   102    115     zRbu = argv[argc-1];
................................................................................
   107    120     ** regular RBU update handle otherwise.  */
   108    121     if( bVacuum ){
   109    122       pRbu = sqlite3rbu_vacuum(zTarget, zRbu);
   110    123     }else{
   111    124       pRbu = sqlite3rbu_open(zTarget, zRbu, 0);
   112    125     }
   113    126     report_rbu_vfs(pRbu);
          127  +
          128  +  if( zPreSql && pRbu ){
          129  +    sqlite3 *db = sqlite3rbu_db(pRbu, 0);
          130  +    rc = sqlite3_exec(db, zPreSql, 0, 0, 0);
          131  +    if( rc==SQLITE_OK ){
          132  +      sqlite3 *db = sqlite3rbu_db(pRbu, 1);
          133  +      rc = sqlite3_exec(db, zPreSql, 0, 0, 0);
          134  +    }
          135  +  }
   114    136   
   115    137     /* If nStep is less than or equal to zero, call
   116    138     ** sqlite3rbu_step() until either the RBU has been completely applied
   117    139     ** or an error occurs. Or, if nStep is greater than zero, call
   118    140     ** sqlite3rbu_step() a maximum of nStep times.  */
   119         -  for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++);
   120         -  nProgress = sqlite3rbu_progress(pRbu);
   121         -  rc = sqlite3rbu_close(pRbu, &zErrmsg);
          141  +  if( rc==SQLITE_OK ){
          142  +    for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++){
          143  +      if( nStatStep>0 && (i % nStatStep)==0 ){
          144  +        sqlite3_int64 nUsed;
          145  +        sqlite3_int64 nHighwater;
          146  +        sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0);
          147  +        fprintf(stdout, "memory used=%lld highwater=%lld", nUsed, nHighwater);
          148  +        if( bVacuum==0 ){
          149  +          int one;
          150  +          int two;
          151  +          sqlite3rbu_bp_progress(pRbu, &one, &two);
          152  +          fprintf(stdout, "  progress=%d/%d\n", one, two);
          153  +        }else{
          154  +          fprintf(stdout, "\n");
          155  +        }
          156  +        fflush(stdout);
          157  +      }
          158  +    }
          159  +    nProgress = sqlite3rbu_progress(pRbu);
          160  +    rc = sqlite3rbu_close(pRbu, &zErrmsg);
          161  +  }
   122    162   
   123    163     /* Let the user know what happened. */
   124    164     switch( rc ){
   125    165       case SQLITE_OK:
   126    166         sqlite3_snprintf(sizeof(zBuf), zBuf,
   127    167             "SQLITE_OK: rbu update incomplete (%lld operations so far)\n",
   128    168             nProgress

Changes to ext/rbu/rbu_common.tcl.

    67     67       rbu close
    68     68       if {$rc != "SQLITE_OK"} break
    69     69     }
    70     70     set rc
    71     71   }
    72     72   
    73     73   proc do_rbu_vacuum_test {tn step} {
           74  +  forcedelete state.db
    74     75     uplevel [list do_test $tn.1 {
    75     76       if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
    76     77       while 1 {
    77     78         if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
    78     79         set state [rbu state]
    79     80         check_prestep_state test.db $state
    80     81         set rc [rbu step]

Added ext/rbu/rbucollate.test.

            1  +# 2018 March 22
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] rbu_common.tcl]
           14  +set ::testprefix rbucollate
           15  +
           16  +ifcapable !icu_collations {
           17  +  finish_test
           18  +  return
           19  +}
           20  +
           21  +db close
           22  +sqlite3_shutdown
           23  +sqlite3_config_uri 1
           24  +reset_db
           25  + 
           26  +# Create a simple RBU database. That expects to write to a table:
           27  +#
           28  +#   CREATE TABLE t1(a PRIMARY KEY, b, c);
           29  +#
           30  +proc create_rbu1 {filename} {
           31  +  forcedelete $filename
           32  +  sqlite3 rbu1 $filename  
           33  +  rbu1 eval {
           34  +    CREATE TABLE data_t1(a, b, c, rbu_control);
           35  +    INSERT INTO data_t1 VALUES('a', 'one', 1, 0);
           36  +    INSERT INTO data_t1 VALUES('b', 'two', 2, 0);
           37  +    INSERT INTO data_t1 VALUES('c', 'three', 3, 0);
           38  +  }
           39  +  rbu1 close
           40  +  return $filename
           41  +}
           42  +
           43  +do_execsql_test 1.0 {
           44  +  SELECT icu_load_collation('en_US', 'my-collate');
           45  +  CREATE TABLE t1(a COLLATE "my-collate" PRIMARY KEY, b, c);
           46  +} {{}}
           47  +
           48  +do_test 1.2 {
           49  +  create_rbu1 testrbu.db
           50  +  sqlite3rbu rbu test.db testrbu.db
           51  +  rbu dbMain_eval { SELECT icu_load_collation('en_US', 'my-collate') }
           52  +  rbu dbRbu_eval { SELECT icu_load_collation('en_US', 'my-collate') }
           53  +  while 1 {
           54  +    set rc [rbu step]
           55  +    if {$rc!="SQLITE_OK"} break
           56  +  }
           57  +  rbu close
           58  +  db eval { SELECT * FROM t1 }
           59  +} {a one 1 b two 2 c three 3}
           60  +
           61  +#forcedelete testrbu.db
           62  +finish_test
           63  +

Changes to ext/rbu/sqlite3rbu.c.

  1802   1802   
  1803   1803       while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
  1804   1804         int bKey = sqlite3_column_int(pXInfo, 5);
  1805   1805         if( bKey ){
  1806   1806           int iCid = sqlite3_column_int(pXInfo, 1);
  1807   1807           int bDesc = sqlite3_column_int(pXInfo, 3);
  1808   1808           const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
  1809         -        zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, 
         1809  +        zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma, 
  1810   1810               iCid, pIter->azTblType[iCid], zCollate
  1811   1811           );
  1812   1812           zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":"");
  1813   1813           zComma = ", ";
  1814   1814         }
  1815   1815       }
  1816   1816       zCols = rbuMPrintf(p, "%z, id INTEGER", zCols);
................................................................................
  1863   1863         );
  1864   1864   
  1865   1865         if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){
  1866   1866           /* If the target table column is an "INTEGER PRIMARY KEY", add
  1867   1867           ** "PRIMARY KEY" to the imposter table column declaration. */
  1868   1868           zPk = "PRIMARY KEY ";
  1869   1869         }
  1870         -      zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", 
         1870  +      zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s", 
  1871   1871             zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl,
  1872   1872             (pIter->abNotNull[iCol] ? " NOT NULL" : "")
  1873   1873         );
  1874   1874         zComma = ", ";
  1875   1875       }
  1876   1876   
  1877   1877       if( pIter->eType==RBU_PK_WITHOUT_ROWID ){

Changes to ext/rbu/test_rbu.c.

    77     77       {"bp_progress", 2, ""},          /* 5 */
    78     78       {"db", 3, "RBU"},                /* 6 */
    79     79       {"state", 2, ""},                /* 7 */
    80     80       {"progress", 2, ""},             /* 8 */
    81     81       {"close_no_error", 2, ""},       /* 9 */
    82     82       {"temp_size_limit", 3, "LIMIT"}, /* 10 */
    83     83       {"temp_size", 2, ""},            /* 11 */
           84  +    {"dbRbu_eval", 3, "SQL"},        /* 12 */
    84     85       {0,0,0}
    85     86     };
    86     87     int iCmd;
    87     88   
    88     89     if( objc<2 ){
    89     90       Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
    90     91       return TCL_ERROR;
................................................................................
   142    143       case 3: /* savestate */ {
   143    144         int rc = sqlite3rbu_savestate(pRbu);
   144    145         Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
   145    146         ret = (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
   146    147         break;
   147    148       }
   148    149   
   149         -    case 4: /* dbMain_eval */ {
   150         -      sqlite3 *db = sqlite3rbu_db(pRbu, 0);
          150  +    case 12: /* dbRbu_eval */ 
          151  +    case 4:  /* dbMain_eval */ {
          152  +      sqlite3 *db = sqlite3rbu_db(pRbu, (iCmd==12));
   151    153         int rc = sqlite3_exec(db, Tcl_GetString(objv[2]), 0, 0, 0);
   152    154         if( rc!=SQLITE_OK ){
   153    155           Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(db), -1));
   154    156           ret = TCL_ERROR;
   155    157         }
   156    158         break;
   157    159       }

Changes to ext/session/session1.test.

   607    607   do_iterator_test $tn.12.2 * {
   608    608     UPDATE t1 SET b='one' WHERE a=1;
   609    609   } {
   610    610     {UPDATE t1 0 X.. {i 1 {} {} i 1} {{} {} {} {} t one}}
   611    611     {UPDATE t1 0 X.. {i 2 {} {} i 2} {{} {} {} {} t one}}
   612    612     {UPDATE t1 0 X.. {i 3 {} {} i 3} {{} {} {} {} t one}}
   613    613   }
          614  +
          615  +#-------------------------------------------------------------------------
          616  +# Test that no savepoint is used if -nosavepoint is specified.
          617  +#
          618  +do_execsql_test $tn.13.1 {
          619  +  CREATE TABLE x1(a INTEGER PRIMARY KEY, b)%WR%;
          620  +}
          621  +do_test $tn.13.2 {
          622  +  execsql BEGIN
          623  +  set C [changeset_from_sql {
          624  +    INSERT INTO x1 VALUES(1, 'one');
          625  +    INSERT INTO x1 VALUES(2, 'two');
          626  +    INSERT INTO x1 VALUES(3, 'three');
          627  +  }]
          628  +  execsql ROLLBACK
          629  +  execsql {
          630  +    INSERT INTO x1 VALUES(1, 'i');
          631  +    INSERT INTO x1 VALUES(2, 'ii');
          632  +    INSERT INTO x1 VALUES(3, 'iii');
          633  +  }
          634  +} {}
          635  +
          636  +proc xConflict {args} {
          637  +  set ret [lindex $::CONFLICT_HANDLERS 0]
          638  +  set ::CONFLICT_HANDLERS [lrange $::CONFLICT_HANDLERS 1 end]
          639  +  set ret
          640  +}
          641  +do_test $tn.13.3 {
          642  +  set CONFLICT_HANDLERS [list REPLACE REPLACE ABORT]
          643  +  execsql BEGIN
          644  +  catch { sqlite3changeset_apply_v2 db $C xConflict } msg
          645  +  execsql {
          646  +    SELECT * FROM x1
          647  +  }
          648  +} {1 i 2 ii 3 iii}
          649  +do_test $tn.13.3 {
          650  +  set CONFLICT_HANDLERS [list REPLACE REPLACE ABORT]
          651  +  execsql ROLLBACK
          652  +  execsql BEGIN
          653  +  catch { sqlite3changeset_apply_v2 -nosavepoint db $C xConflict } msg
          654  +  execsql { SELECT * FROM x1 }
          655  +} {1 one 2 two 3 iii}
          656  +execsql ROLLBACK
   614    657   
   615    658   }]
   616    659   }
   617    660   
   618    661   
   619    662   finish_test

Changes to ext/session/session4.test.

     6      6   #    May you do good and not evil.
     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   # This file implements regression tests for the session module. 
    12     12   # 
           13  +
           14  +package require Tcl 8.6
    13     15   
    14     16   if {![info exists testdir]} {
    15     17     set testdir [file join [file dirname [info script]] .. .. test]
    16     18   } 
    17     19   source [file join [file dirname [info script]] session_common.tcl]
    18     20   source $testdir/tester.tcl
    19     21   ifcapable !session {finish_test; return}

Changes to ext/session/sessionG.test.

   200    200     db2 eval {
   201    201       SELECT * FROM t1;
   202    202       SELECT * FROM t2;
   203    203       SELECT * FROM t3;
   204    204     }
   205    205   } {1 2 3 7 8 9}
   206    206   
          207  +#-------------------------------------------------------------------------
          208  +
          209  +reset_db 
          210  +db func number_name number_name
          211  +do_execsql_test 6.0 {
          212  +  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
          213  +  CREATE UNIQUE INDEX t1b ON t1(b);
          214  +  WITH s(i) AS (
          215  +    SELECT 1
          216  +    UNION ALL
          217  +    SELECT i+1 FROM s WHERE i<1000
          218  +  )
          219  +  INSERT INTO t1 SELECT i, number_name(i) FROM s;
          220  +}
          221  +
          222  +do_test 6.1 {
          223  +  db eval BEGIN
          224  +  set ::C [changeset_from_sql {
          225  +    DELETE FROM t1;
          226  +    WITH s(i) AS (
          227  +        SELECT 1
          228  +        UNION ALL
          229  +        SELECT i+1 FROM s WHERE i<1000
          230  +    )
          231  +    INSERT INTO t1 SELECT i, number_name(i+1) FROM s;
          232  +  }]
          233  +  db eval ROLLBACK
          234  +  execsql { SELECT count(*) FROM t1 WHERE number_name(a) IS NOT b }
          235  +} {0}
          236  +
          237  +proc xConflict {args} { exit ; return "OMIT" }
          238  +do_test 6.2 {
          239  +  sqlite3changeset_apply db $C xConflict
          240  +} {}
          241  +
          242  +do_execsql_test 6.3 { SELECT count(*) FROM t1; } {1000}
          243  +do_execsql_test 6.4 { 
          244  +  SELECT count(*) FROM t1 WHERE number_name(a+1) IS NOT b; 
          245  +} {0}
   207    246   
          247  +# db eval { SELECT * FROM t1 } { puts "$a || $b" }
   208    248   
   209    249   
   210    250   finish_test
   211    251   

Changes to ext/session/session_common.tcl.

   165    165   }
   166    166   
   167    167   proc changeset_to_list {c} {
   168    168     set list [list]
   169    169     sqlite3session_foreach elem $c { lappend list $elem }
   170    170     lsort $list
   171    171   }
          172  +
          173  +set ones {zero one two three four five six seven eight nine
          174  +          ten eleven twelve thirteen fourteen fifteen sixteen seventeen
          175  +          eighteen nineteen}
          176  +set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety}
          177  +proc number_name {n} {
          178  +  if {$n>=1000} {
          179  +    set txt "[number_name [expr {$n/1000}]] thousand"
          180  +    set n [expr {$n%1000}]
          181  +  } else {
          182  +    set txt {}
          183  +  }
          184  +  if {$n>=100} {
          185  +    append txt " [lindex $::ones [expr {$n/100}]] hundred"
          186  +    set n [expr {$n%100}]
          187  +  }
          188  +  if {$n>=20} {
          189  +    append txt " [lindex $::tens [expr {$n/10}]]"
          190  +    set n [expr {$n%10}]
          191  +  }
          192  +  if {$n>0} {
          193  +    append txt " [lindex $::ones $n]"
          194  +  }
          195  +  set txt [string trim $txt]
          196  +  if {$txt==""} {set txt zero}
          197  +  return $txt
          198  +}

Changes to ext/session/sessionfault2.test.

    15     15   if {![info exists testdir]} {
    16     16     set testdir [file join [file dirname [info script]] .. .. test]
    17     17   } 
    18     18   source [file join [file dirname [info script]] session_common.tcl]
    19     19   source $testdir/tester.tcl
    20     20   ifcapable !session {finish_test; return}
    21     21   set testprefix sessionfault2
           22  +
           23  +if 1 {
    22     24   
    23     25   do_execsql_test 1.0.0 {
    24     26     CREATE TABLE t1(a PRIMARY KEY, b UNIQUE);
    25     27     INSERT INTO t1 VALUES(1, 1);
    26     28     INSERT INTO t1 VALUES(2, 2);
    27     29     INSERT INTO t1 VALUES(3, 3);
    28     30   
................................................................................
    98    100     faultsim_restore_and_reopen
    99    101   } -body {
   100    102     sqlite3changeset_apply db $::C xConflict
   101    103   } -test {
   102    104     faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
   103    105     faultsim_integrity_check
   104    106   }
          107  +
          108  +#-------------------------------------------------------------------------
          109  +# OOM when collecting and apply a changeset that uses sqlite_stat1.
          110  +#
          111  +reset_db
          112  +forcedelete test.db2
          113  +sqlite3 db2 test.db2
          114  +do_common_sql {
          115  +  CREATE TABLE t1(a PRIMARY KEY, b UNIQUE, c);
          116  +  CREATE INDEX i1 ON t1(c);
          117  +  INSERT INTO t1 VALUES(1, 2, 3);
          118  +  INSERT INTO t1 VALUES(4, 5, 6);
          119  +  INSERT INTO t1 VALUES(7, 8, 9);
          120  +  CREATE TABLE t2(a, b, c);
          121  +  INSERT INTO t2 VALUES(1, 2, 3);
          122  +  INSERT INTO t2 VALUES(4, 5, 6);
          123  +  INSERT INTO t2 VALUES(7, 8, 9);
          124  +  ANALYZE;
          125  +}
          126  +faultsim_save_and_close
          127  +db2 close
          128  +
          129  +do_faultsim_test 1.1 -faults oom-* -prep {
          130  +  catch {db2 close}
          131  +  catch {db close}
          132  +  faultsim_restore_and_reopen
          133  +  sqlite3 db2 test.db2
          134  +} -body {
          135  +  do_then_apply_sql {
          136  +    INSERT INTO sqlite_stat1 VALUES('x', 'y', 45);
          137  +    UPDATE sqlite_stat1 SET stat = 123 WHERE tbl='t1' AND idx='i1';
          138  +    UPDATE sqlite_stat1 SET stat = 456 WHERE tbl='t2';
          139  +  }
          140  +} -test {
          141  +  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
          142  +  faultsim_integrity_check
          143  +  if {$testrc==0} { compare_db db db2 }
          144  +}
          145  +
          146  +#-------------------------------------------------------------------------
          147  +# OOM when collecting and using a rebase changeset.
          148  +#
          149  +reset_db
          150  +do_execsql_test 2.0 {
          151  +  CREATE TABLE t3(a, b, c, PRIMARY KEY(b, c));
          152  +  CREATE TABLE t4(x PRIMARY KEY, y, z);
          153  +
          154  +  INSERT INTO t3 VALUES(1, 2, 3);
          155  +  INSERT INTO t3 VALUES(4, 2, 5);
          156  +  INSERT INTO t3 VALUES(7, 2, 9);
          157  +
          158  +  INSERT INTO t4 VALUES('a', 'b', 'c');
          159  +  INSERT INTO t4 VALUES('d', 'e', 'f');
          160  +  INSERT INTO t4 VALUES('g', 'h', 'i');
          161  +}
          162  +faultsim_save_and_close
          163  +db2 close
          164  +
          165  +proc xConflict {ret args} { return $ret }
          166  +
          167  +do_test 2.1 {
          168  +  faultsim_restore_and_reopen
          169  +  set C1 [changeset_from_sql {
          170  +    INSERT INTO t3 VALUES(10, 11, 12);
          171  +    UPDATE t4 SET y='j' WHERE x='g';
          172  +    DELETE FROM t4 WHERE x='a';
          173  +  }]
          174  +
          175  +  faultsim_restore_and_reopen
          176  +  set C2 [changeset_from_sql {
          177  +    INSERT INTO t3 VALUES(1000, 11, 12);
          178  +    DELETE FROM t4 WHERE x='g';
          179  +  }]
          180  +
          181  +  faultsim_restore_and_reopen
          182  +  sqlite3changeset_apply db $C1 [list xConflict OMIT]
          183  +  faultsim_save_and_close
          184  +} {}
          185  +
          186  +do_faultsim_test 2.2 -faults oom* -prep {
          187  +  catch {db2 close}
          188  +  catch {db close}
          189  +  faultsim_restore_and_reopen
          190  +  sqlite3 db2 test.db2
          191  +} -body {
          192  +  set rebase [sqlite3changeset_apply_v2 db $::C2 [list xConflict OMIT]]
          193  +  set {} {}
          194  +} -test {
          195  +  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
          196  +}
          197  +do_faultsim_test 2.3 -faults oom* -prep {
          198  +  catch {db2 close}
          199  +  catch {db close}
          200  +  faultsim_restore_and_reopen
          201  +  sqlite3 db2 test.db2
          202  +} -body {
          203  +  set rebase [sqlite3changeset_apply_v2 db $::C2 [list xConflict REPLACE]]
          204  +  set {} {}
          205  +} -test {
          206  +  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
          207  +}
          208  +do_faultsim_test 2.4 -faults oom* -prep {
          209  +  catch {db2 close}
          210  +  catch {db close}
          211  +  faultsim_restore_and_reopen
          212  +  set ::rebase [sqlite3changeset_apply_v2 db $::C2 [list xConflict REPLACE]]
          213  +} -body {
          214  +  sqlite3rebaser_create R
          215  +  R configure $::rebase
          216  +  R rebase $::C1
          217  +  set {} {}
          218  +} -test {
          219  +  catch { R delete } 
          220  +  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
          221  +}
          222  +do_faultsim_test 2.5 -faults oom* -prep {
          223  +  catch {db2 close}
          224  +  catch {db close}
          225  +  faultsim_restore_and_reopen
          226  +  set ::rebase [sqlite3changeset_apply_v2 db $::C2 [list xConflict OMIT]]
          227  +} -body {
          228  +  sqlite3rebaser_create R
          229  +  R configure $::rebase
          230  +  R rebase $::C1
          231  +  set {} {}
          232  +} -test {
          233  +  catch { R delete } 
          234  +  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
          235  +}
          236  +
          237  +}
          238  +
          239  +reset_db
          240  +do_execsql_test 3.0 {
          241  +  CREATE TABLE t1(x PRIMARY KEY, y, z);
          242  +  INSERT INTO t1 VALUES(3, 1, 4);
          243  +  INSERT INTO t1 VALUES(1, 5, 9);
          244  +}
          245  +faultsim_save_and_close
          246  +
          247  +proc xConflict {ret args} { return $ret }
          248  +
          249  +do_test 3.1 {
          250  +  faultsim_restore_and_reopen
          251  +
          252  +  execsql { BEGIN; UPDATE t1 SET z=11; }
          253  +  set C1 [changeset_from_sql {
          254  +    UPDATE t1 SET z=10 WHERE x=1;
          255  +  }]
          256  +  execsql { ROLLBACK }
          257  +
          258  +  execsql { BEGIN; UPDATE t1 SET z=11; }
          259  +  set C2 [changeset_from_sql {
          260  +    UPDATE t1 SET z=55 WHERE x=1;
          261  +  }]
          262  +  execsql { ROLLBACK }
          263  +
          264  +  set ::rebase1 [sqlite3changeset_apply_v2 db $::C1 [list xConflict OMIT]]
          265  +  set ::rebase2 [sqlite3changeset_apply_v2 db $::C2 [list xConflict OMIT]]
          266  +  set {} {}
          267  +  execsql { SELECT * FROM t1 }
          268  +} {3 1 4 1 5 9}
          269  +
          270  +
          271  +do_faultsim_test 3.2 -faults oom* -prep {
          272  +  faultsim_restore_and_reopen
          273  +} -body {
          274  +  sqlite3rebaser_create R
          275  +  R configure $::rebase1
          276  +  R configure $::rebase2
          277  +  set {} {}
          278  +} -test {
          279  +  catch { R delete } 
          280  +  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
          281  +}
          282  +
   105    283   
   106    284   finish_test
   107    285   

Added ext/session/sessionrebase.test.

            1  +# 2018 March 14
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.
           12  +#
           13  +
           14  +if {![info exists testdir]} {
           15  +  set testdir [file join [file dirname [info script]] .. .. test]
           16  +} 
           17  +source [file join [file dirname [info script]] session_common.tcl]
           18  +source $testdir/tester.tcl
           19  +ifcapable !session {finish_test; return}
           20  +
           21  +set testprefix sessionrebase
           22  +
           23  +set ::lConflict [list]
           24  +proc xConflict {args} {
           25  +  set res [lindex $::lConflict 0]
           26  +  set ::lConflict [lrange $::lConflict 1 end]
           27  +  return $res
           28  +}
           29  +
           30  +#-------------------------------------------------------------------------
           31  +# The following test cases - 1.* - test that the rebase blobs output by
           32  +# sqlite3_changeset_apply_v2 look correct in some simple cases. The blob
           33  +# is itself a changeset, containing records determined as follows:
           34  +#
           35  +#   * For each conflict resolved with REPLACE, the rebase blob contains
           36  +#     a DELETE record. All fields other than the PK fields are undefined.
           37  +#
           38  +#   * For each conflict resolved with OMIT, the rebase blob contains an
           39  +#     INSERT record. For an INSERT or UPDATE operation, the indirect flag
           40  +#     is clear and all updated fields are defined. For a DELETE operation,
           41  +#     the indirect flag is set and all non-PK fields left undefined.
           42  +#
           43  +proc do_apply_v2_test {tn sql modsql conflict_handler res} {
           44  +  
           45  +  execsql BEGIN
           46  +  sqlite3session S db main
           47  +  S attach *
           48  +  execsql $sql
           49  +  set changeset [S changeset]
           50  +  S delete
           51  +  execsql ROLLBACK
           52  +
           53  +  execsql BEGIN
           54  +  execsql $modsql
           55  +  set ::lConflict $conflict_handler
           56  +  set blob [sqlite3changeset_apply_v2 db $changeset xConflict]
           57  +  execsql ROLLBACK
           58  +
           59  +  uplevel [list do_test $tn [list changeset_to_list $blob] [list {*}$res]]
           60  +}
           61  +
           62  +
           63  +set ::lConflict [list]
           64  +proc xConflict {args} {
           65  +  set res [lindex $::lConflict 0]
           66  +  set ::lConflict [lrange $::lConflict 1 end]
           67  +  return $res
           68  +}
           69  +
           70  +# Take a copy of database test.db in file test.db2. Execute $sql1
           71  +# against test.db and $sql2 against test.db2. Capture a changeset
           72  +# for each. Then send the test.db2 changeset to test.db and apply
           73  +# it with the conflict handlers in $conflict_handler. Patch the
           74  +# test.db changeset and then execute it against test.db2. Test that
           75  +# the two databases come out the same.
           76  +#
           77  +proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} {
           78  +
           79  +  for {set i 1} {$i <= 2} {incr i} {
           80  +    forcedelete test.db2 test.db2-journal test.db2-wal
           81  +    forcecopy test.db test.db2
           82  +    sqlite3 db2 test.db2
           83  +
           84  +    db eval BEGIN
           85  +
           86  +    sqlite3session S1 db main
           87  +    S1 attach *
           88  +    execsql $sql1 db
           89  +    set c1 [S1 changeset]
           90  +    S1 delete
           91  +
           92  +    if {$i==1} {
           93  +      sqlite3session S2 db2 main
           94  +      S2 attach *
           95  +      execsql $sql2 db2
           96  +      set c2 [S2 changeset]
           97  +      S2 delete
           98  +    } else {
           99  +      set c2 [list]
          100  +      foreach sql [split $sql2 ";"] {
          101  +        if {[string is space $sql]} continue
          102  +        sqlite3session S2 db2 main
          103  +        S2 attach *
          104  +        execsql $sql db2
          105  +        lappend c2 [S2 changeset]
          106  +        S2 delete
          107  +      }
          108  +    }
          109  +
          110  +    set ::lConflict $conflict_handler
          111  +    set rebase [list]
          112  +    if {$i==1} {
          113  +      lappend rebase [sqlite3changeset_apply_v2 db $c2 xConflict]
          114  +    } else {
          115  +      foreach c $c2 {
          116  +#puts "apply_v2: [changeset_to_list $c]"
          117  +        lappend rebase [sqlite3changeset_apply_v2 db $c xConflict]
          118  +      }
          119  +      #puts "llength: [llength $rebase]"
          120  +    }
          121  +    #if {$tn=="2.1.4"} { puts [changeset_to_list $rebase] ; breakpoint }
          122  +    #puts [changeset_to_list [lindex $rebase 0]] ; breakpoint
          123  +    #puts [llength $rebase]
          124  +  
          125  +    sqlite3rebaser_create R
          126  +    foreach r $rebase {
          127  +#puts [changeset_to_list $r]
          128  +      R configure $r
          129  +    }
          130  +    set c1r [R rebase $c1]
          131  +    R delete
          132  +    #if {$tn=="2.1.4"} { puts [changeset_to_list $c1r] }
          133  +  
          134  +    sqlite3changeset_apply_v2 db2 $c1r xConflictAbort
          135  +  
          136  +    if {[string range $tn end end]!="*"} {
          137  +      uplevel [list do_test $tn.$i.1 [list compare_db db db2] {}]
          138  +    }
          139  +    db2 close
          140  +  
          141  +    if {$testsql!=""} {
          142  +      uplevel [list do_execsql_test $tn.$i.2 $testsql $testres]
          143  +    }
          144  +  
          145  +    db eval ROLLBACK
          146  +  }
          147  +}
          148  +
          149  +do_execsql_test 1.0 {
          150  +  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
          151  +  INSERT INTO t1 VALUES(1, 'value A');
          152  +}
          153  +
          154  +do_apply_v2_test 1.1.1 {
          155  +  UPDATE t1 SET b = 'value B' WHERE a=1;
          156  +} {
          157  +  UPDATE t1 SET b = 'value C' WHERE a=1;
          158  +} {
          159  +  OMIT
          160  +} {
          161  +  {INSERT t1 0 X. {} {i 1 t {value B}}}
          162  +}
          163  +
          164  +do_apply_v2_test 1.1.2 {
          165  +  UPDATE t1 SET b = 'value B' WHERE a=1;
          166  +} {
          167  +  UPDATE t1 SET b = 'value C' WHERE a=1;
          168  +} {
          169  +  REPLACE
          170  +} {
          171  +  {INSERT t1 1 X. {} {i 1 t {value B}}}
          172  +}
          173  +
          174  +do_apply_v2_test 1.2.1 {
          175  +  INSERT INTO t1 VALUES(2, 'first');
          176  +} {
          177  +  INSERT INTO t1 VALUES(2, 'second');
          178  +} {
          179  +  OMIT
          180  +} {
          181  +  {INSERT t1 0 X. {} {i 2 t first}}
          182  +}
          183  +do_apply_v2_test 1.2.2 {
          184  +  INSERT INTO t1 VALUES(2, 'first');
          185  +} {
          186  +  INSERT INTO t1 VALUES(2, 'second');
          187  +} {
          188  +  REPLACE
          189  +} {
          190  +  {INSERT t1 1 X. {} {i 2 t first}}
          191  +}
          192  +
          193  +do_apply_v2_test 1.3.1 {
          194  +  DELETE FROM t1 WHERE a=1;
          195  +} {
          196  +  UPDATE t1 SET b='value D' WHERE a=1;
          197  +} {
          198  +  OMIT
          199  +} {
          200  +  {DELETE t1 0 X. {i 1 t {value A}} {}}
          201  +}
          202  +do_apply_v2_test 1.3.2 {
          203  +  DELETE FROM t1 WHERE a=1;
          204  +} {
          205  +  UPDATE t1 SET b='value D' WHERE a=1;
          206  +} {
          207  +  REPLACE
          208  +} {
          209  +  {DELETE t1 1 X. {i 1 t {value A}} {}}
          210  +}
          211  +
          212  +#-------------------------------------------------------------------------
          213  +# Test cases 2.* - simple tests of rebasing actual changesets.
          214  +#
          215  +#    2.1.1 - 1u2u1r
          216  +#    2.1.2 - 1u2u2r
          217  +#    2.1.3 - 1d2d
          218  +#    2.1.4 - 1d2u1r
          219  +#    2.1.5 - 1d2u2r !!
          220  +#    2.1.6 - 1u2d1r
          221  +#    2.1.7 - 1u2d2r
          222  +#
          223  +#    2.1.8 - 1i2i2r
          224  +#    2.1.9 - 1i2i1r
          225  +#
          226  +
          227  +proc xConflictAbort {args} {
          228  +  return "ABORT"
          229  +}
          230  +
          231  +reset_db
          232  +do_execsql_test 2.1.0 {
          233  +  CREATE TABLE t1 (a INTEGER PRIMARY KEY, b TEXT);
          234  +  INSERT INTO t1 VALUES(1, 'one');
          235  +  INSERT INTO t1 VALUES(2, 'two');
          236  +  INSERT INTO t1 VALUES(3, 'three');
          237  +}
          238  +do_rebase_test 2.1.1 {
          239  +  UPDATE t1 SET b = 'two.1' WHERE a=2
          240  +} {
          241  +  UPDATE t1 SET b = 'two.2' WHERE a=2;
          242  +} {
          243  +  OMIT
          244  +} { SELECT * FROM t1 } {1 one 2 two.1 3 three}
          245  +
          246  +do_rebase_test 2.1.2 {
          247  +  UPDATE t1 SET b = 'two.1' WHERE a=2
          248  +} {
          249  +  UPDATE t1 SET b = 'two.2' WHERE a=2;
          250  +} {
          251  +  REPLACE
          252  +} { SELECT * FROM t1 } {1 one 2 two.2 3 three}
          253  +
          254  +do_rebase_test 2.1.3 {
          255  +  DELETE FROM t1 WHERE a=3
          256  +} {
          257  +  DELETE FROM t1 WHERE a=3;
          258  +} {
          259  +  OMIT
          260  +} { SELECT * FROM t1 } {1 one 2 two}
          261  +
          262  +do_rebase_test 2.1.4 {
          263  +  DELETE FROM t1 WHERE a=1
          264  +} {
          265  +  UPDATE t1 SET b='one.2' WHERE a=1
          266  +} {
          267  +  OMIT
          268  +} { SELECT * FROM t1 } {2 two 3 three}
          269  +
          270  +#do_rebase_test 2.1.5 {
          271  +#  DELETE FROM t1 WHERE a=1;
          272  +#} {
          273  +#  UPDATE t1 SET b='one.2' WHERE a=1
          274  +#} {
          275  +#  REPLACE
          276  +#} { SELECT * FROM t1 } {2 two 3 three}
          277  +
          278  +do_rebase_test 2.1.6 {
          279  +  UPDATE t1 SET b='three.1' WHERE a=3
          280  +} {
          281  +  DELETE FROM t1 WHERE a=3;
          282  +} {
          283  +  OMIT
          284  +} { SELECT * FROM t1 } {1 one 2 two 3 three.1}
          285  +
          286  +do_rebase_test 2.1.7 {
          287  +  UPDATE t1 SET b='three.1' WHERE a=3
          288  +} {
          289  +  DELETE FROM t1 WHERE a=3;
          290  +} {
          291  +  REPLACE
          292  +} { SELECT * FROM t1 } {1 one 2 two}
          293  +
          294  +do_rebase_test 2.1.8 {
          295  +  INSERT INTO t1 VALUES(4, 'four.1')
          296  +} {
          297  +  INSERT INTO t1 VALUES(4, 'four.2');
          298  +} {
          299  +  REPLACE
          300  +} { SELECT * FROM t1 } {1 one 2 two 3 three 4 four.2}
          301  +
          302  +do_rebase_test 2.1.9 {
          303  +  INSERT INTO t1 VALUES(4, 'four.1')
          304  +} {
          305  +  INSERT INTO t1 VALUES(4, 'four.2');
          306  +} {
          307  +  OMIT
          308  +} { SELECT * FROM t1 } {1 one 2 two 3 three 4 four.1}
          309  +
          310  +do_execsql_test 2.2.0 {
          311  +  CREATE TABLE t2(x, y, z PRIMARY KEY);
          312  +  INSERT INTO t2 VALUES('i', 'a', 'A');
          313  +  INSERT INTO t2 VALUES('ii', 'b', 'B');
          314  +  INSERT INTO t2 VALUES('iii', 'c', 'C');
          315  +
          316  +  CREATE TABLE t3(a INTEGER PRIMARY KEY, b, c);
          317  +  INSERT INTO t3 VALUES(-1, 'z', 'Z');
          318  +  INSERT INTO t3 VALUES(-2, 'y', 'Y');
          319  +}
          320  +
          321  +do_rebase_test 2.2.1 {
          322  +  UPDATE t2 SET x=1 WHERE z='A'
          323  +} {
          324  +  UPDATE t2 SET y='one' WHERE z='A';
          325  +} {
          326  +} { SELECT * FROM t2 WHERE z='A' } { 1 one A }
          327  +
          328  +do_rebase_test 2.2.2 {
          329  +  UPDATE t2 SET x=1, y='one' WHERE z='B'
          330  +} {
          331  +  UPDATE t2 SET y='two' WHERE z='B';
          332  +} {
          333  +  REPLACE
          334  +} { SELECT * FROM t2 WHERE z='B' } { 1 two B }
          335  +
          336  +do_rebase_test 2.2.3 {
          337  +  UPDATE t2 SET x=1, y='one' WHERE z='B'
          338  +} {
          339  +  UPDATE t2 SET y='two' WHERE z='B';
          340  +} {
          341  +  OMIT
          342  +} { SELECT * FROM t2 WHERE z='B' } { 1 one B }
          343  +
          344  +#-------------------------------------------------------------------------
          345  +reset_db
          346  +do_execsql_test 3.0 {
          347  +  CREATE TABLE t3(a, b, c, PRIMARY KEY(b, c));
          348  +  CREATE TABLE abcdefghijkl(x PRIMARY KEY, y, z);
          349  +
          350  +  INSERT INTO t3 VALUES(1, 2, 3);
          351  +  INSERT INTO t3 VALUES(4, 2, 5);
          352  +  INSERT INTO t3 VALUES(7, 2, 9);
          353  +
          354  +  INSERT INTO abcdefghijkl VALUES('a', 'b', 'c');
          355  +  INSERT INTO abcdefghijkl VALUES('d', 'e', 'f');
          356  +  INSERT INTO abcdefghijkl VALUES('g', 'h', 'i');
          357  +}
          358  +
          359  +breakpoint
          360  +#  do_rebase_test 3.6.tn {
          361  +#    UPDATE abcdefghijkl SET z='X', y='X' WHERE x='d';
          362  +#  } {
          363  +#    UPDATE abcdefghijkl SET y=1 WHERE x='d';
          364  +#    UPDATE abcdefghijkl SET z=1 WHERE x='d';
          365  +#  } [list REPLACE REPLACE REPLACE]
          366  +
          367  +foreach {tn p} {
          368  +    1 OMIT 2 REPLACE
          369  +} {
          370  +  do_rebase_test 3.1.$tn {
          371  +    INSERT INTO t3 VALUES(1, 1, 1);
          372  +    UPDATE abcdefghijkl SET y=2;
          373  +  } {
          374  +    INSERT INTO t3 VALUES(4, 1, 1);
          375  +    DELETE FROM abcdefghijkl;
          376  +  } [list $p $p $p $p $p $p $p $p]
          377  +
          378  +  do_rebase_test 3.2.$tn {
          379  +    INSERT INTO abcdefghijkl SELECT * FROM t3;
          380  +    UPDATE t3 SET b=b+1;
          381  +  } {
          382  +    INSERT INTO t3 VALUES(3, 3, 3);
          383  +    INSERT INTO abcdefghijkl SELECT * FROM t3;
          384  +  } [list $p $p $p $p $p $p $p $p]
          385  +
          386  +  do_rebase_test 3.3.$tn {
          387  +    INSERT INTO abcdefghijkl VALUES(22, 23, 24);
          388  +  } {
          389  +    INSERT INTO abcdefghijkl VALUES(22, 25, 26);
          390  +    UPDATE abcdefghijkl SET y=400 WHERE x=22;
          391  +  } [list $p $p $p $p $p $p $p $p]
          392  +
          393  +  do_rebase_test 3.4.$tn {
          394  +    INSERT INTO abcdefghijkl VALUES(22, 23, 24);
          395  +  } {
          396  +    INSERT INTO abcdefghijkl VALUES(22, 25, 26);
          397  +    UPDATE abcdefghijkl SET y=400 WHERE x=22;
          398  +  } [list REPLACE $p]
          399  +
          400  +  do_rebase_test 3.5.$tn* {
          401  +    UPDATE abcdefghijkl SET y='X' WHERE x='d';
          402  +  } {
          403  +    DELETE FROM abcdefghijkl WHERE x='d';
          404  +    INSERT INTO abcdefghijkl VALUES('d', NULL, NULL);
          405  +  } [list $p $p $p]
          406  +  do_rebase_test 3.5.$tn {
          407  +    UPDATE abcdefghijkl SET y='X' WHERE x='d';
          408  +  } {
          409  +    DELETE FROM abcdefghijkl WHERE x='d';
          410  +    INSERT INTO abcdefghijkl VALUES('d', NULL, NULL);
          411  +  } [list REPLACE $p $p]
          412  +
          413  +  do_rebase_test 3.6.$tn {
          414  +    UPDATE abcdefghijkl SET z='X', y='X' WHERE x='d';
          415  +  } {
          416  +    UPDATE abcdefghijkl SET y=1 WHERE x='d';
          417  +    UPDATE abcdefghijkl SET z=1 WHERE x='d';
          418  +  } [list REPLACE $p $p]
          419  +}
          420  +
          421  +#-------------------------------------------------------------------------
          422  +# Check that apply_v2() does not create a rebase buffer for a patchset.
          423  +# And that it is not possible to rebase a patchset.
          424  +#
          425  +do_execsql_test 4.0 {
          426  +  CREATE TABLE t5(o PRIMARY KEY, p, q);
          427  +  INSERT INTO t5 VALUES(1, 2, 3);
          428  +  INSERT INTO t5 VALUES(4, 5, 6);
          429  +}
          430  +foreach {tn cmd rebasable} {
          431  +  1 patchset 0
          432  +  2 changeset 1
          433  +} {
          434  +  proc xConflict {args} { return "OMIT" }
          435  +  do_test 4.1.$tn {
          436  +    execsql {
          437  +      BEGIN;
          438  +      DELETE FROM t5 WHERE o=4;
          439  +    }
          440  +
          441  +    sqlite3session S db main
          442  +    S attach *
          443  +    execsql {
          444  +      INSERT INTO t5 VALUES(4, 'five', 'six');
          445  +    }
          446  +    set P [S $cmd]
          447  +    S delete
          448  +
          449  +    execsql ROLLBACK;
          450  +
          451  +    set ::rebase [sqlite3changeset_apply_v2 db $P xConflict]
          452  +    expr [llength $::rebase]>0
          453  +  } $rebasable
          454  +}
          455  +
          456  +foreach {tn cmd rebasable} {
          457  +  1 patchset 0
          458  +  2 changeset 1
          459  +} {
          460  +  do_test 4.2.$tn {
          461  +    sqlite3session S db main
          462  +    S attach *
          463  +    execsql {
          464  +      INSERT INTO t5 VALUES(5+$tn, 'five', 'six');
          465  +    }
          466  +    set P [S $cmd]
          467  +    S delete
          468  +
          469  +    sqlite3rebaser_create R
          470  +    R configure $::rebase
          471  +    expr [catch {R rebase $P}]==0
          472  +  } $rebasable
          473  +
          474  +  catch { R delete }
          475  +}
          476  +finish_test
          477  +

Changes to ext/session/sqlite3session.c.

    71     71   /*
    72     72   ** An object of this type is used internally as an abstraction for 
    73     73   ** input data. Input data may be supplied either as a single large buffer
    74     74   ** (e.g. sqlite3changeset_start()) or using a stream function (e.g.
    75     75   **  sqlite3changeset_start_strm()).
    76     76   */
    77     77   struct SessionInput {
    78         -  int bNoDiscard;                 /* If true, discard no data */
           78  +  int bNoDiscard;                 /* If true, do not discard in InputBuffer() */
    79     79     int iCurrent;                   /* Offset in aData[] of current change */
    80     80     int iNext;                      /* Offset in aData[] of next change */
    81     81     u8 *aData;                      /* Pointer to buffer containing changeset */
    82     82     int nData;                      /* Number of bytes in aData */
    83     83   
    84     84     SessionBuffer buf;              /* Current read buffer */
    85     85     int (*xInput)(void*, void*, int*);        /* Input stream call (or NULL) */
................................................................................
   235    235   **
   236    236   ** As in the changeset format, each field of the single record that is part
   237    237   ** of a patchset change is associated with the correspondingly positioned
   238    238   ** table column, counting from left to right within the CREATE TABLE 
   239    239   ** statement.
   240    240   **
   241    241   ** For a DELETE change, all fields within the record except those associated
   242         -** with PRIMARY KEY columns are set to "undefined". The PRIMARY KEY fields
   243         -** contain the values identifying the row to delete.
          242  +** with PRIMARY KEY columns are omitted. The PRIMARY KEY fields contain the
          243  +** values identifying the row to delete.
   244    244   **
   245    245   ** For an UPDATE change, all fields except those associated with PRIMARY KEY
   246    246   ** columns and columns that are modified by the UPDATE are set to "undefined".
   247    247   ** PRIMARY KEY fields contain the values identifying the table row to update,
   248    248   ** and fields associated with modified columns contain the new column values.
   249    249   **
   250    250   ** The records associated with INSERT changes are in the same format as for
................................................................................
   519    519   ** The buffer that the argument points to contains a serialized SQL value.
   520    520   ** Return the number of bytes of space occupied by the value (including
   521    521   ** the type byte).
   522    522   */
   523    523   static int sessionSerialLen(u8 *a){
   524    524     int e = *a;
   525    525     int n;
   526         -  if( e==0 ) return 1;
          526  +  if( e==0 || e==0xFF ) return 1;
   527    527     if( e==SQLITE_NULL ) return 1;
   528    528     if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9;
   529    529     return sessionVarintGet(&a[1], &n) + 1 + n;
   530    530   }
   531    531   
   532    532   /*
   533    533   ** Based on the primary key values stored in change aRecord, calculate a
................................................................................
   599    599     int iCol;                       /* Used to iterate through table columns */
   600    600   
   601    601     for(iCol=0; iCol<pTab->nCol; iCol++){
   602    602       if( pTab->abPK[iCol] ){
   603    603         int n1 = sessionSerialLen(a1);
   604    604         int n2 = sessionSerialLen(a2);
   605    605   
   606         -      if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){
          606  +      if( n1!=n2 || memcmp(a1, a2, n1) ){
   607    607           return 0;
   608    608         }
   609    609         a1 += n1;
   610    610         a2 += n2;
   611    611       }else{
   612    612         if( bLeftPkOnly==0 ) a1 += sessionSerialLen(a1);
   613    613         if( bRightPkOnly==0 ) a2 += sessionSerialLen(a2);
................................................................................
   842    842           a += sessionVarintGet(a, &n);
   843    843           if( sqlite3_value_bytes(pVal)!=n ) return 0;
   844    844           if( eType==SQLITE_TEXT ){
   845    845             z = sqlite3_value_text(pVal);
   846    846           }else{
   847    847             z = sqlite3_value_blob(pVal);
   848    848           }
   849         -        if( memcmp(a, z, n) ) return 0;
          849  +        if( n>0 && memcmp(a, z, n) ) return 0;
   850    850           a += n;
   851    851         }
   852    852       }
   853    853     }
   854    854   
   855    855     return 1;
   856    856   }
................................................................................
  2186   2186     int nSql = -1;
  2187   2187   
  2188   2188     if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){
  2189   2189       zSql = sqlite3_mprintf(
  2190   2190           "SELECT tbl, ?2, stat FROM %Q.sqlite_stat1 WHERE tbl IS ?1 AND "
  2191   2191           "idx IS (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", zDb
  2192   2192       );
         2193  +    if( zSql==0 ) rc = SQLITE_NOMEM;
  2193   2194     }else{
  2194   2195       int i;
  2195   2196       const char *zSep = "";
  2196   2197       SessionBuffer buf = {0, 0, 0};
  2197   2198   
  2198   2199       sessionAppendStr(&buf, "SELECT * FROM ", &rc);
  2199   2200       sessionAppendIdent(&buf, zDb, &rc);
................................................................................
  2609   2610   }
  2610   2611   
  2611   2612   /*
  2612   2613   ** If the SessionInput object passed as the only argument is a streaming
  2613   2614   ** object and the buffer is full, discard some data to free up space.
  2614   2615   */
  2615   2616   static void sessionDiscardData(SessionInput *pIn){
  2616         -  if( pIn->bEof && pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){
         2617  +  if( pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){
  2617   2618       int nMove = pIn->buf.nBuf - pIn->iNext;
  2618   2619       assert( nMove>=0 );
  2619   2620       if( nMove>0 ){
  2620   2621         memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove);
  2621   2622       }
  2622   2623       pIn->buf.nBuf -= pIn->iNext;
  2623   2624       pIn->iNext = 0;
................................................................................
  2935   2936   ** successfully advanced to the next change in the changeset, an SQLite 
  2936   2937   ** error code if an error occurs, or SQLITE_DONE if there are no further 
  2937   2938   ** changes in the changeset.
  2938   2939   */
  2939   2940   static int sessionChangesetNext(
  2940   2941     sqlite3_changeset_iter *p,      /* Changeset iterator */
  2941   2942     u8 **paRec,                     /* If non-NULL, store record pointer here */
  2942         -  int *pnRec                      /* If non-NULL, store size of record here */
         2943  +  int *pnRec,                     /* If non-NULL, store size of record here */
         2944  +  int *pbNew                      /* If non-NULL, true if new table */
  2943   2945   ){
  2944   2946     int i;
  2945   2947     u8 op;
  2946   2948   
  2947   2949     assert( (paRec==0 && pnRec==0) || (paRec && pnRec) );
  2948   2950   
  2949   2951     /* If the iterator is in the error-state, return immediately. */
................................................................................
  2970   2972     }
  2971   2973   
  2972   2974     sessionDiscardData(&p->in);
  2973   2975     p->in.iCurrent = p->in.iNext;
  2974   2976   
  2975   2977     op = p->in.aData[p->in.iNext++];
  2976   2978     while( op=='T' || op=='P' ){
         2979  +    if( pbNew ) *pbNew = 1;
  2977   2980       p->bPatchset = (op=='P');
  2978   2981       if( sessionChangesetReadTblhdr(p) ) return p->rc;
  2979   2982       if( (p->rc = sessionInputBuffer(&p->in, 2)) ) return p->rc;
  2980   2983       p->in.iCurrent = p->in.iNext;
  2981   2984       if( p->in.iNext>=p->in.nData ) return SQLITE_DONE;
  2982   2985       op = p->in.aData[p->in.iNext++];
  2983   2986     }
................................................................................
  3048   3051   ** change in the changeset. This function may return SQLITE_ROW, SQLITE_DONE
  3049   3052   ** or SQLITE_CORRUPT.
  3050   3053   **
  3051   3054   ** This function may not be called on iterators passed to a conflict handler
  3052   3055   ** callback by changeset_apply().
  3053   3056   */
  3054   3057   int sqlite3changeset_next(sqlite3_changeset_iter *p){
  3055         -  return sessionChangesetNext(p, 0, 0);
         3058  +  return sessionChangesetNext(p, 0, 0, 0);
  3056   3059   }
  3057   3060   
  3058   3061   /*
  3059   3062   ** The following function extracts information on the current change
  3060   3063   ** from a changeset iterator. It may only be called after changeset_next()
  3061   3064   ** has returned SQLITE_ROW.
  3062   3065   */
................................................................................
  3427   3430     sqlite3_stmt *pSelect;          /* SELECT statement */
  3428   3431     int nCol;                       /* Size of azCol[] and abPK[] arrays */
  3429   3432     const char **azCol;             /* Array of column names */
  3430   3433     u8 *abPK;                       /* Boolean array - true if column is in PK */
  3431   3434     int bStat1;                     /* True if table is sqlite_stat1 */
  3432   3435     int bDeferConstraints;          /* True to defer constraints */
  3433   3436     SessionBuffer constraints;      /* Deferred constraints are stored here */
         3437  +  SessionBuffer rebase;           /* Rebase information (if any) here */
         3438  +  int bRebaseStarted;             /* If table header is already in rebase */
  3434   3439   };
  3435   3440   
  3436   3441   /*
  3437   3442   ** Formulate a statement to DELETE a row from database db. Assuming a table
  3438   3443   ** structure like this:
  3439   3444   **
  3440   3445   **     CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c));
................................................................................
  3693   3698     if( rc==SQLITE_OK ){
  3694   3699       rc = sessionPrepare(db, &p->pDelete,
  3695   3700           "DELETE FROM main.sqlite_stat1 WHERE tbl=?1 AND idx IS "
  3696   3701           "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END "
  3697   3702           "AND (?4 OR stat IS ?3)"
  3698   3703       );
  3699   3704     }
  3700         -  assert( rc==SQLITE_OK );
  3701   3705     return rc;
  3702   3706   }
  3703   3707   
  3704   3708   /*
  3705   3709   ** A wrapper around sqlite3_bind_value() that detects an extra problem. 
  3706   3710   ** See comments in the body of this function for details.
  3707   3711   */
................................................................................
  3807   3811     if( rc==SQLITE_OK ){
  3808   3812       rc = sqlite3_step(pSelect);
  3809   3813       if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect);
  3810   3814     }
  3811   3815   
  3812   3816     return rc;
  3813   3817   }
         3818  +
         3819  +/*
         3820  +** This function is called from within sqlite3changset_apply_v2() when
         3821  +** a conflict is encountered and resolved using conflict resolution
         3822  +** mode eType (either SQLITE_CHANGESET_OMIT or SQLITE_CHANGESET_REPLACE)..
         3823  +** It adds a conflict resolution record to the buffer in 
         3824  +** SessionApplyCtx.rebase, which will eventually be returned to the caller
         3825  +** of apply_v2() as the "rebase" buffer.
         3826  +**
         3827  +** Return SQLITE_OK if successful, or an SQLite error code otherwise.
         3828  +*/
         3829  +static int sessionRebaseAdd(
         3830  +  SessionApplyCtx *p,             /* Apply context */
         3831  +  int eType,                      /* Conflict resolution (OMIT or REPLACE) */
         3832  +  sqlite3_changeset_iter *pIter   /* Iterator pointing at current change */
         3833  +){
         3834  +  int rc = SQLITE_OK;
         3835  +  int i;
         3836  +  int eOp = pIter->op;
         3837  +  if( p->bRebaseStarted==0 ){
         3838  +    /* Append a table-header to the rebase buffer */
         3839  +    const char *zTab = pIter->zTab;
         3840  +    sessionAppendByte(&p->rebase, 'T', &rc);
         3841  +    sessionAppendVarint(&p->rebase, p->nCol, &rc);
         3842  +    sessionAppendBlob(&p->rebase, p->abPK, p->nCol, &rc);
         3843  +    sessionAppendBlob(&p->rebase, (u8*)zTab, (int)strlen(zTab)+1, &rc);
         3844  +    p->bRebaseStarted = 1;
         3845  +  }
         3846  +
         3847  +  assert( eType==SQLITE_CHANGESET_REPLACE||eType==SQLITE_CHANGESET_OMIT );
         3848  +  assert( eOp==SQLITE_DELETE || eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE );
         3849  +
         3850  +  sessionAppendByte(&p->rebase, 
         3851  +      (eOp==SQLITE_DELETE ? SQLITE_DELETE : SQLITE_INSERT), &rc
         3852  +  );
         3853  +  sessionAppendByte(&p->rebase, (eType==SQLITE_CHANGESET_REPLACE), &rc);
         3854  +  for(i=0; i<p->nCol; i++){
         3855  +    sqlite3_value *pVal = 0;
         3856  +    if( eOp==SQLITE_DELETE || (eOp==SQLITE_UPDATE && p->abPK[i]) ){
         3857  +      sqlite3changeset_old(pIter, i, &pVal);
         3858  +    }else{
         3859  +      sqlite3changeset_new(pIter, i, &pVal);
         3860  +    }
         3861  +    sessionAppendValue(&p->rebase, pVal, &rc);
         3862  +  }
         3863  +
         3864  +  return rc;
         3865  +}
  3814   3866   
  3815   3867   /*
  3816   3868   ** Invoke the conflict handler for the change that the changeset iterator
  3817   3869   ** currently points to.
  3818   3870   **
  3819   3871   ** Argument eType must be either CHANGESET_DATA or CHANGESET_CONFLICT.
  3820   3872   ** If argument pbReplace is NULL, then the type of conflict handler invoked
................................................................................
  3883   3935     }else if( rc==SQLITE_OK ){
  3884   3936       if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){
  3885   3937         /* Instead of invoking the conflict handler, append the change blob
  3886   3938         ** to the SessionApplyCtx.constraints buffer. */
  3887   3939         u8 *aBlob = &pIter->in.aData[pIter->in.iCurrent];
  3888   3940         int nBlob = pIter->in.iNext - pIter->in.iCurrent;
  3889   3941         sessionAppendBlob(&p->constraints, aBlob, nBlob, &rc);
  3890         -      res = SQLITE_CHANGESET_OMIT;
         3942  +      return SQLITE_OK;
  3891   3943       }else{
  3892   3944         /* No other row with the new.* primary key. */
  3893   3945         res = xConflict(pCtx, eType+1, pIter);
  3894   3946         if( res==SQLITE_CHANGESET_REPLACE ) rc = SQLITE_MISUSE;
  3895   3947       }
  3896   3948     }
  3897   3949   
................................................................................
  3909   3961           rc = SQLITE_ABORT;
  3910   3962           break;
  3911   3963   
  3912   3964         default:
  3913   3965           rc = SQLITE_MISUSE;
  3914   3966           break;
  3915   3967       }
         3968  +    if( rc==SQLITE_OK ){
         3969  +      rc = sessionRebaseAdd(p, res, pIter);
         3970  +    }
  3916   3971     }
  3917   3972   
  3918   3973     return rc;
  3919   3974   }
  3920   3975   
  3921   3976   /*
  3922   3977   ** Attempt to apply the change that the iterator passed as the first argument
................................................................................
  4084   4139     void *pCtx                      /* First argument passed to xConflict */
  4085   4140   ){
  4086   4141     int bReplace = 0;
  4087   4142     int bRetry = 0;
  4088   4143     int rc;
  4089   4144   
  4090   4145     rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, &bReplace, &bRetry);
  4091         -  assert( rc==SQLITE_OK || (bRetry==0 && bReplace==0) );
  4092         -
  4093         -  /* If the bRetry flag is set, the change has not been applied due to an
  4094         -  ** SQLITE_CHANGESET_DATA problem (i.e. this is an UPDATE or DELETE and
  4095         -  ** a row with the correct PK is present in the db, but one or more other
  4096         -  ** fields do not contain the expected values) and the conflict handler 
  4097         -  ** returned SQLITE_CHANGESET_REPLACE. In this case retry the operation,
  4098         -  ** but pass NULL as the final argument so that sessionApplyOneOp() ignores
  4099         -  ** the SQLITE_CHANGESET_DATA problem.  */
  4100         -  if( bRetry ){
  4101         -    assert( pIter->op==SQLITE_UPDATE || pIter->op==SQLITE_DELETE );
  4102         -    rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0);
  4103         -  }
  4104         -
  4105         -  /* If the bReplace flag is set, the change is an INSERT that has not
  4106         -  ** been performed because the database already contains a row with the
  4107         -  ** specified primary key and the conflict handler returned
  4108         -  ** SQLITE_CHANGESET_REPLACE. In this case remove the conflicting row
  4109         -  ** before reattempting the INSERT.  */
  4110         -  else if( bReplace ){
  4111         -    assert( pIter->op==SQLITE_INSERT );
  4112         -    rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0);
  4113         -    if( rc==SQLITE_OK ){
  4114         -      rc = sessionBindRow(pIter, 
  4115         -          sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete);
  4116         -      sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1);
  4117         -    }
  4118         -    if( rc==SQLITE_OK ){
  4119         -      sqlite3_step(pApply->pDelete);
  4120         -      rc = sqlite3_reset(pApply->pDelete);
  4121         -    }
  4122         -    if( rc==SQLITE_OK ){
         4146  +  if( rc==SQLITE_OK ){
         4147  +    /* If the bRetry flag is set, the change has not been applied due to an
         4148  +    ** SQLITE_CHANGESET_DATA problem (i.e. this is an UPDATE or DELETE and
         4149  +    ** a row with the correct PK is present in the db, but one or more other
         4150  +    ** fields do not contain the expected values) and the conflict handler 
         4151  +    ** returned SQLITE_CHANGESET_REPLACE. In this case retry the operation,
         4152  +    ** but pass NULL as the final argument so that sessionApplyOneOp() ignores
         4153  +    ** the SQLITE_CHANGESET_DATA problem.  */
         4154  +    if( bRetry ){
         4155  +      assert( pIter->op==SQLITE_UPDATE || pIter->op==SQLITE_DELETE );
  4123   4156         rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0);
  4124   4157       }
  4125         -    if( rc==SQLITE_OK ){
  4126         -      rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0);
         4158  +
         4159  +    /* If the bReplace flag is set, the change is an INSERT that has not
         4160  +    ** been performed because the database already contains a row with the
         4161  +    ** specified primary key and the conflict handler returned
         4162  +    ** SQLITE_CHANGESET_REPLACE. In this case remove the conflicting row
         4163  +    ** before reattempting the INSERT.  */
         4164  +    else if( bReplace ){
         4165  +      assert( pIter->op==SQLITE_INSERT );
         4166  +      rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0);
         4167  +      if( rc==SQLITE_OK ){
         4168  +        rc = sessionBindRow(pIter, 
         4169  +            sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete);
         4170  +        sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1);
         4171  +      }
         4172  +      if( rc==SQLITE_OK ){
         4173  +        sqlite3_step(pApply->pDelete);
         4174  +        rc = sqlite3_reset(pApply->pDelete);
         4175  +      }
         4176  +      if( rc==SQLITE_OK ){
         4177  +        rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0);
         4178  +      }
         4179  +      if( rc==SQLITE_OK ){
         4180  +        rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0);
         4181  +      }
  4127   4182       }
  4128   4183     }
  4129   4184   
  4130   4185     return rc;
  4131   4186   }
  4132   4187   
  4133   4188   /*
................................................................................
  4195   4250       const char *zTab              /* Table name */
  4196   4251     ),
  4197   4252     int(*xConflict)(
  4198   4253       void *pCtx,                   /* Copy of fifth arg to _apply() */
  4199   4254       int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
  4200   4255       sqlite3_changeset_iter *p     /* Handle describing change and conflict */
  4201   4256     ),
  4202         -  void *pCtx                      /* First argument passed to xConflict */
         4257  +  void *pCtx,                     /* First argument passed to xConflict */
         4258  +  void **ppRebase, int *pnRebase, /* OUT: Rebase information */
         4259  +  int flags                       /* SESSION_APPLY_XXX flags */
  4203   4260   ){
  4204   4261     int schemaMismatch = 0;
  4205         -  int rc;                         /* Return code */
         4262  +  int rc = SQLITE_OK;             /* Return code */
  4206   4263     const char *zTab = 0;           /* Name of current table */
  4207   4264     int nTab = 0;                   /* Result of sqlite3Strlen30(zTab) */
  4208   4265     SessionApplyCtx sApply;         /* changeset_apply() context object */
  4209   4266     int bPatchset;
  4210   4267   
  4211   4268     assert( xConflict!=0 );
  4212   4269   
  4213   4270     pIter->in.bNoDiscard = 1;
  4214   4271     memset(&sApply, 0, sizeof(sApply));
  4215   4272     sqlite3_mutex_enter(sqlite3_db_mutex(db));
  4216         -  rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0);
         4273  +  if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){
         4274  +    rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0);
         4275  +  }
  4217   4276     if( rc==SQLITE_OK ){
  4218   4277       rc = sqlite3_exec(db, "PRAGMA defer_foreign_keys = 1", 0, 0, 0);
  4219   4278     }
  4220   4279     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter) ){
  4221   4280       int nCol;
  4222   4281       int op;
  4223   4282       const char *zNew;
................................................................................
  4233   4292         if( rc!=SQLITE_OK ) break;
  4234   4293   
  4235   4294         sqlite3_free((char*)sApply.azCol);  /* cast works around VC++ bug */
  4236   4295         sqlite3_finalize(sApply.pDelete);
  4237   4296         sqlite3_finalize(sApply.pUpdate); 
  4238   4297         sqlite3_finalize(sApply.pInsert);
  4239   4298         sqlite3_finalize(sApply.pSelect);
  4240         -      memset(&sApply, 0, sizeof(sApply));
  4241   4299         sApply.db = db;
         4300  +      sApply.pDelete = 0;
         4301  +      sApply.pUpdate = 0;
         4302  +      sApply.pInsert = 0;
         4303  +      sApply.pSelect = 0;
         4304  +      sApply.nCol = 0;
         4305  +      sApply.azCol = 0;
         4306  +      sApply.abPK = 0;
         4307  +      sApply.bStat1 = 0;
  4242   4308         sApply.bDeferConstraints = 1;
         4309  +      sApply.bRebaseStarted = 0;
         4310  +      memset(&sApply.constraints, 0, sizeof(SessionBuffer));
  4243   4311   
  4244   4312         /* If an xFilter() callback was specified, invoke it now. If the 
  4245   4313         ** xFilter callback returns zero, skip this table. If it returns
  4246   4314         ** non-zero, proceed. */
  4247   4315         schemaMismatch = (xFilter && (0==xFilter(pCtx, zNew)));
  4248   4316         if( schemaMismatch ){
  4249   4317           zTab = sqlite3_mprintf("%s", zNew);
................................................................................
  4338   4406         if( res!=SQLITE_CHANGESET_OMIT ){
  4339   4407           rc = SQLITE_CONSTRAINT;
  4340   4408         }
  4341   4409       }
  4342   4410     }
  4343   4411     sqlite3_exec(db, "PRAGMA defer_foreign_keys = 0", 0, 0, 0);
  4344   4412   
  4345         -  if( rc==SQLITE_OK ){
  4346         -    rc = sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0);
  4347         -  }else{
  4348         -    sqlite3_exec(db, "ROLLBACK TO changeset_apply", 0, 0, 0);
  4349         -    sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0);
         4413  +  if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){
         4414  +    if( rc==SQLITE_OK ){
         4415  +      rc = sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0);
         4416  +    }else{
         4417  +      sqlite3_exec(db, "ROLLBACK TO changeset_apply", 0, 0, 0);
         4418  +      sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0);
         4419  +    }
  4350   4420     }
  4351   4421   
         4422  +  if( rc==SQLITE_OK && bPatchset==0 && ppRebase && pnRebase ){
         4423  +    *ppRebase = (void*)sApply.rebase.aBuf;
         4424  +    *pnRebase = sApply.rebase.nBuf;
         4425  +    sApply.rebase.aBuf = 0;
         4426  +  }
  4352   4427     sqlite3_finalize(sApply.pInsert);
  4353   4428     sqlite3_finalize(sApply.pDelete);
  4354   4429     sqlite3_finalize(sApply.pUpdate);
  4355   4430     sqlite3_finalize(sApply.pSelect);
  4356   4431     sqlite3_free((char*)sApply.azCol);  /* cast works around VC++ bug */
  4357   4432     sqlite3_free((char*)sApply.constraints.aBuf);
         4433  +  sqlite3_free((char*)sApply.rebase.aBuf);
  4358   4434     sqlite3_mutex_leave(sqlite3_db_mutex(db));
  4359   4435     return rc;
  4360   4436   }
         4437  +
         4438  +/*
         4439  +** Apply the changeset passed via pChangeset/nChangeset to the main 
         4440  +** database attached to handle "db".
         4441  +*/
         4442  +int sqlite3changeset_apply_v2(
         4443  +  sqlite3 *db,                    /* Apply change to "main" db of this handle */
         4444  +  int nChangeset,                 /* Size of changeset in bytes */
         4445  +  void *pChangeset,               /* Changeset blob */
         4446  +  int(*xFilter)(
         4447  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         4448  +    const char *zTab              /* Table name */
         4449  +  ),
         4450  +  int(*xConflict)(
         4451  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         4452  +    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
         4453  +    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
         4454  +  ),
         4455  +  void *pCtx,                     /* First argument passed to xConflict */
         4456  +  void **ppRebase, int *pnRebase,
         4457  +  int flags
         4458  +){
         4459  +  sqlite3_changeset_iter *pIter;  /* Iterator to skip through changeset */  
         4460  +  int rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset);
         4461  +  if( rc==SQLITE_OK ){
         4462  +    rc = sessionChangesetApply(
         4463  +        db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags
         4464  +    );
         4465  +  }
         4466  +  return rc;
         4467  +}
  4361   4468   
  4362   4469   /*
  4363   4470   ** Apply the changeset passed via pChangeset/nChangeset to the main database
  4364   4471   ** attached to handle "db". Invoke the supplied conflict handler callback
  4365   4472   ** to resolve any conflicts encountered while applying the change.
  4366   4473   */
  4367   4474   int sqlite3changeset_apply(
................................................................................
  4375   4482     int(*xConflict)(
  4376   4483       void *pCtx,                   /* Copy of fifth arg to _apply() */
  4377   4484       int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
  4378   4485       sqlite3_changeset_iter *p     /* Handle describing change and conflict */
  4379   4486     ),
  4380   4487     void *pCtx                      /* First argument passed to xConflict */
  4381   4488   ){
  4382         -  sqlite3_changeset_iter *pIter;  /* Iterator to skip through changeset */  
  4383         -  int rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset);
  4384         -  if( rc==SQLITE_OK ){
  4385         -    rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx);
  4386         -  }
  4387         -  return rc;
         4489  +  return sqlite3changeset_apply_v2(
         4490  +      db, nChangeset, pChangeset, xFilter, xConflict, pCtx, 0, 0, 0
         4491  +  );
  4388   4492   }
  4389   4493   
  4390   4494   /*
  4391   4495   ** Apply the changeset passed via xInput/pIn to the main database
  4392   4496   ** attached to handle "db". Invoke the supplied conflict handler callback
  4393   4497   ** to resolve any conflicts encountered while applying the change.
  4394   4498   */
         4499  +int sqlite3changeset_apply_v2_strm(
         4500  +  sqlite3 *db,                    /* Apply change to "main" db of this handle */
         4501  +  int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */
         4502  +  void *pIn,                                          /* First arg for xInput */
         4503  +  int(*xFilter)(
         4504  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         4505  +    const char *zTab              /* Table name */
         4506  +  ),
         4507  +  int(*xConflict)(
         4508  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         4509  +    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
         4510  +    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
         4511  +  ),
         4512  +  void *pCtx,                     /* First argument passed to xConflict */
         4513  +  void **ppRebase, int *pnRebase,
         4514  +  int flags
         4515  +){
         4516  +  sqlite3_changeset_iter *pIter;  /* Iterator to skip through changeset */  
         4517  +  int rc = sqlite3changeset_start_strm(&pIter, xInput, pIn);
         4518  +  if( rc==SQLITE_OK ){
         4519  +    rc = sessionChangesetApply(
         4520  +        db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags
         4521  +    );
         4522  +  }
         4523  +  return rc;
         4524  +}
  4395   4525   int sqlite3changeset_apply_strm(
  4396   4526     sqlite3 *db,                    /* Apply change to "main" db of this handle */
  4397   4527     int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */
  4398   4528     void *pIn,                                          /* First arg for xInput */
  4399   4529     int(*xFilter)(
  4400   4530       void *pCtx,                   /* Copy of sixth arg to _apply() */
  4401   4531       const char *zTab              /* Table name */
................................................................................
  4403   4533     int(*xConflict)(
  4404   4534       void *pCtx,                   /* Copy of sixth arg to _apply() */
  4405   4535       int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
  4406   4536       sqlite3_changeset_iter *p     /* Handle describing change and conflict */
  4407   4537     ),
  4408   4538     void *pCtx                      /* First argument passed to xConflict */
  4409   4539   ){
  4410         -  sqlite3_changeset_iter *pIter;  /* Iterator to skip through changeset */  
  4411         -  int rc = sqlite3changeset_start_strm(&pIter, xInput, pIn);
  4412         -  if( rc==SQLITE_OK ){
  4413         -    rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx);
  4414         -  }
  4415         -  return rc;
         4540  +  return sqlite3changeset_apply_v2_strm(
         4541  +      db, xInput, pIn, xFilter, xConflict, pCtx, 0, 0, 0
         4542  +  );
  4416   4543   }
  4417   4544   
  4418   4545   /*
  4419   4546   ** sqlite3_changegroup handle.
  4420   4547   */
  4421   4548   struct sqlite3_changegroup {
  4422   4549     int rc;                         /* Error code */
................................................................................
  4427   4554   /*
  4428   4555   ** This function is called to merge two changes to the same row together as
  4429   4556   ** part of an sqlite3changeset_concat() operation. A new change object is
  4430   4557   ** allocated and a pointer to it stored in *ppNew.
  4431   4558   */
  4432   4559   static int sessionChangeMerge(
  4433   4560     SessionTable *pTab,             /* Table structure */
         4561  +  int bRebase,                    /* True for a rebase hash-table */
  4434   4562     int bPatchset,                  /* True for patchsets */
  4435   4563     SessionChange *pExist,          /* Existing change */
  4436   4564     int op2,                        /* Second change operation */
  4437   4565     int bIndirect,                  /* True if second change is indirect */
  4438   4566     u8 *aRec,                       /* Second change record */
  4439   4567     int nRec,                       /* Number of bytes in aRec */
  4440   4568     SessionChange **ppNew           /* OUT: Merged change */
  4441   4569   ){
  4442   4570     SessionChange *pNew = 0;
         4571  +  int rc = SQLITE_OK;
  4443   4572   
  4444   4573     if( !pExist ){
  4445   4574       pNew = (SessionChange *)sqlite3_malloc(sizeof(SessionChange) + nRec);
  4446   4575       if( !pNew ){
  4447   4576         return SQLITE_NOMEM;
  4448   4577       }
  4449   4578       memset(pNew, 0, sizeof(SessionChange));
  4450   4579       pNew->op = op2;
  4451   4580       pNew->bIndirect = bIndirect;
  4452         -    pNew->nRecord = nRec;
  4453   4581       pNew->aRecord = (u8*)&pNew[1];
  4454         -    memcpy(pNew->aRecord, aRec, nRec);
         4582  +    if( bIndirect==0 || bRebase==0 ){
         4583  +      pNew->nRecord = nRec;
         4584  +      memcpy(pNew->aRecord, aRec, nRec);
         4585  +    }else{
         4586  +      int i;
         4587  +      u8 *pIn = aRec;
         4588  +      u8 *pOut = pNew->aRecord;
         4589  +      for(i=0; i<pTab->nCol; i++){
         4590  +        int nIn = sessionSerialLen(pIn);
         4591  +        if( *pIn==0 ){
         4592  +          *pOut++ = 0;
         4593  +        }else if( pTab->abPK[i]==0 ){
         4594  +          *pOut++ = 0xFF;
         4595  +        }else{
         4596  +          memcpy(pOut, pIn, nIn);
         4597  +          pOut += nIn;
         4598  +        }
         4599  +        pIn += nIn;
         4600  +      }
         4601  +      pNew->nRecord = pOut - pNew->aRecord;
         4602  +    }
         4603  +  }else if( bRebase ){
         4604  +    if( pExist->op==SQLITE_DELETE && pExist->bIndirect ){
         4605  +      *ppNew = pExist;
         4606  +    }else{
         4607  +      int nByte = nRec + pExist->nRecord + sizeof(SessionChange);
         4608  +      pNew = (SessionChange*)sqlite3_malloc(nByte);
         4609  +      if( pNew==0 ){
         4610  +        rc = SQLITE_NOMEM;
         4611  +      }else{
         4612  +        int i;
         4613  +        u8 *a1 = pExist->aRecord;
         4614  +        u8 *a2 = aRec;
         4615  +        u8 *pOut;
         4616  +
         4617  +        memset(pNew, 0, nByte);
         4618  +        pNew->bIndirect = bIndirect || pExist->bIndirect;
         4619  +        pNew->op = op2;
         4620  +        pOut = pNew->aRecord = (u8*)&pNew[1];
         4621  +
         4622  +        for(i=0; i<pTab->nCol; i++){
         4623  +          int n1 = sessionSerialLen(a1);
         4624  +          int n2 = sessionSerialLen(a2);
         4625  +          if( *a1==0xFF || (pTab->abPK[i]==0 && bIndirect) ){
         4626  +            *pOut++ = 0xFF;
         4627  +          }else if( *a2==0 ){
         4628  +            memcpy(pOut, a1, n1);
         4629  +            pOut += n1;
         4630  +          }else{
         4631  +            memcpy(pOut, a2, n2);
         4632  +            pOut += n2;
         4633  +          }
         4634  +          a1 += n1;
         4635  +          a2 += n2;
         4636  +        }
         4637  +        pNew->nRecord = pOut - pNew->aRecord;
         4638  +      }
         4639  +      sqlite3_free(pExist);
         4640  +    }
  4455   4641     }else{
  4456   4642       int op1 = pExist->op;
  4457   4643   
  4458   4644       /* 
  4459   4645       **   op1=INSERT, op2=INSERT      ->      Unsupported. Discard op2.
  4460   4646       **   op1=INSERT, op2=UPDATE      ->      INSERT.
  4461   4647       **   op1=INSERT, op2=DELETE      ->      (none)
................................................................................
  4541   4727           pNew->nRecord = (int)(aCsr - pNew->aRecord);
  4542   4728         }
  4543   4729         sqlite3_free(pExist);
  4544   4730       }
  4545   4731     }
  4546   4732   
  4547   4733     *ppNew = pNew;
  4548         -  return SQLITE_OK;
         4734  +  return rc;
  4549   4735   }
  4550   4736   
  4551   4737   /*
  4552   4738   ** Add all changes in the changeset traversed by the iterator passed as
  4553   4739   ** the first argument to the changegroup hash tables.
  4554   4740   */
  4555   4741   static int sessionChangesetToHash(
  4556   4742     sqlite3_changeset_iter *pIter,   /* Iterator to read from */
  4557         -  sqlite3_changegroup *pGrp        /* Changegroup object to add changeset to */
         4743  +  sqlite3_changegroup *pGrp,       /* Changegroup object to add changeset to */
         4744  +  int bRebase                      /* True if hash table is for rebasing */
  4558   4745   ){
  4559   4746     u8 *aRec;
  4560   4747     int nRec;
  4561   4748     int rc = SQLITE_OK;
  4562   4749     SessionTable *pTab = 0;
  4563   4750   
  4564         -
  4565         -  while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec) ){
         4751  +  while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){
  4566   4752       const char *zNew;
  4567   4753       int nCol;
  4568   4754       int op;
  4569   4755       int iHash;
  4570   4756       int bIndirect;
  4571   4757       SessionChange *pChange;
  4572   4758       SessionChange *pExist = 0;
................................................................................
  4638   4824           pExist = *pp;
  4639   4825           *pp = (*pp)->pNext;
  4640   4826           pTab->nEntry--;
  4641   4827           break;
  4642   4828         }
  4643   4829       }
  4644   4830   
  4645         -    rc = sessionChangeMerge(pTab, 
         4831  +    rc = sessionChangeMerge(pTab, bRebase, 
  4646   4832           pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange
  4647   4833       );
  4648   4834       if( rc ) break;
  4649   4835       if( pChange ){
  4650   4836         pChange->pNext = pTab->apChange[iHash];
  4651   4837         pTab->apChange[iHash] = pChange;
  4652   4838         pTab->nEntry++;
................................................................................
  4747   4933   */
  4748   4934   int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void *pData){
  4749   4935     sqlite3_changeset_iter *pIter;  /* Iterator opened on pData/nData */
  4750   4936     int rc;                         /* Return code */
  4751   4937   
  4752   4938     rc = sqlite3changeset_start(&pIter, nData, pData);
  4753   4939     if( rc==SQLITE_OK ){
  4754         -    rc = sessionChangesetToHash(pIter, pGrp);
         4940  +    rc = sessionChangesetToHash(pIter, pGrp, 0);
  4755   4941     }
  4756   4942     sqlite3changeset_finalize(pIter);
  4757   4943     return rc;
  4758   4944   }
  4759   4945   
  4760   4946   /*
  4761   4947   ** Obtain a buffer containing a changeset representing the concatenation
................................................................................
  4778   4964     void *pIn
  4779   4965   ){
  4780   4966     sqlite3_changeset_iter *pIter;  /* Iterator opened on pData/nData */
  4781   4967     int rc;                         /* Return code */
  4782   4968   
  4783   4969     rc = sqlite3changeset_start_strm(&pIter, xInput, pIn);
  4784   4970     if( rc==SQLITE_OK ){
  4785         -    rc = sessionChangesetToHash(pIter, pGrp);
         4971  +    rc = sessionChangesetToHash(pIter, pGrp, 0);
  4786   4972     }
  4787   4973     sqlite3changeset_finalize(pIter);
  4788   4974     return rc;
  4789   4975   }
  4790   4976   
  4791   4977   /*
  4792   4978   ** Streaming versions of changegroup_output().
................................................................................
  4862   5048     if( rc==SQLITE_OK ){
  4863   5049       rc = sqlite3changegroup_output_strm(pGrp, xOutput, pOut);
  4864   5050     }
  4865   5051     sqlite3changegroup_delete(pGrp);
  4866   5052   
  4867   5053     return rc;
  4868   5054   }
         5055  +
         5056  +/*
         5057  +** Changeset rebaser handle.
         5058  +*/
         5059  +struct sqlite3_rebaser {
         5060  +  sqlite3_changegroup grp;        /* Hash table */
         5061  +};
         5062  +
         5063  +/*
         5064  +** Buffers a1 and a2 must both contain a sessions module record nCol
         5065  +** fields in size. This function appends an nCol sessions module 
         5066  +** record to buffer pBuf that is a copy of a1, except that for
         5067  +** each field that is undefined in a1[], swap in the field from a2[].
         5068  +*/
         5069  +static void sessionAppendRecordMerge(
         5070  +  SessionBuffer *pBuf,            /* Buffer to append to */
         5071  +  int nCol,                       /* Number of columns in each record */
         5072  +  u8 *a1, int n1,                 /* Record 1 */
         5073  +  u8 *a2, int n2,                 /* Record 2 */
         5074  +  int *pRc                        /* IN/OUT: error code */
         5075  +){
         5076  +  sessionBufferGrow(pBuf, n1+n2, pRc);
         5077  +  if( *pRc==SQLITE_OK ){
         5078  +    int i;
         5079  +    u8 *pOut = &pBuf->aBuf[pBuf->nBuf];
         5080  +    for(i=0; i<nCol; i++){
         5081  +      int nn1 = sessionSerialLen(a1);
         5082  +      int nn2 = sessionSerialLen(a2);
         5083  +      if( *a1==0 || *a1==0xFF ){
         5084  +        memcpy(pOut, a2, nn2);
         5085  +        pOut += nn2;
         5086  +      }else{
         5087  +        memcpy(pOut, a1, nn1);
         5088  +        pOut += nn1;
         5089  +      }
         5090  +      a1 += nn1;
         5091  +      a2 += nn2;
         5092  +    }
         5093  +
         5094  +    pBuf->nBuf = pOut-pBuf->aBuf;
         5095  +    assert( pBuf->nBuf<=pBuf->nAlloc );
         5096  +  }
         5097  +}
         5098  +
         5099  +/*
         5100  +** This function is called when rebasing a local UPDATE change against one 
         5101  +** or more remote UPDATE changes. The aRec/nRec buffer contains the current
         5102  +** old.* and new.* records for the change. The rebase buffer (a single
         5103  +** record) is in aChange/nChange. The rebased change is appended to buffer
         5104  +** pBuf.
         5105  +**
         5106  +** Rebasing the UPDATE involves: 
         5107  +**
         5108  +**   * Removing any changes to fields for which the corresponding field
         5109  +**     in the rebase buffer is set to "replaced" (type 0xFF). If this
         5110  +**     means the UPDATE change updates no fields, nothing is appended
         5111  +**     to the output buffer.
         5112  +**
         5113  +**   * For each field modified by the local change for which the 
         5114  +**     corresponding field in the rebase buffer is not "undefined" (0x00)
         5115  +**     or "replaced" (0xFF), the old.* value is replaced by the value
         5116  +**     in the rebase buffer.
         5117  +*/
         5118  +static void sessionAppendPartialUpdate(
         5119  +  SessionBuffer *pBuf,            /* Append record here */
         5120  +  sqlite3_changeset_iter *pIter,  /* Iterator pointed at local change */
         5121  +  u8 *aRec, int nRec,             /* Local change */
         5122  +  u8 *aChange, int nChange,       /* Record to rebase against */
         5123  +  int *pRc                        /* IN/OUT: Return Code */
         5124  +){
         5125  +  sessionBufferGrow(pBuf, 2+nRec+nChange, pRc);
         5126  +  if( *pRc==SQLITE_OK ){
         5127  +    int bData = 0;
         5128  +    u8 *pOut = &pBuf->aBuf[pBuf->nBuf];
         5129  +    int i;
         5130  +    u8 *a1 = aRec;
         5131  +    u8 *a2 = aChange;
         5132  +
         5133  +    *pOut++ = SQLITE_UPDATE;
         5134  +    *pOut++ = pIter->bIndirect;
         5135  +    for(i=0; i<pIter->nCol; i++){
         5136  +      int n1 = sessionSerialLen(a1);
         5137  +      int n2 = sessionSerialLen(a2);
         5138  +      if( pIter->abPK[i] || a2[0]==0 ){
         5139  +        if( !pIter->abPK[i] ) bData = 1;
         5140  +        memcpy(pOut, a1, n1);
         5141  +        pOut += n1;
         5142  +      }else if( a2[0]!=0xFF ){
         5143  +        bData = 1;
         5144  +        memcpy(pOut, a2, n2);
         5145  +        pOut += n2;
         5146  +      }else{
         5147  +        *pOut++ = '\0';
         5148  +      }
         5149  +      a1 += n1;
         5150  +      a2 += n2;
         5151  +    }
         5152  +    if( bData ){
         5153  +      a2 = aChange;
         5154  +      for(i=0; i<pIter->nCol; i++){
         5155  +        int n1 = sessionSerialLen(a1);
         5156  +        int n2 = sessionSerialLen(a2);
         5157  +        if( pIter->abPK[i] || a2[0]!=0xFF ){
         5158  +          memcpy(pOut, a1, n1);
         5159  +          pOut += n1;
         5160  +        }else{
         5161  +          *pOut++ = '\0';
         5162  +        }
         5163  +        a1 += n1;
         5164  +        a2 += n2;
         5165  +      }
         5166  +      pBuf->nBuf = (pOut - pBuf->aBuf);
         5167  +    }
         5168  +  }
         5169  +}
         5170  +
         5171  +/*
         5172  +** pIter is configured to iterate through a changeset. This function rebases 
         5173  +** that changeset according to the current configuration of the rebaser 
         5174  +** object passed as the first argument. If no error occurs and argument xOutput
         5175  +** is not NULL, then the changeset is returned to the caller by invoking
         5176  +** xOutput zero or more times and SQLITE_OK returned. Or, if xOutput is NULL,
         5177  +** then (*ppOut) is set to point to a buffer containing the rebased changeset
         5178  +** before this function returns. In this case (*pnOut) is set to the size of
         5179  +** the buffer in bytes.  It is the responsibility of the caller to eventually
         5180  +** free the (*ppOut) buffer using sqlite3_free(). 
         5181  +**
         5182  +** If an error occurs, an SQLite error code is returned. If ppOut and
         5183  +** pnOut are not NULL, then the two output parameters are set to 0 before
         5184  +** returning.
         5185  +*/
         5186  +static int sessionRebase(
         5187  +  sqlite3_rebaser *p,             /* Rebaser hash table */
         5188  +  sqlite3_changeset_iter *pIter,  /* Input data */
         5189  +  int (*xOutput)(void *pOut, const void *pData, int nData),
         5190  +  void *pOut,                     /* Context for xOutput callback */
         5191  +  int *pnOut,                     /* OUT: Number of bytes in output changeset */
         5192  +  void **ppOut                    /* OUT: Inverse of pChangeset */
         5193  +){
         5194  +  int rc = SQLITE_OK;
         5195  +  u8 *aRec = 0;
         5196  +  int nRec = 0;
         5197  +  int bNew = 0;
         5198  +  SessionTable *pTab = 0;
         5199  +  SessionBuffer sOut = {0,0,0};
         5200  +
         5201  +  while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, &bNew) ){
         5202  +    SessionChange *pChange = 0;
         5203  +    int bDone = 0;
         5204  +
         5205  +    if( bNew ){
         5206  +      const char *zTab = pIter->zTab;
         5207  +      for(pTab=p->grp.pList; pTab; pTab=pTab->pNext){
         5208  +        if( 0==sqlite3_stricmp(pTab->zName, zTab) ) break;
         5209  +      }
         5210  +      bNew = 0;
         5211  +
         5212  +      /* A patchset may not be rebased */
         5213  +      if( pIter->bPatchset ){
         5214  +        rc = SQLITE_ERROR;
         5215  +      }
         5216  +
         5217  +      /* Append a table header to the output for this new table */
         5218  +      sessionAppendByte(&sOut, pIter->bPatchset ? 'P' : 'T', &rc);
         5219  +      sessionAppendVarint(&sOut, pIter->nCol, &rc);
         5220  +      sessionAppendBlob(&sOut, pIter->abPK, pIter->nCol, &rc);
         5221  +      sessionAppendBlob(&sOut,(u8*)pIter->zTab,(int)strlen(pIter->zTab)+1,&rc);
         5222  +    }
         5223  +
         5224  +    if( pTab && rc==SQLITE_OK ){
         5225  +      int iHash = sessionChangeHash(pTab, 0, aRec, pTab->nChange);
         5226  +
         5227  +      for(pChange=pTab->apChange[iHash]; pChange; pChange=pChange->pNext){
         5228  +        if( sessionChangeEqual(pTab, 0, aRec, 0, pChange->aRecord) ){
         5229  +          break;
         5230  +        }
         5231  +      }
         5232  +    }
         5233  +
         5234  +    if( pChange ){
         5235  +      assert( pChange->op==SQLITE_DELETE || pChange->op==SQLITE_INSERT );
         5236  +      switch( pIter->op ){
         5237  +        case SQLITE_INSERT:
         5238  +          if( pChange->op==SQLITE_INSERT ){
         5239  +            bDone = 1;
         5240  +            if( pChange->bIndirect==0 ){
         5241  +              sessionAppendByte(&sOut, SQLITE_UPDATE, &rc);
         5242  +              sessionAppendByte(&sOut, pIter->bIndirect, &rc);
         5243  +              sessionAppendBlob(&sOut, pChange->aRecord, pChange->nRecord, &rc);
         5244  +              sessionAppendBlob(&sOut, aRec, nRec, &rc);
         5245  +            }
         5246  +          }
         5247  +          break;
         5248  +
         5249  +        case SQLITE_UPDATE:
         5250  +          bDone = 1;
         5251  +          if( pChange->op==SQLITE_DELETE ){
         5252  +            if( pChange->bIndirect==0 ){
         5253  +              u8 *pCsr = aRec;
         5254  +              sessionSkipRecord(&pCsr, pIter->nCol);
         5255  +              sessionAppendByte(&sOut, SQLITE_INSERT, &rc);
         5256  +              sessionAppendByte(&sOut, pIter->bIndirect, &rc);
         5257  +              sessionAppendRecordMerge(&sOut, pIter->nCol,
         5258  +                  pCsr, nRec-(pCsr-aRec), 
         5259  +                  pChange->aRecord, pChange->nRecord, &rc
         5260  +              );
         5261  +            }
         5262  +          }else{
         5263  +            sessionAppendPartialUpdate(&sOut, pIter,
         5264  +                aRec, nRec, pChange->aRecord, pChange->nRecord, &rc
         5265  +            );
         5266  +          }
         5267  +          break;
         5268  +
         5269  +        default:
         5270  +          assert( pIter->op==SQLITE_DELETE );
         5271  +          bDone = 1;
         5272  +          if( pChange->op==SQLITE_INSERT ){
         5273  +            sessionAppendByte(&sOut, SQLITE_DELETE, &rc);
         5274  +            sessionAppendByte(&sOut, pIter->bIndirect, &rc);
         5275  +            sessionAppendRecordMerge(&sOut, pIter->nCol,
         5276  +                pChange->aRecord, pChange->nRecord, aRec, nRec, &rc
         5277  +            );
         5278  +          }
         5279  +          break;
         5280  +      }
         5281  +    }
         5282  +
         5283  +    if( bDone==0 ){
         5284  +      sessionAppendByte(&sOut, pIter->op, &rc);
         5285  +      sessionAppendByte(&sOut, pIter->bIndirect, &rc);
         5286  +      sessionAppendBlob(&sOut, aRec, nRec, &rc);
         5287  +    }
         5288  +    if( rc==SQLITE_OK && xOutput && sOut.nBuf>SESSIONS_STRM_CHUNK_SIZE ){
         5289  +      rc = xOutput(pOut, sOut.aBuf, sOut.nBuf);
         5290  +      sOut.nBuf = 0;
         5291  +    }
         5292  +    if( rc ) break;
         5293  +  }
         5294  +
         5295  +  if( rc!=SQLITE_OK ){
         5296  +    sqlite3_free(sOut.aBuf);
         5297  +    memset(&sOut, 0, sizeof(sOut));
         5298  +  }
         5299  +
         5300  +  if( rc==SQLITE_OK ){
         5301  +    if( xOutput ){
         5302  +      if( sOut.nBuf>0 ){
         5303  +        rc = xOutput(pOut, sOut.aBuf, sOut.nBuf);
         5304  +      }
         5305  +    }else{
         5306  +      *ppOut = (void*)sOut.aBuf;
         5307  +      *pnOut = sOut.nBuf;
         5308  +      sOut.aBuf = 0;
         5309  +    }
         5310  +  }
         5311  +  sqlite3_free(sOut.aBuf);
         5312  +  return rc;
         5313  +}
         5314  +
         5315  +/* 
         5316  +** Create a new rebaser object.
         5317  +*/
         5318  +int sqlite3rebaser_create(sqlite3_rebaser **ppNew){
         5319  +  int rc = SQLITE_OK;
         5320  +  sqlite3_rebaser *pNew;
         5321  +
         5322  +  pNew = sqlite3_malloc(sizeof(sqlite3_rebaser));
         5323  +  if( pNew==0 ){
         5324  +    rc = SQLITE_NOMEM;
         5325  +  }else{
         5326  +    memset(pNew, 0, sizeof(sqlite3_rebaser));
         5327  +  }
         5328  +  *ppNew = pNew;
         5329  +  return rc;
         5330  +}
         5331  +
         5332  +/* 
         5333  +** Call this one or more times to configure a rebaser.
         5334  +*/
         5335  +int sqlite3rebaser_configure(
         5336  +  sqlite3_rebaser *p, 
         5337  +  int nRebase, const void *pRebase
         5338  +){
         5339  +  sqlite3_changeset_iter *pIter = 0;   /* Iterator opened on pData/nData */
         5340  +  int rc;                              /* Return code */
         5341  +  rc = sqlite3changeset_start(&pIter, nRebase, (void*)pRebase);
         5342  +  if( rc==SQLITE_OK ){
         5343  +    rc = sessionChangesetToHash(pIter, &p->grp, 1);
         5344  +  }
         5345  +  sqlite3changeset_finalize(pIter);
         5346  +  return rc;
         5347  +}
         5348  +
         5349  +/* 
         5350  +** Rebase a changeset according to current rebaser configuration 
         5351  +*/
         5352  +int sqlite3rebaser_rebase(
         5353  +  sqlite3_rebaser *p,
         5354  +  int nIn, const void *pIn, 
         5355  +  int *pnOut, void **ppOut 
         5356  +){
         5357  +  sqlite3_changeset_iter *pIter = 0;   /* Iterator to skip through input */  
         5358  +  int rc = sqlite3changeset_start(&pIter, nIn, (void*)pIn);
         5359  +
         5360  +  if( rc==SQLITE_OK ){
         5361  +    rc = sessionRebase(p, pIter, 0, 0, pnOut, ppOut);
         5362  +    sqlite3changeset_finalize(pIter);
         5363  +  }
         5364  +
         5365  +  return rc;
         5366  +}
         5367  +
         5368  +/* 
         5369  +** Rebase a changeset according to current rebaser configuration 
         5370  +*/
         5371  +int sqlite3rebaser_rebase_strm(
         5372  +  sqlite3_rebaser *p,
         5373  +  int (*xInput)(void *pIn, void *pData, int *pnData),
         5374  +  void *pIn,
         5375  +  int (*xOutput)(void *pOut, const void *pData, int nData),
         5376  +  void *pOut
         5377  +){
         5378  +  sqlite3_changeset_iter *pIter = 0;   /* Iterator to skip through input */  
         5379  +  int rc = sqlite3changeset_start_strm(&pIter, xInput, pIn);
         5380  +
         5381  +  if( rc==SQLITE_OK ){
         5382  +    rc = sessionRebase(p, pIter, xOutput, pOut, 0, 0);
         5383  +    sqlite3changeset_finalize(pIter);
         5384  +  }
         5385  +
         5386  +  return rc;
         5387  +}
         5388  +
         5389  +/* 
         5390  +** Destroy a rebaser object 
         5391  +*/
         5392  +void sqlite3rebaser_delete(sqlite3_rebaser *p){
         5393  +  if( p ){
         5394  +    sessionDeleteTable(p->grp.pList);
         5395  +    sqlite3_free(p);
         5396  +  }
         5397  +}
  4869   5398   
  4870   5399   #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */

Changes to ext/session/sqlite3session.h.

   957    957   ** DESTRUCTOR: sqlite3_changegroup
   958    958   */
   959    959   void sqlite3changegroup_delete(sqlite3_changegroup*);
   960    960   
   961    961   /*
   962    962   ** CAPI3REF: Apply A Changeset To A Database
   963    963   **
   964         -** Apply a changeset to a database. This function attempts to update the
   965         -** "main" database attached to handle db with the changes found in the
   966         -** changeset passed via the second and third arguments.
          964  +** Apply a changeset or patchset to a database. These functions attempt to
          965  +** update the "main" database attached to handle db with the changes found in
          966  +** the changeset passed via the second and third arguments. 
   967    967   **
   968         -** The fourth argument (xFilter) passed to this function is the "filter
          968  +** The fourth argument (xFilter) passed to these functions is the "filter
   969    969   ** callback". If it is not NULL, then for each table affected by at least one
   970    970   ** change in the changeset, the filter callback is invoked with
   971    971   ** the table name as the second argument, and a copy of the context pointer
   972         -** passed as the sixth argument to this function as the first. If the "filter
   973         -** callback" returns zero, then no attempt is made to apply any changes to 
   974         -** the table. Otherwise, if the return value is non-zero or the xFilter
   975         -** argument to this function is NULL, all changes related to the table are
   976         -** attempted.
          972  +** passed as the sixth argument as the first. If the "filter callback"
          973  +** returns zero, then no attempt is made to apply any changes to the table.
          974  +** Otherwise, if the return value is non-zero or the xFilter argument to
          975  +** is NULL, all changes related to the table are attempted.
   977    976   **
   978    977   ** For each table that is not excluded by the filter callback, this function 
   979    978   ** tests that the target database contains a compatible table. A table is 
   980    979   ** considered compatible if all of the following are true:
   981    980   **
   982    981   ** <ul>
   983    982   **   <li> The table has the same name as the name recorded in the 
................................................................................
  1014   1013   ** actions are taken by sqlite3changeset_apply() depending on the value
  1015   1014   ** returned by each invocation of the conflict-handler function. Refer to
  1016   1015   ** the documentation for the three 
  1017   1016   ** [SQLITE_CHANGESET_OMIT|available return values] for details.
  1018   1017   **
  1019   1018   ** <dl>
  1020   1019   ** <dt>DELETE Changes<dd>
  1021         -**   For each DELETE change, this function checks if the target database 
         1020  +**   For each DELETE change, the function checks if the target database 
  1022   1021   **   contains a row with the same primary key value (or values) as the 
  1023   1022   **   original row values stored in the changeset. If it does, and the values 
  1024   1023   **   stored in all non-primary key columns also match the values stored in 
  1025   1024   **   the changeset the row is deleted from the target database.
  1026   1025   **
  1027   1026   **   If a row with matching primary key values is found, but one or more of
  1028   1027   **   the non-primary key fields contains a value different from the original
................................................................................
  1059   1058   **   violation (e.g. NOT NULL or UNIQUE), the conflict handler function is 
  1060   1059   **   invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT].
  1061   1060   **   This includes the case where the INSERT operation is re-attempted because 
  1062   1061   **   an earlier call to the conflict handler function returned 
  1063   1062   **   [SQLITE_CHANGESET_REPLACE].
  1064   1063   **
  1065   1064   ** <dt>UPDATE Changes<dd>
  1066         -**   For each UPDATE change, this function checks if the target database 
         1065  +**   For each UPDATE change, the function checks if the target database 
  1067   1066   **   contains a row with the same primary key value (or values) as the 
  1068   1067   **   original row values stored in the changeset. If it does, and the values 
  1069   1068   **   stored in all modified non-primary key columns also match the values
  1070   1069   **   stored in the changeset the row is updated within the target database.
  1071   1070   **
  1072   1071   **   If a row with matching primary key values is found, but one or more of
  1073   1072   **   the modified non-primary key fields contains a value different from an
................................................................................
  1090   1089   ** </dl>
  1091   1090   **
  1092   1091   ** It is safe to execute SQL statements, including those that write to the
  1093   1092   ** table that the callback related to, from within the xConflict callback.
  1094   1093   ** This can be used to further customize the applications conflict
  1095   1094   ** resolution strategy.
  1096   1095   **
  1097         -** All changes made by this function are enclosed in a savepoint transaction.
         1096  +** All changes made by these functions are enclosed in a savepoint transaction.
  1098   1097   ** If any other error (aside from a constraint failure when attempting to
  1099   1098   ** write to the target database) occurs, then the savepoint transaction is
  1100   1099   ** rolled back, restoring the target database to its original state, and an 
  1101   1100   ** SQLite error code returned.
         1101  +**
         1102  +** If the output parameters (ppRebase) and (pnRebase) are non-NULL and
         1103  +** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2()
         1104  +** may set (*ppRebase) to point to a "rebase" that may be used with the 
         1105  +** sqlite3_rebaser APIs buffer before returning. In this case (*pnRebase)
         1106  +** is set to the size of the buffer in bytes. It is the responsibility of the
         1107  +** caller to eventually free any such buffer using sqlite3_free(). The buffer
         1108  +** is only allocated and populated if one or more conflicts were encountered
         1109  +** while applying the patchset. See comments surrounding the sqlite3_rebaser
         1110  +** APIs for further details.
         1111  +**
         1112  +** The behavior of sqlite3changeset_apply_v2() and its streaming equivalent
         1113  +** may be modified by passing a combination of
         1114  +** [SQLITE_CHANGESETAPPLY_NOSAVEPOINT | supported flags] as the 9th parameter.
         1115  +**
         1116  +** Note that the sqlite3changeset_apply_v2() API is still <b>experimental</b>
         1117  +** and therefore subject to change.
  1102   1118   */
  1103   1119   int sqlite3changeset_apply(
  1104   1120     sqlite3 *db,                    /* Apply change to "main" db of this handle */
  1105   1121     int nChangeset,                 /* Size of changeset in bytes */
  1106   1122     void *pChangeset,               /* Changeset blob */
  1107   1123     int(*xFilter)(
  1108   1124       void *pCtx,                   /* Copy of sixth arg to _apply() */
................................................................................
  1111   1127     int(*xConflict)(
  1112   1128       void *pCtx,                   /* Copy of sixth arg to _apply() */
  1113   1129       int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
  1114   1130       sqlite3_changeset_iter *p     /* Handle describing change and conflict */
  1115   1131     ),
  1116   1132     void *pCtx                      /* First argument passed to xConflict */
  1117   1133   );
         1134  +int sqlite3changeset_apply_v2(
         1135  +  sqlite3 *db,                    /* Apply change to "main" db of this handle */
         1136  +  int nChangeset,                 /* Size of changeset in bytes */
         1137  +  void *pChangeset,               /* Changeset blob */
         1138  +  int(*xFilter)(
         1139  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         1140  +    const char *zTab              /* Table name */
         1141  +  ),
         1142  +  int(*xConflict)(
         1143  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         1144  +    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
         1145  +    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
         1146  +  ),
         1147  +  void *pCtx,                     /* First argument passed to xConflict */
         1148  +  void **ppRebase, int *pnRebase, /* OUT: Rebase data */
         1149  +  int flags                       /* Combination of SESSION_APPLY_* flags */
         1150  +);
         1151  +
         1152  +/*
         1153  +** CAPI3REF: Flags for sqlite3changeset_apply_v2
         1154  +**
         1155  +** The following flags may passed via the 9th parameter to
         1156  +** [sqlite3changeset_apply_v2] and [sqlite3changeset_apply_v2_strm]:
         1157  +**
         1158  +** <dl>
         1159  +** <dt>SQLITE_CHANGESETAPPLY_NOSAVEPOINT <dd>
         1160  +**   Usually, the sessions module encloses all operations performed by
         1161  +**   a single call to apply_v2() or apply_v2_strm() in a [SAVEPOINT]. The
         1162  +**   SAVEPOINT is committed if the changeset or patchset is successfully
         1163  +**   applied, or rolled back if an error occurs. Specifying this flag
         1164  +**   causes the sessions module to omit this savepoint. In this case, if the
         1165  +**   caller has an open transaction or savepoint when apply_v2() is called, 
         1166  +**   it may revert the partially applied changeset by rolling it back.
         1167  +*/
         1168  +#define SQLITE_CHANGESETAPPLY_NOSAVEPOINT   0x0001
  1118   1169   
  1119   1170   /* 
  1120   1171   ** CAPI3REF: Constants Passed To The Conflict Handler
  1121   1172   **
  1122   1173   ** Values that may be passed as the second argument to a conflict-handler.
  1123   1174   **
  1124   1175   ** <dl>
................................................................................
  1208   1259   **   and the call to sqlite3changeset_apply() returns SQLITE_ABORT.
  1209   1260   ** </dl>
  1210   1261   */
  1211   1262   #define SQLITE_CHANGESET_OMIT       0
  1212   1263   #define SQLITE_CHANGESET_REPLACE    1
  1213   1264   #define SQLITE_CHANGESET_ABORT      2
  1214   1265   
         1266  +/* 
         1267  +** CAPI3REF: Rebasing changesets
         1268  +** EXPERIMENTAL
         1269  +**
         1270  +** Suppose there is a site hosting a database in state S0. And that
         1271  +** modifications are made that move that database to state S1 and a
         1272  +** changeset recorded (the "local" changeset). Then, a changeset based
         1273  +** on S0 is received from another site (the "remote" changeset) and 
         1274  +** applied to the database. The database is then in state 
         1275  +** (S1+"remote"), where the exact state depends on any conflict
         1276  +** resolution decisions (OMIT or REPLACE) made while applying "remote".
         1277  +** Rebasing a changeset is to update it to take those conflict 
         1278  +** resolution decisions into account, so that the same conflicts
         1279  +** do not have to be resolved elsewhere in the network. 
         1280  +**
         1281  +** For example, if both the local and remote changesets contain an
         1282  +** INSERT of the same key on "CREATE TABLE t1(a PRIMARY KEY, b)":
         1283  +**
         1284  +**   local:  INSERT INTO t1 VALUES(1, 'v1');
         1285  +**   remote: INSERT INTO t1 VALUES(1, 'v2');
         1286  +**
         1287  +** and the conflict resolution is REPLACE, then the INSERT change is
         1288  +** removed from the local changeset (it was overridden). Or, if the
         1289  +** conflict resolution was "OMIT", then the local changeset is modified
         1290  +** to instead contain:
         1291  +**
         1292  +**           UPDATE t1 SET b = 'v2' WHERE a=1;
         1293  +**
         1294  +** Changes within the local changeset are rebased as follows:
         1295  +**
         1296  +** <dl>
         1297  +** <dt>Local INSERT<dd>
         1298  +**   This may only conflict with a remote INSERT. If the conflict 
         1299  +**   resolution was OMIT, then add an UPDATE change to the rebased
         1300  +**   changeset. Or, if the conflict resolution was REPLACE, add
         1301  +**   nothing to the rebased changeset.
         1302  +**
         1303  +** <dt>Local DELETE<dd>
         1304  +**   This may conflict with a remote UPDATE or DELETE. In both cases the
         1305  +**   only possible resolution is OMIT. If the remote operation was a
         1306  +**   DELETE, then add no change to the rebased changeset. If the remote
         1307  +**   operation was an UPDATE, then the old.* fields of change are updated
         1308  +**   to reflect the new.* values in the UPDATE.
         1309  +**
         1310  +** <dt>Local UPDATE<dd>
         1311  +**   This may conflict with a remote UPDATE or DELETE. If it conflicts
         1312  +**   with a DELETE, and the conflict resolution was OMIT, then the update
         1313  +**   is changed into an INSERT. Any undefined values in the new.* record
         1314  +**   from the update change are filled in using the old.* values from
         1315  +**   the conflicting DELETE. Or, if the conflict resolution was REPLACE,
         1316  +**   the UPDATE change is simply omitted from the rebased changeset.
         1317  +**
         1318  +**   If conflict is with a remote UPDATE and the resolution is OMIT, then
         1319  +**   the old.* values are rebased using the new.* values in the remote
         1320  +**   change. Or, if the resolution is REPLACE, then the change is copied
         1321  +**   into the rebased changeset with updates to columns also updated by
         1322  +**   the conflicting remote UPDATE removed. If this means no columns would 
         1323  +**   be updated, the change is omitted.
         1324  +** </dl>
         1325  +**
         1326  +** A local change may be rebased against multiple remote changes 
         1327  +** simultaneously. If a single key is modified by multiple remote 
         1328  +** changesets, they are combined as follows before the local changeset
         1329  +** is rebased:
         1330  +**
         1331  +** <ul>
         1332  +**    <li> If there has been one or more REPLACE resolutions on a
         1333  +**         key, it is rebased according to a REPLACE.
         1334  +**
         1335  +**    <li> If there have been no REPLACE resolutions on a key, then
         1336  +**         the local changeset is rebased according to the most recent
         1337  +**         of the OMIT resolutions.
         1338  +** </ul>
         1339  +**
         1340  +** Note that conflict resolutions from multiple remote changesets are 
         1341  +** combined on a per-field basis, not per-row. This means that in the 
         1342  +** case of multiple remote UPDATE operations, some fields of a single 
         1343  +** local change may be rebased for REPLACE while others are rebased for 
         1344  +** OMIT.
         1345  +**
         1346  +** In order to rebase a local changeset, the remote changeset must first
         1347  +** be applied to the local database using sqlite3changeset_apply_v2() and
         1348  +** the buffer of rebase information captured. Then:
         1349  +**
         1350  +** <ol>
         1351  +**   <li> An sqlite3_rebaser object is created by calling 
         1352  +**        sqlite3rebaser_create().
         1353  +**   <li> The new object is configured with the rebase buffer obtained from
         1354  +**        sqlite3changeset_apply_v2() by calling sqlite3rebaser_configure().
         1355  +**        If the local changeset is to be rebased against multiple remote
         1356  +**        changesets, then sqlite3rebaser_configure() should be called
         1357  +**        multiple times, in the same order that the multiple
         1358  +**        sqlite3changeset_apply_v2() calls were made.
         1359  +**   <li> Each local changeset is rebased by calling sqlite3rebaser_rebase().
         1360  +**   <li> The sqlite3_rebaser object is deleted by calling
         1361  +**        sqlite3rebaser_delete().
         1362  +** </ol>
         1363  +*/
         1364  +typedef struct sqlite3_rebaser sqlite3_rebaser;
         1365  +
         1366  +/*
         1367  +** CAPI3REF: Create a changeset rebaser object.
         1368  +** EXPERIMENTAL
         1369  +**
         1370  +** Allocate a new changeset rebaser object. If successful, set (*ppNew) to
         1371  +** point to the new object and return SQLITE_OK. Otherwise, if an error
         1372  +** occurs, return an SQLite error code (e.g. SQLITE_NOMEM) and set (*ppNew) 
         1373  +** to NULL. 
         1374  +*/
         1375  +int sqlite3rebaser_create(sqlite3_rebaser **ppNew);
         1376  +
         1377  +/*
         1378  +** CAPI3REF: Configure a changeset rebaser object.
         1379  +** EXPERIMENTAL
         1380  +**
         1381  +** Configure the changeset rebaser object to rebase changesets according
         1382  +** to the conflict resolutions described by buffer pRebase (size nRebase
         1383  +** bytes), which must have been obtained from a previous call to
         1384  +** sqlite3changeset_apply_v2().
         1385  +*/
         1386  +int sqlite3rebaser_configure(
         1387  +  sqlite3_rebaser*, 
         1388  +  int nRebase, const void *pRebase
         1389  +); 
         1390  +
         1391  +/*
         1392  +** CAPI3REF: Rebase a changeset
         1393  +** EXPERIMENTAL
         1394  +**
         1395  +** Argument pIn must point to a buffer containing a changeset nIn bytes
         1396  +** in size. This function allocates and populates a buffer with a copy
         1397  +** of the changeset rebased rebased according to the configuration of the
         1398  +** rebaser object passed as the first argument. If successful, (*ppOut)
         1399  +** is set to point to the new buffer containing the rebased changset and 
         1400  +** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the
         1401  +** responsibility of the caller to eventually free the new buffer using
         1402  +** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut)
         1403  +** are set to zero and an SQLite error code returned.
         1404  +*/
         1405  +int sqlite3rebaser_rebase(
         1406  +  sqlite3_rebaser*,
         1407  +  int nIn, const void *pIn, 
         1408  +  int *pnOut, void **ppOut 
         1409  +);
         1410  +
         1411  +/*
         1412  +** CAPI3REF: Delete a changeset rebaser object.
         1413  +** EXPERIMENTAL
         1414  +**
         1415  +** Delete the changeset rebaser object and all associated resources. There
         1416  +** should be one call to this function for each successful invocation
         1417  +** of sqlite3rebaser_create().
         1418  +*/
         1419  +void sqlite3rebaser_delete(sqlite3_rebaser *p); 
         1420  +
  1215   1421   /*
  1216   1422   ** CAPI3REF: Streaming Versions of API functions.
  1217   1423   **
  1218   1424   ** The six streaming API xxx_strm() functions serve similar purposes to the 
  1219   1425   ** corresponding non-streaming API functions:
  1220   1426   **
  1221   1427   ** <table border=1 style="margin-left:8ex;margin-right:8ex">
  1222   1428   **   <tr><th>Streaming function<th>Non-streaming equivalent</th>
  1223   1429   **   <tr><td>sqlite3changeset_apply_strm<td>[sqlite3changeset_apply] 
         1430  +**   <tr><td>sqlite3changeset_apply_strm_v2<td>[sqlite3changeset_apply_v2] 
  1224   1431   **   <tr><td>sqlite3changeset_concat_strm<td>[sqlite3changeset_concat] 
  1225   1432   **   <tr><td>sqlite3changeset_invert_strm<td>[sqlite3changeset_invert] 
  1226   1433   **   <tr><td>sqlite3changeset_start_strm<td>[sqlite3changeset_start] 
  1227   1434   **   <tr><td>sqlite3session_changeset_strm<td>[sqlite3session_changeset] 
  1228   1435   **   <tr><td>sqlite3session_patchset_strm<td>[sqlite3session_patchset] 
  1229   1436   ** </table>
  1230   1437   **
................................................................................
  1311   1518     ),
  1312   1519     int(*xConflict)(
  1313   1520       void *pCtx,                   /* Copy of sixth arg to _apply() */
  1314   1521       int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
  1315   1522       sqlite3_changeset_iter *p     /* Handle describing change and conflict */
  1316   1523     ),
  1317   1524     void *pCtx                      /* First argument passed to xConflict */
         1525  +);
         1526  +int sqlite3changeset_apply_v2_strm(
         1527  +  sqlite3 *db,                    /* Apply change to "main" db of this handle */
         1528  +  int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */
         1529  +  void *pIn,                                          /* First arg for xInput */
         1530  +  int(*xFilter)(
         1531  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         1532  +    const char *zTab              /* Table name */
         1533  +  ),
         1534  +  int(*xConflict)(
         1535  +    void *pCtx,                   /* Copy of sixth arg to _apply() */
         1536  +    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
         1537  +    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
         1538  +  ),
         1539  +  void *pCtx,                     /* First argument passed to xConflict */
         1540  +  void **ppRebase, int *pnRebase,
         1541  +  int flags
  1318   1542   );
  1319   1543   int sqlite3changeset_concat_strm(
  1320   1544     int (*xInputA)(void *pIn, void *pData, int *pnData),
  1321   1545     void *pInA,
  1322   1546     int (*xInputB)(void *pIn, void *pData, int *pnData),
  1323   1547     void *pInB,
  1324   1548     int (*xOutput)(void *pOut, const void *pData, int nData),
................................................................................
  1348   1572   int sqlite3changegroup_add_strm(sqlite3_changegroup*, 
  1349   1573       int (*xInput)(void *pIn, void *pData, int *pnData),
  1350   1574       void *pIn
  1351   1575   );
  1352   1576   int sqlite3changegroup_output_strm(sqlite3_changegroup*,
  1353   1577       int (*xOutput)(void *pOut, const void *pData, int nData), 
  1354   1578       void *pOut
         1579  +);
         1580  +int sqlite3rebaser_rebase_strm(
         1581  +  sqlite3_rebaser *pRebaser,
         1582  +  int (*xInput)(void *pIn, void *pData, int *pnData),
         1583  +  void *pIn,
         1584  +  int (*xOutput)(void *pOut, const void *pData, int nData),
         1585  +  void *pOut
  1355   1586   );
  1356   1587   
  1357   1588   
  1358   1589   /*
  1359   1590   ** Make sure we can call this stuff from C++.
  1360   1591   */
  1361   1592   #ifdef __cplusplus
  1362   1593   }
  1363   1594   #endif
  1364   1595   
  1365   1596   #endif  /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */

Changes to ext/session/test_session.c.

     9      9   #  include "sqlite_tcl.h"
    10     10   #else
    11     11   #  include "tcl.h"
    12     12   #  ifndef SQLITE_TCLAPI
    13     13   #    define SQLITE_TCLAPI
    14     14   #  endif
    15     15   #endif
           16  +
           17  +#ifndef SQLITE_AMALGAMATION
           18  +  typedef unsigned char u8;
           19  +#endif
    16     20   
    17     21   typedef struct TestSession TestSession;
    18     22   struct TestSession {
    19     23     sqlite3_session *pSession;
    20     24     Tcl_Interp *interp;
    21     25     Tcl_Obj *pFilterScript;
    22     26   };
................................................................................
   712    716     }
   713    717   
   714    718     *pnData = nRet;
   715    719     return SQLITE_OK;
   716    720   }
   717    721   
   718    722   
   719         -/*
   720         -** sqlite3changeset_apply DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?
   721         -*/
   722         -static int SQLITE_TCLAPI test_sqlite3changeset_apply(
          723  +static int SQLITE_TCLAPI testSqlite3changesetApply(
          724  +  int bV2,
   723    725     void * clientData,
   724    726     Tcl_Interp *interp,
   725    727     int objc,
   726    728     Tcl_Obj *CONST objv[]
   727    729   ){
   728    730     sqlite3 *db;                    /* Database handle */
   729    731     Tcl_CmdInfo info;               /* Database Tcl command (objv[1]) info */
   730    732     int rc;                         /* Return code from changeset_invert() */
   731    733     void *pChangeset;               /* Buffer containing changeset */
   732    734     int nChangeset;                 /* Size of buffer aChangeset in bytes */
   733    735     TestConflictHandler ctx;
   734    736     TestStreamInput sStr;
          737  +  void *pRebase = 0;
          738  +  int nRebase = 0;
          739  +  int flags = 0;                  /* Flags for apply_v2() */
   735    740   
   736    741     memset(&sStr, 0, sizeof(sStr));
   737    742     sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR);
          743  +
          744  +  /* Check for the -nosavepoint flag */
          745  +  if( bV2 && objc>1 ){
          746  +    const char *z1 = Tcl_GetString(objv[1]);
          747  +    int n = strlen(z1);
          748  +    if( n>1 && n<=12 && 0==sqlite3_strnicmp("-nosavepoint", z1, n) ){
          749  +      flags = SQLITE_CHANGESETAPPLY_NOSAVEPOINT;
          750  +      objc--;
          751  +      objv++;
          752  +    }
          753  +  }
   738    754   
   739    755     if( objc!=4 && objc!=5 ){
   740         -    Tcl_WrongNumArgs(interp, 1, objv, 
   741         -        "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?"
   742         -    );
          756  +    const char *zMsg;
          757  +    if( bV2 ){
          758  +      zMsg = "?-nosavepoint? DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?";
          759  +    }else{
          760  +      zMsg = "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?";
          761  +    }
          762  +    Tcl_WrongNumArgs(interp, 1, objv, zMsg);
   743    763       return TCL_ERROR;
   744    764     }
   745    765     if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &info) ){
   746         -    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0);
          766  +    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[1]), 0);
   747    767       return TCL_ERROR;
   748    768     }
   749    769     db = *(sqlite3 **)info.objClientData;
   750    770     pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset);
   751    771     ctx.pConflictScript = objv[3];
   752    772     ctx.pFilterScript = objc==5 ? objv[4] : 0;
   753    773     ctx.interp = interp;
   754    774   
   755    775     if( sStr.nStream==0 ){
   756         -    rc = sqlite3changeset_apply(db, nChangeset, pChangeset, 
   757         -        (objc==5) ? test_filter_handler : 0, test_conflict_handler, (void *)&ctx
   758         -    );
          776  +    if( bV2==0 ){
          777  +      rc = sqlite3changeset_apply(db, nChangeset, pChangeset, 
          778  +          (objc==5)?test_filter_handler:0, test_conflict_handler, (void *)&ctx
          779  +      );
          780  +    }else{
          781  +      rc = sqlite3changeset_apply_v2(db, nChangeset, pChangeset, 
          782  +          (objc==5)?test_filter_handler:0, test_conflict_handler, (void *)&ctx,
          783  +          &pRebase, &nRebase, flags
          784  +      );
          785  +    }
   759    786     }else{
   760    787       sStr.aData = (unsigned char*)pChangeset;
   761    788       sStr.nData = nChangeset;
   762         -    rc = sqlite3changeset_apply_strm(db, testStreamInput, (void*)&sStr,
   763         -        (objc==5) ? test_filter_handler : 0, test_conflict_handler, (void *)&ctx
   764         -    );
          789  +    if( bV2==0 ){
          790  +      rc = sqlite3changeset_apply_strm(db, testStreamInput, (void*)&sStr,
          791  +          (objc==5) ? test_filter_handler : 0, 
          792  +          test_conflict_handler, (void *)&ctx
          793  +      );
          794  +    }else{
          795  +      rc = sqlite3changeset_apply_v2_strm(db, testStreamInput, (void*)&sStr,
          796  +          (objc==5) ? test_filter_handler : 0, 
          797  +          test_conflict_handler, (void *)&ctx,
          798  +          &pRebase, &nRebase, flags
          799  +      );
          800  +    }
   765    801     }
   766    802   
   767    803     if( rc!=SQLITE_OK ){
   768    804       return test_session_error(interp, rc, 0);
          805  +  }else{
          806  +    Tcl_ResetResult(interp);
          807  +    if( bV2 && pRebase ){
          808  +      Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pRebase, nRebase));
          809  +    }
   769    810     }
   770         -  Tcl_ResetResult(interp);
          811  +  sqlite3_free(pRebase);
   771    812     return TCL_OK;
   772    813   }
          814  +
          815  +/*
          816  +** sqlite3changeset_apply DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?
          817  +*/
          818  +static int SQLITE_TCLAPI test_sqlite3changeset_apply(
          819  +  void * clientData,
          820  +  Tcl_Interp *interp,
          821  +  int objc,
          822  +  Tcl_Obj *CONST objv[]
          823  +){
          824  +  return testSqlite3changesetApply(0, clientData, interp, objc, objv);
          825  +}
          826  +/*
          827  +** sqlite3changeset_apply_v2 DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?
          828  +*/
          829  +static int SQLITE_TCLAPI test_sqlite3changeset_apply_v2(
          830  +  void * clientData,
          831  +  Tcl_Interp *interp,
          832  +  int objc,
          833  +  Tcl_Obj *CONST objv[]
          834  +){
          835  +  return testSqlite3changesetApply(1, clientData, interp, objc, objv);
          836  +}
   773    837   
   774    838   /*
   775    839   ** sqlite3changeset_apply_replace_all DB CHANGESET 
   776    840   */
   777    841   static int SQLITE_TCLAPI test_sqlite3changeset_apply_replace_all(
   778    842     void * clientData,
   779    843     Tcl_Interp *interp,
................................................................................
  1140   1204     Tcl_CreateObjCommand(
  1141   1205         interp, Tcl_GetString(objv[1]), test_changebatch_cmd, (ClientData)p,
  1142   1206         test_changebatch_del
  1143   1207     );
  1144   1208     Tcl_SetObjResult(interp, objv[1]);
  1145   1209     return TCL_OK;
  1146   1210   }
         1211  +
         1212  +/*
         1213  +** tclcmd: CMD configure REBASE-BLOB
         1214  +** tclcmd: CMD rebase CHANGESET
         1215  +** tclcmd: CMD delete
         1216  +*/
         1217  +static int SQLITE_TCLAPI test_rebaser_cmd(
         1218  +  void * clientData,
         1219  +  Tcl_Interp *interp,
         1220  +  int objc,
         1221  +  Tcl_Obj *CONST objv[]
         1222  +){
         1223  +  struct RebaseSubcmd {
         1224  +    const char *zSub;
         1225  +    int nArg;
         1226  +    const char *zMsg;
         1227  +    int iSub;
         1228  +  } aSub[] = {
         1229  +    { "configure",    1, "REBASE-BLOB" }, /* 0 */
         1230  +    { "delete",       0, ""            }, /* 1 */
         1231  +    { "rebase",       1, "CHANGESET"   }, /* 2 */
         1232  +    { 0 }
         1233  +  };
         1234  +
         1235  +  sqlite3_rebaser *p = (sqlite3_rebaser*)clientData;
         1236  +  int iSub;
         1237  +  int rc;
         1238  +
         1239  +  if( objc<2 ){
         1240  +    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
         1241  +    return TCL_ERROR;
         1242  +  }
         1243  +  rc = Tcl_GetIndexFromObjStruct(interp, 
         1244  +      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
         1245  +  );
         1246  +  if( rc!=TCL_OK ) return rc;
         1247  +  if( objc!=2+aSub[iSub].nArg ){
         1248  +    Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
         1249  +    return TCL_ERROR;
         1250  +  }
         1251  +
         1252  +  assert( iSub==0 || iSub==1 || iSub==2 );
         1253  +  assert( rc==SQLITE_OK );
         1254  +  switch( iSub ){
         1255  +    case 0: {   /* configure */
         1256  +      int nRebase = 0;
         1257  +      unsigned char *pRebase = Tcl_GetByteArrayFromObj(objv[2], &nRebase);
         1258  +      rc = sqlite3rebaser_configure(p, nRebase, pRebase);
         1259  +      break;
         1260  +    }
         1261  +
         1262  +    case 1:     /* delete */
         1263  +      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
         1264  +      break;
         1265  +
         1266  +    default: {  /* rebase */
         1267  +      TestStreamInput sStr;                 /* Input stream */
         1268  +      TestSessionsBlob sOut;                /* Output blob */
         1269  +
         1270  +      memset(&sStr, 0, sizeof(sStr));
         1271  +      memset(&sOut, 0, sizeof(sOut));
         1272  +      sStr.aData = Tcl_GetByteArrayFromObj(objv[2], &sStr.nData);
         1273  +      sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR);
         1274  +
         1275  +      if( sStr.nStream ){
         1276  +        rc = sqlite3rebaser_rebase_strm(p, 
         1277  +            testStreamInput, (void*)&sStr,
         1278  +            testStreamOutput, (void*)&sOut
         1279  +        );
         1280  +      }else{
         1281  +        rc = sqlite3rebaser_rebase(p, sStr.nData, sStr.aData, &sOut.n, &sOut.p);
         1282  +      }
         1283  +
         1284  +      if( rc==SQLITE_OK ){
         1285  +        Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(sOut.p, sOut.n));
         1286  +      }
         1287  +      sqlite3_free(sOut.p);
         1288  +      break;
         1289  +    }
         1290  +  }
         1291  +
         1292  +  if( rc!=SQLITE_OK ){
         1293  +    return test_session_error(interp, rc, 0);
         1294  +  }
         1295  +  return TCL_OK;
         1296  +}
         1297  +
         1298  +static void SQLITE_TCLAPI test_rebaser_del(void *clientData){
         1299  +  sqlite3_rebaser *p = (sqlite3_rebaser*)clientData;
         1300  +  sqlite3rebaser_delete(p);
         1301  +}
         1302  +
         1303  +/*
         1304  +** tclcmd: sqlite3rebaser_create NAME
         1305  +*/
         1306  +static int SQLITE_TCLAPI test_sqlite3rebaser_create(
         1307  +  void * clientData,
         1308  +  Tcl_Interp *interp,
         1309  +  int objc,
         1310  +  Tcl_Obj *CONST objv[]
         1311  +){
         1312  +  int rc;
         1313  +  sqlite3_rebaser *pNew = 0;
         1314  +  if( objc!=2 ){
         1315  +    Tcl_WrongNumArgs(interp, 1, objv, "NAME");
         1316  +    return SQLITE_ERROR;
         1317  +  }
         1318  +
         1319  +  rc = sqlite3rebaser_create(&pNew);
         1320  +  if( rc!=SQLITE_OK ){
         1321  +    return test_session_error(interp, rc, 0);
         1322  +  }
         1323  +
         1324  +  Tcl_CreateObjCommand(interp, Tcl_GetString(objv[1]), test_rebaser_cmd,
         1325  +      (ClientData)pNew, test_rebaser_del
         1326  +  );
         1327  +  Tcl_SetObjResult(interp, objv[1]);
         1328  +  return TCL_OK;
         1329  +}
  1147   1330   
  1148   1331   int TestSession_Init(Tcl_Interp *interp){
  1149   1332     struct Cmd {
  1150   1333       const char *zCmd;
  1151   1334       Tcl_ObjCmdProc *xProc;
  1152   1335     } aCmd[] = {
  1153   1336       { "sqlite3session", test_sqlite3session },
  1154   1337       { "sqlite3session_foreach", test_sqlite3session_foreach },
  1155   1338       { "sqlite3changeset_invert", test_sqlite3changeset_invert },
  1156   1339       { "sqlite3changeset_concat", test_sqlite3changeset_concat },
  1157   1340       { "sqlite3changeset_apply", test_sqlite3changeset_apply },
         1341  +    { "sqlite3changeset_apply_v2", test_sqlite3changeset_apply_v2 },
  1158   1342       { "sqlite3changeset_apply_replace_all", 
  1159   1343         test_sqlite3changeset_apply_replace_all },
  1160   1344       { "sql_exec_changeset", test_sql_exec_changeset },
         1345  +    { "sqlite3rebaser_create", test_sqlite3rebaser_create },
  1161   1346     };
  1162   1347     int i;
  1163   1348   
  1164   1349     for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
  1165   1350       struct Cmd *p = &aCmd[i];
  1166   1351       Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
  1167   1352     }

Changes to main.mk.

    68     68            memdb.o memjournal.o \
    69     69            mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \
    70     70            notify.o opcodes.o os.o os_unix.o os_win.o \
    71     71            pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \
    72     72            random.o resolve.o rowset.o rtree.o \
    73     73            select.o sqlite3rbu.o status.o stmt.o \
    74     74            table.o threads.o tokenize.o treeview.o trigger.o \
    75         -         update.o userauth.o util.o vacuum.o \
           75  +         update.o upsert.o userauth.o util.o vacuum.o \
    76     76            vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
    77     77   	 vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \
    78     78            utf.o vtab.o
    79     79   
    80     80   LIBOBJ += sqlite3session.o
    81     81   
    82     82   # All of the source code files.
................................................................................
   158    158     $(TOP)/src/tclsqlite.c \
   159    159     $(TOP)/src/threads.c \
   160    160     $(TOP)/src/tokenize.c \
   161    161     $(TOP)/src/treeview.c \
   162    162     $(TOP)/src/trigger.c \
   163    163     $(TOP)/src/utf.c \
   164    164     $(TOP)/src/update.c \
          165  +  $(TOP)/src/upsert.c \
   165    166     $(TOP)/src/util.c \
   166    167     $(TOP)/src/vacuum.c \
   167    168     $(TOP)/src/vdbe.c \
   168    169     $(TOP)/src/vdbe.h \
   169    170     $(TOP)/src/vdbeapi.c \
   170    171     $(TOP)/src/vdbeaux.c \
   171    172     $(TOP)/src/vdbeblob.c \

Changes to src/analyze.c.

  1011   1011     if( v==0 || NEVER(pTab==0) ){
  1012   1012       return;
  1013   1013     }
  1014   1014     if( pTab->tnum==0 ){
  1015   1015       /* Do not gather statistics on views or virtual tables */
  1016   1016       return;
  1017   1017     }
  1018         -  if( sqlite3_strlike("sqlite_%", pTab->zName, 0)==0 ){
         1018  +  if( sqlite3_strlike("sqlite\\_%", pTab->zName, '\\')==0 ){
  1019   1019       /* Do not gather statistics on system tables */
  1020   1020       return;
  1021   1021     }
  1022   1022     assert( sqlite3BtreeHoldsAllMutexes(db) );
  1023   1023     iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
  1024   1024     assert( iDb>=0 );
  1025   1025     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );

Changes to src/attach.c.

   498    498         pItem->zDatabase = 0;
   499    499         pItem->pSchema = pFix->pSchema;
   500    500       }
   501    501   #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER)
   502    502       if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1;
   503    503       if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1;
   504    504   #endif
          505  +    if( pItem->fg.isTabFunc && sqlite3FixExprList(pFix, pItem->u1.pFuncArg) ){
          506  +      return 1;
          507  +    }
   505    508     }
   506    509     return 0;
   507    510   }
   508    511   #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER)
   509    512   int sqlite3FixSelect(
   510    513     DbFixer *pFix,       /* Context of the fixation */
   511    514     Select *pSelect      /* The SELECT statement to be fixed to one database */
................................................................................
   597    600       }
   598    601       if( sqlite3FixExpr(pFix, pStep->pWhere) ){
   599    602         return 1;
   600    603       }
   601    604       if( sqlite3FixExprList(pFix, pStep->pExprList) ){
   602    605         return 1;
   603    606       }
          607  +#ifndef SQLITE_OMIT_UPSERT
          608  +    if( pStep->pUpsert ){
          609  +      Upsert *pUp = pStep->pUpsert;
          610  +      if( sqlite3FixExprList(pFix, pUp->pUpsertTarget)
          611  +       || sqlite3FixExpr(pFix, pUp->pUpsertTargetWhere)
          612  +       || sqlite3FixExprList(pFix, pUp->pUpsertSet)
          613  +       || sqlite3FixExpr(pFix, pUp->pUpsertWhere)
          614  +      ){
          615  +        return 1;
          616  +      }
          617  +    }
          618  +#endif
   604    619       pStep = pStep->pNext;
   605    620     }
   606    621     return 0;
   607    622   }
   608    623   #endif

Changes to src/btree.c.

  2478   2478   /*
  2479   2479   ** Invoke the busy handler for a btree.
  2480   2480   */
  2481   2481   static int btreeInvokeBusyHandler(void *pArg){
  2482   2482     BtShared *pBt = (BtShared*)pArg;
  2483   2483     assert( pBt->db );
  2484   2484     assert( sqlite3_mutex_held(pBt->db->mutex) );
  2485         -  return sqlite3InvokeBusyHandler(&pBt->db->busyHandler);
         2485  +  return sqlite3InvokeBusyHandler(&pBt->db->busyHandler,
         2486  +                                  sqlite3PagerFile(pBt->pPager));
  2486   2487   }
  2487   2488   
  2488   2489   /*
  2489   2490   ** Open a database file.
  2490   2491   ** 
  2491   2492   ** zFilename is the name of the database file.  If zFilename is NULL
  2492   2493   ** then an ephemeral database is created.  The ephemeral database might
................................................................................
  2656   2657         rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader);
  2657   2658       }
  2658   2659       if( rc!=SQLITE_OK ){
  2659   2660         goto btree_open_out;
  2660   2661       }
  2661   2662       pBt->openFlags = (u8)flags;
  2662   2663       pBt->db = db;
  2663         -    sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt);
         2664  +    sqlite3PagerSetBusyHandler(pBt->pPager, btreeInvokeBusyHandler, pBt);
  2664   2665       p->pBt = pBt;
  2665   2666     
  2666   2667       pBt->pCursor = 0;
  2667   2668       pBt->pPage1 = 0;
  2668   2669       if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY;
  2669   2670   #if defined(SQLITE_SECURE_DELETE)
  2670   2671       pBt->btsFlags |= BTS_SECURE_DELETE;
................................................................................
  3621   3622       }
  3622   3623     
  3623   3624       if( rc!=SQLITE_OK ){
  3624   3625         unlockBtreeIfUnused(pBt);
  3625   3626       }
  3626   3627     }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE &&
  3627   3628             btreeInvokeBusyHandler(pBt) );
         3629  +  sqlite3PagerResetLockTimeout(pBt->pPager);
  3628   3630   
  3629   3631     if( rc==SQLITE_OK ){
  3630   3632       if( p->inTrans==TRANS_NONE ){
  3631   3633         pBt->nTransaction++;
  3632   3634   #ifndef SQLITE_OMIT_SHARED_CACHE
  3633   3635         if( p->sharable ){
  3634   3636           assert( p->lock.pBtree==p && p->lock.iTable==1 );

Changes to src/build.c.

  1093   1093     pCol = &p->aCol[p->nCol];
  1094   1094     memset(pCol, 0, sizeof(p->aCol[0]));
  1095   1095     pCol->zName = z;
  1096   1096     sqlite3ColumnPropertiesFromName(p, pCol);
  1097   1097    
  1098   1098     if( pType->n==0 ){
  1099   1099       /* If there is no type specified, columns have the default affinity
  1100         -    ** 'BLOB'. */
         1100  +    ** 'BLOB' with a default size of 4 bytes. */
  1101   1101       pCol->affinity = SQLITE_AFF_BLOB;
  1102   1102       pCol->szEst = 1;
         1103  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1104  +    if( 4>=sqlite3GlobalConfig.szSorterRef ){
         1105  +      pCol->colFlags |= COLFLAG_SORTERREF;
         1106  +    }
         1107  +#endif
  1103   1108     }else{
  1104   1109       zType = z + sqlite3Strlen30(z) + 1;
  1105   1110       memcpy(zType, pType->z, pType->n);
  1106   1111       zType[pType->n] = 0;
  1107   1112       sqlite3Dequote(zType);
  1108         -    pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst);
         1113  +    pCol->affinity = sqlite3AffinityType(zType, pCol);
  1109   1114       pCol->colFlags |= COLFLAG_HASTYPE;
  1110   1115     }
  1111   1116     p->nCol++;
  1112   1117     pParse->constraintName.n = 0;
  1113   1118   }
  1114   1119   
  1115   1120   /*
................................................................................
  1161   1166   ** 'REAL'        | SQLITE_AFF_REAL
  1162   1167   ** 'FLOA'        | SQLITE_AFF_REAL
  1163   1168   ** 'DOUB'        | SQLITE_AFF_REAL
  1164   1169   **
  1165   1170   ** If none of the substrings in the above table are found,
  1166   1171   ** SQLITE_AFF_NUMERIC is returned.
  1167   1172   */
  1168         -char sqlite3AffinityType(const char *zIn, u8 *pszEst){
         1173  +char sqlite3AffinityType(const char *zIn, Column *pCol){
  1169   1174     u32 h = 0;
  1170   1175     char aff = SQLITE_AFF_NUMERIC;
  1171   1176     const char *zChar = 0;
  1172   1177   
  1173   1178     assert( zIn!=0 );
  1174   1179     while( zIn[0] ){
  1175   1180       h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff];
................................................................................
  1198   1203   #endif
  1199   1204       }else if( (h&0x00FFFFFF)==(('i'<<16)+('n'<<8)+'t') ){    /* INT */
  1200   1205         aff = SQLITE_AFF_INTEGER;
  1201   1206         break;
  1202   1207       }
  1203   1208     }
  1204   1209   
  1205         -  /* If pszEst is not NULL, store an estimate of the field size.  The
         1210  +  /* If pCol is not NULL, store an estimate of the field size.  The
  1206   1211     ** estimate is scaled so that the size of an integer is 1.  */
  1207         -  if( pszEst ){
  1208         -    *pszEst = 1;   /* default size is approx 4 bytes */
         1212  +  if( pCol ){
         1213  +    int v = 0;   /* default size is approx 4 bytes */
  1209   1214       if( aff<SQLITE_AFF_NUMERIC ){
  1210   1215         if( zChar ){
  1211   1216           while( zChar[0] ){
  1212   1217             if( sqlite3Isdigit(zChar[0]) ){
  1213         -            int v = 0;
         1218  +            /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */
  1214   1219               sqlite3GetInt32(zChar, &v);
  1215         -            v = v/4 + 1;
  1216         -            if( v>255 ) v = 255;
  1217         -            *pszEst = v; /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */
  1218   1220               break;
  1219   1221             }
  1220   1222             zChar++;
  1221   1223           }
  1222   1224         }else{
  1223         -        *pszEst = 5;   /* BLOB, TEXT, CLOB -> r=5  (approx 20 bytes)*/
         1225  +        v = 16;   /* BLOB, TEXT, CLOB -> r=5  (approx 20 bytes)*/
  1224   1226         }
  1225   1227       }
         1228  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1229  +    if( v>=sqlite3GlobalConfig.szSorterRef ){
         1230  +      pCol->colFlags |= COLFLAG_SORTERREF;
         1231  +    }
         1232  +#endif
         1233  +    v = v/4 + 1;
         1234  +    if( v>255 ) v = 255;
         1235  +    pCol->szEst = v;
  1226   1236     }
  1227   1237     return aff;
  1228   1238   }
  1229   1239   
  1230   1240   /*
  1231   1241   ** The expression is the default value for the most recently added column
  1232   1242   ** of the table currently under construction.
................................................................................
  1250   1260     if( p!=0 ){
  1251   1261       pCol = &(p->aCol[p->nCol-1]);
  1252   1262       if( !sqlite3ExprIsConstantOrFunction(pExpr, db->init.busy) ){
  1253   1263         sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant",
  1254   1264             pCol->zName);
  1255   1265       }else{
  1256   1266         /* A copy of pExpr is used instead of the original, as pExpr contains
  1257         -      ** tokens that point to volatile memory.	
         1267  +      ** tokens that point to volatile memory.
  1258   1268         */
  1259   1269         Expr x;
  1260   1270         sqlite3ExprDelete(db, pCol->pDflt);
  1261   1271         memset(&x, 0, sizeof(x));
  1262   1272         x.op = TK_SPAN;
  1263   1273         x.u.zToken = sqlite3DbSpanDup(db, zStart, zEnd);
  1264   1274         x.pLeft = pExpr;
................................................................................
  1494   1504   ** the schema-version whenever the schema changes.
  1495   1505   */
  1496   1506   void sqlite3ChangeCookie(Parse *pParse, int iDb){
  1497   1507     sqlite3 *db = pParse->db;
  1498   1508     Vdbe *v = pParse->pVdbe;
  1499   1509     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
  1500   1510     sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, 
  1501         -                    db->aDb[iDb].pSchema->schema_cookie+1);
         1511  +                   (int)(1+(unsigned)db->aDb[iDb].pSchema->schema_cookie));
  1502   1512   }
  1503   1513   
  1504   1514   /*
  1505   1515   ** Measure the number of characters needed to output the given
  1506   1516   ** identifier.  The number returned includes any quotes used
  1507   1517   ** but does not include the null terminator.
  1508   1518   **
................................................................................
  2172   2182   */
  2173   2183   int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
  2174   2184     Table *pSelTab;   /* A fake table from which we get the result set */
  2175   2185     Select *pSel;     /* Copy of the SELECT that implements the view */
  2176   2186     int nErr = 0;     /* Number of errors encountered */
  2177   2187     int n;            /* Temporarily holds the number of cursors assigned */
  2178   2188     sqlite3 *db = pParse->db;  /* Database connection for malloc errors */
  2179         -#ifndef SQLITE_OMIT_VIRTUALTABLE	
         2189  +#ifndef SQLITE_OMIT_VIRTUALTABLE
  2180   2190     int rc;
  2181   2191   #endif
  2182   2192   #ifndef SQLITE_OMIT_AUTHORIZATION
  2183   2193     sqlite3_xauth xAuth;       /* Saved xAuth pointer */
  2184   2194   #endif
  2185   2195   
  2186   2196     assert( pTable );

Changes to src/ctime.c.

   281    281     "ENABLE_SELECTTRACE",
   282    282   #endif
   283    283   #if SQLITE_ENABLE_SESSION
   284    284     "ENABLE_SESSION",
   285    285   #endif
   286    286   #if SQLITE_ENABLE_SNAPSHOT
   287    287     "ENABLE_SNAPSHOT",
          288  +#endif
          289  +#if SQLITE_ENABLE_SORTER_REFERENCES
          290  +  "ENABLE_SORTER_REFERENCES",
   288    291   #endif
   289    292   #if SQLITE_ENABLE_SQLLOG
   290    293     "ENABLE_SQLLOG",
   291    294   #endif
   292    295   #if defined(SQLITE_ENABLE_STAT4)
   293    296     "ENABLE_STAT4",
   294    297   #elif defined(SQLITE_ENABLE_STAT3)

Changes to src/dbstat.c.

   420    420     pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1);
   421    421   
   422    422     /* If connected to a ZIPVFS backend, override the page size and
   423    423     ** offset with actual values obtained from ZIPVFS.
   424    424     */
   425    425     fd = sqlite3PagerFile(pPager);
   426    426     x[0] = pCsr->iPageno;
   427         -  if( fd->pMethods!=0 && sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){
          427  +  if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){
   428    428       pCsr->iOffset = x[0];
   429    429       pCsr->szPage = (int)x[1];
   430    430     }
   431    431   }
   432    432   
   433    433   /*
   434    434   ** Move a statvfs cursor to the next entry in the file.

Changes to src/delete.c.

   234    234     int iDataCur = 0;      /* VDBE cursor for the canonical data source */
   235    235     int iIdxCur = 0;       /* Cursor number of the first index */
   236    236     int nIdx;              /* Number of indices */
   237    237     sqlite3 *db;           /* Main database structure */
   238    238     AuthContext sContext;  /* Authorization context */
   239    239     NameContext sNC;       /* Name context to resolve expressions in */
   240    240     int iDb;               /* Database number */
   241         -  int memCnt = -1;       /* Memory cell used for change counting */
          241  +  int memCnt = 0;        /* Memory cell used for change counting */
   242    242     int rcauth;            /* Value returned by authorization callback */
   243    243     int eOnePass;          /* ONEPASS_OFF or _SINGLE or _MULTI */
   244    244     int aiCurOnePass[2];   /* The write cursors opened by WHERE_ONEPASS */
   245    245     u8 *aToOpen = 0;       /* Open cursor iTabCur+j if aToOpen[j] is true */
   246    246     Index *pPk;            /* The PRIMARY KEY index on the table */
   247    247     int iPk = 0;           /* First of nPk registers holding PRIMARY KEY value */
   248    248     i16 nPk = 1;           /* Number of columns in the PRIMARY KEY */
................................................................................
   339    339     /* Begin generating code.
   340    340     */
   341    341     v = sqlite3GetVdbe(pParse);
   342    342     if( v==0 ){
   343    343       goto delete_from_cleanup;
   344    344     }
   345    345     if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
   346         -  sqlite3BeginWriteOperation(pParse, 1, iDb);
          346  +  sqlite3BeginWriteOperation(pParse, bComplex, iDb);
   347    347   
   348    348     /* If we are trying to delete from a view, realize that view into
   349    349     ** an ephemeral table.
   350    350     */
   351    351   #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   352    352     if( isView ){
   353    353       sqlite3MaterializeView(pParse, pTab, 
................................................................................
   367    367     if( sqlite3ResolveExprNames(&sNC, pWhere) ){
   368    368       goto delete_from_cleanup;
   369    369     }
   370    370   
   371    371     /* Initialize the counter of the number of rows deleted, if
   372    372     ** we are counting rows.
   373    373     */
   374         -  if( db->flags & SQLITE_CountRows ){
          374  +  if( (db->flags & SQLITE_CountRows)!=0
          375  +   && !pParse->nested
          376  +   && !pParse->pTriggerTab
          377  +  ){
   375    378       memCnt = ++pParse->nMem;
   376    379       sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt);
   377    380     }
   378    381   
   379    382   #ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION
   380    383     /* Special case: A DELETE without a WHERE clause deletes everything.
   381    384     ** It is easier just to erase the whole table. Prior to version 3.6.5,
................................................................................
   395    398   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   396    399      && db->xPreUpdateCallback==0
   397    400   #endif
   398    401     ){
   399    402       assert( !isView );
   400    403       sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName);
   401    404       if( HasRowid(pTab) ){
   402         -      sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt,
          405  +      sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt ? memCnt : -1,
   403    406                           pTab->zName, P4_STATIC);
   404    407       }
   405    408       for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
   406    409         assert( pIdx->pSchema==pTab->pSchema );
   407    410         sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb);
   408    411       }
   409    412     }else
................................................................................
   440    443       **  ONEPASS_MULTI:  One-pass approach - any number of rows may be deleted.
   441    444       */
   442    445       pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1);
   443    446       if( pWInfo==0 ) goto delete_from_cleanup;
   444    447       eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
   445    448       assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI );
   446    449       assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF );
          450  +    if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse);
   447    451     
   448    452       /* Keep track of the number of rows to be deleted */
   449         -    if( db->flags & SQLITE_CountRows ){
          453  +    if( memCnt ){
   450    454         sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
   451    455       }
   452    456     
   453    457       /* Extract the rowid or primary key for the current row */
   454    458       if( pPk ){
   455    459         for(i=0; i<nPk; i++){
   456    460           assert( pPk->aiColumn[i]>=0 );
................................................................................
   585    589       sqlite3AutoincrementEnd(pParse);
   586    590     }
   587    591   
   588    592     /* Return the number of rows that were deleted. If this routine is 
   589    593     ** generating code because of a call to sqlite3NestedParse(), do not
   590    594     ** invoke the callback function.
   591    595     */
   592         -  if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){
          596  +  if( memCnt ){
   593    597       sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1);
   594    598       sqlite3VdbeSetNumCols(v, 1);
   595    599       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC);
   596    600     }
   597    601   
   598    602   delete_from_cleanup:
   599    603     sqlite3AuthContextPop(&sContext);

Changes to src/expr.c.

  1359   1359         }
  1360   1360       }
  1361   1361       pItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
  1362   1362       pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan);
  1363   1363       pItem->sortOrder = pOldItem->sortOrder;
  1364   1364       pItem->done = 0;
  1365   1365       pItem->bSpanIsTab = pOldItem->bSpanIsTab;
         1366  +    pItem->bSorterRef = pOldItem->bSorterRef;
  1366   1367       pItem->u = pOldItem->u;
  1367   1368     }
  1368   1369     return pNew;
  1369   1370   }
  1370   1371   
  1371   1372   /*
  1372   1373   ** If cursors, triggers, views and subqueries are all omitted from
................................................................................
  1821   1822         testcase( pExpr->op==TK_AGG_FUNCTION );
  1822   1823         testcase( pExpr->op==TK_AGG_COLUMN );
  1823   1824         if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){
  1824   1825           return WRC_Continue;
  1825   1826         }
  1826   1827         /* Fall through */
  1827   1828       case TK_IF_NULL_ROW:
         1829  +    case TK_REGISTER:
         1830  +      testcase( pExpr->op==TK_REGISTER );
  1828   1831         testcase( pExpr->op==TK_IF_NULL_ROW );
  1829   1832         pWalker->eCode = 0;
  1830   1833         return WRC_Abort;
  1831   1834       case TK_VARIABLE:
  1832   1835         if( pWalker->eCode==5 ){
  1833   1836           /* Silently convert bound parameters that appear inside of CREATE
  1834   1837           ** statements into a NULL when parsing the CREATE statement text out
................................................................................
  1838   1841           /* A bound parameter in a CREATE statement that originates from
  1839   1842           ** sqlite3_prepare() causes an error */
  1840   1843           pWalker->eCode = 0;
  1841   1844           return WRC_Abort;
  1842   1845         }
  1843   1846         /* Fall through */
  1844   1847       default:
  1845         -      testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail will disallow */
  1846         -      testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail will disallow */
         1848  +      testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail() disallows */
         1849  +      testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail() disallows */
  1847   1850         return WRC_Continue;
  1848   1851     }
  1849   1852   }
  1850   1853   static int exprIsConst(Expr *p, int initFlag, int iCur){
  1851   1854     Walker w;
  1852   1855     w.eCode = initFlag;
  1853   1856     w.xExprCallback = exprNodeIsConstant;
................................................................................
  2602   2605     ** If all of the above are false, then we can run this code just once
  2603   2606     ** save the results, and reuse the same result on subsequent invocations.
  2604   2607     */
  2605   2608     if( !ExprHasProperty(pExpr, EP_VarSelect) ){
  2606   2609       jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
  2607   2610     }
  2608   2611   
  2609         -#ifndef SQLITE_OMIT_EXPLAIN
  2610         -  if( pParse->explain==2 ){
  2611         -    char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d",
  2612         -        jmpIfDynamic>=0?"":"CORRELATED ",
  2613         -        pExpr->op==TK_IN?"LIST":"SCALAR",
  2614         -        pParse->iNextSelectId
  2615         -    );
  2616         -    sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
  2617         -  }
  2618         -#endif
  2619         -
  2620   2612     switch( pExpr->op ){
  2621   2613       case TK_IN: {
  2622   2614         int addr;                   /* Address of OP_OpenEphemeral instruction */
  2623   2615         Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */
  2624   2616         KeyInfo *pKeyInfo = 0;      /* Key information */
  2625   2617         int nVal;                   /* Size of vector pLeft */
  2626   2618         
................................................................................
  2649   2641           /* Case 1:     expr IN (SELECT ...)
  2650   2642           **
  2651   2643           ** Generate code to write the results of the select into the temporary
  2652   2644           ** table allocated and opened above.
  2653   2645           */
  2654   2646           Select *pSelect = pExpr->x.pSelect;
  2655   2647           ExprList *pEList = pSelect->pEList;
         2648  +
         2649  +#ifndef SQLITE_OMIT_EXPLAIN
         2650  +        if( pParse->explain==2 ){
         2651  +          char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %sLIST SUBQUERY %d",
         2652  +            jmpIfDynamic>=0?"":"CORRELATED ",
         2653  +            pParse->iNextSelectId
         2654  +          );
         2655  +          sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg,
         2656  +                            P4_DYNAMIC);
         2657  +        }
         2658  +#endif
  2656   2659   
  2657   2660           assert( !isRowid );
  2658   2661           /* If the LHS and RHS of the IN operator do not match, that
  2659   2662           ** error will have been caught long before we reach this point. */
  2660   2663           if( ALWAYS(pEList->nExpr==nVal) ){
  2661   2664             SelectDest dest;
  2662   2665             int i;
................................................................................
  2691   2694           ** a column, use numeric affinity.
  2692   2695           */
  2693   2696           char affinity;            /* Affinity of the LHS of the IN */
  2694   2697           int i;
  2695   2698           ExprList *pList = pExpr->x.pList;
  2696   2699           struct ExprList_item *pItem;
  2697   2700           int r1, r2, r3;
  2698         -
  2699   2701           affinity = sqlite3ExprAffinity(pLeft);
  2700   2702           if( !affinity ){
  2701   2703             affinity = SQLITE_AFF_BLOB;
  2702   2704           }
  2703   2705           if( pKeyInfo ){
  2704   2706             assert( sqlite3KeyInfoIsWriteable(pKeyInfo) );
  2705   2707             pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
................................................................................
  2770   2772         int nReg;                             /* Registers to allocate */
  2771   2773         Expr *pLimit;                         /* New limit expression */
  2772   2774   
  2773   2775         testcase( pExpr->op==TK_EXISTS );
  2774   2776         testcase( pExpr->op==TK_SELECT );
  2775   2777         assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT );
  2776   2778         assert( ExprHasProperty(pExpr, EP_xIsSelect) );
         2779  +
         2780  +#ifndef SQLITE_OMIT_EXPLAIN
         2781  +      if( pParse->explain==2 ){
         2782  +        char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %sSCALAR SUBQUERY %d",
         2783  +            jmpIfDynamic>=0?"":"CORRELATED ",
         2784  +            pParse->iNextSelectId
         2785  +        );
         2786  +        sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg,
         2787  +                          P4_DYNAMIC);
         2788  +      }
         2789  +#endif
  2777   2790   
  2778   2791         pSel = pExpr->x.pSelect;
  2779   2792         nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1;
  2780   2793         sqlite3SelectDestInit(&dest, 0, pParse->nMem+1);
  2781   2794         pParse->nMem += nReg;
  2782   2795         if( pExpr->op==TK_SELECT ){
  2783   2796           dest.eDest = SRT_Mem;
................................................................................
  4367   4380     assert( pList!=0 );
  4368   4381     assert( target>0 );
  4369   4382     assert( pParse->pVdbe!=0 );  /* Never gets this far otherwise */
  4370   4383     n = pList->nExpr;
  4371   4384     if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR;
  4372   4385     for(pItem=pList->a, i=0; i<n; i++, pItem++){
  4373   4386       Expr *pExpr = pItem->pExpr;
         4387  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         4388  +    if( pItem->bSorterRef ){
         4389  +      i--;
         4390  +      n--;
         4391  +    }else
         4392  +#endif
  4374   4393       if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){
  4375   4394         if( flags & SQLITE_ECEL_OMITREF ){
  4376   4395           i--;
  4377   4396           n--;
  4378   4397         }else{
  4379   4398           sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i);
  4380   4399         }
................................................................................
  4895   4914         return 1;
  4896   4915       }
  4897   4916       return 2;
  4898   4917     }
  4899   4918     if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){
  4900   4919       if( pA->op==TK_FUNCTION ){
  4901   4920         if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2;
         4921  +    }else if( pA->op==TK_COLLATE ){
         4922  +      if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2;
  4902   4923       }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){
  4903         -      return pA->op==TK_COLLATE ? 1 : 2;
         4924  +      return 2;
  4904   4925       }
  4905   4926     }
  4906   4927     if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2;
  4907   4928     if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){
  4908   4929       if( combinedFlags & EP_xIsSelect ) return 2;
  4909   4930       if( sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2;
  4910   4931       if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2;
  4911   4932       if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2;
  4912         -    if( ALWAYS((combinedFlags & EP_Reduced)==0) && pA->op!=TK_STRING ){
         4933  +    assert( (combinedFlags & EP_Reduced)==0 );
         4934  +    if( pA->op!=TK_STRING && pA->op!=TK_TRUEFALSE ){
  4913   4935         if( pA->iColumn!=pB->iColumn ) return 2;
  4914   4936         if( pA->iTable!=pB->iTable 
  4915   4937          && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2;
  4916   4938       }
  4917   4939     }
  4918   4940     return 0;
  4919   4941   }
................................................................................
  4996   5018     if( pE2->op==TK_NOTNULL && pE1->op!=TK_ISNULL && pE1->op!=TK_IS ){
  4997   5019       Expr *pX = sqlite3ExprSkipCollate(pE1->pLeft);
  4998   5020       testcase( pX!=pE1->pLeft );
  4999   5021       if( sqlite3ExprCompare(pParse, pX, pE2->pLeft, iTab)==0 ) return 1;
  5000   5022     }
  5001   5023     return 0;
  5002   5024   }
         5025  +
         5026  +/*
         5027  +** This is the Expr node callback for sqlite3ExprImpliesNotNullRow().
         5028  +** If the expression node requires that the table at pWalker->iCur
         5029  +** have a non-NULL column, then set pWalker->eCode to 1 and abort.
         5030  +*/
         5031  +static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){
         5032  +  /* This routine is only called for WHERE clause expressions and so it
         5033  +  ** cannot have any TK_AGG_COLUMN entries because those are only found
         5034  +  ** in HAVING clauses.  We can get a TK_AGG_FUNCTION in a WHERE clause,
         5035  +  ** but that is an illegal construct and the query will be rejected at
         5036  +  ** a later stage of processing, so the TK_AGG_FUNCTION case does not
         5037  +  ** need to be considered here. */
         5038  +  assert( pExpr->op!=TK_AGG_COLUMN );
         5039  +  testcase( pExpr->op==TK_AGG_FUNCTION );
         5040  +
         5041  +  if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune;
         5042  +  switch( pExpr->op ){
         5043  +    case TK_ISNOT:
         5044  +    case TK_NOT:
         5045  +    case TK_ISNULL:
         5046  +    case TK_IS:
         5047  +    case TK_OR:
         5048  +    case TK_CASE:
         5049  +    case TK_IN:
         5050  +    case TK_FUNCTION:
         5051  +      testcase( pExpr->op==TK_ISNOT );
         5052  +      testcase( pExpr->op==TK_NOT );
         5053  +      testcase( pExpr->op==TK_ISNULL );
         5054  +      testcase( pExpr->op==TK_IS );
         5055  +      testcase( pExpr->op==TK_OR );
         5056  +      testcase( pExpr->op==TK_CASE );
         5057  +      testcase( pExpr->op==TK_IN );
         5058  +      testcase( pExpr->op==TK_FUNCTION );
         5059  +      return WRC_Prune;
         5060  +    case TK_COLUMN:
         5061  +      if( pWalker->u.iCur==pExpr->iTable ){
         5062  +        pWalker->eCode = 1;
         5063  +        return WRC_Abort;
         5064  +      }
         5065  +      return WRC_Prune;
         5066  +
         5067  +    /* Virtual tables are allowed to use constraints like x=NULL.  So
         5068  +    ** a term of the form x=y does not prove that y is not null if x
         5069  +    ** is the column of a virtual table */
         5070  +    case TK_EQ:
         5071  +    case TK_NE:
         5072  +    case TK_LT:
         5073  +    case TK_LE:
         5074  +    case TK_GT:
         5075  +    case TK_GE:
         5076  +      testcase( pExpr->op==TK_EQ );
         5077  +      testcase( pExpr->op==TK_NE );
         5078  +      testcase( pExpr->op==TK_LT );
         5079  +      testcase( pExpr->op==TK_LE );
         5080  +      testcase( pExpr->op==TK_GT );
         5081  +      testcase( pExpr->op==TK_GE );
         5082  +      if( (pExpr->pLeft->op==TK_COLUMN && IsVirtual(pExpr->pLeft->pTab))
         5083  +       || (pExpr->pRight->op==TK_COLUMN && IsVirtual(pExpr->pRight->pTab))
         5084  +      ){
         5085  +       return WRC_Prune;
         5086  +      }
         5087  +    default:
         5088  +      return WRC_Continue;
         5089  +  }
         5090  +}
         5091  +
         5092  +/*
         5093  +** Return true (non-zero) if expression p can only be true if at least
         5094  +** one column of table iTab is non-null.  In other words, return true
         5095  +** if expression p will always be NULL or false if every column of iTab
         5096  +** is NULL.
         5097  +**
         5098  +** False negatives are acceptable.  In other words, it is ok to return
         5099  +** zero even if expression p will never be true of every column of iTab
         5100  +** is NULL.  A false negative is merely a missed optimization opportunity.
         5101  +**
         5102  +** False positives are not allowed, however.  A false positive may result
         5103  +** in an incorrect answer.
         5104  +**
         5105  +** Terms of p that are marked with EP_FromJoin (and hence that come from
         5106  +** the ON or USING clauses of LEFT JOINS) are excluded from the analysis.
         5107  +**
         5108  +** This routine is used to check if a LEFT JOIN can be converted into
         5109  +** an ordinary JOIN.  The p argument is the WHERE clause.  If the WHERE
         5110  +** clause requires that some column of the right table of the LEFT JOIN
         5111  +** be non-NULL, then the LEFT JOIN can be safely converted into an
         5112  +** ordinary join.
         5113  +*/
         5114  +int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){
         5115  +  Walker w;
         5116  +  w.xExprCallback = impliesNotNullRow;
         5117  +  w.xSelectCallback = 0;
         5118  +  w.xSelectCallback2 = 0;
         5119  +  w.eCode = 0;
         5120  +  w.u.iCur = iTab;
         5121  +  sqlite3WalkExpr(&w, p);
         5122  +  return w.eCode;
         5123  +}
  5003   5124   
  5004   5125   /*
  5005   5126   ** An instance of the following structure is used by the tree walker
  5006   5127   ** to determine if an expression can be evaluated by reference to the
  5007   5128   ** index only, without having to do a search for the corresponding
  5008   5129   ** table entry.  The IdxCover.pIdx field is the index.  IdxCover.iCur
  5009   5130   ** is the cursor for the table.
................................................................................
  5152   5273   ** for additional information.
  5153   5274   */
  5154   5275   static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
  5155   5276     int i;
  5156   5277     NameContext *pNC = pWalker->u.pNC;
  5157   5278     Parse *pParse = pNC->pParse;
  5158   5279     SrcList *pSrcList = pNC->pSrcList;
  5159         -  AggInfo *pAggInfo = pNC->pAggInfo;
         5280  +  AggInfo *pAggInfo = pNC->uNC.pAggInfo;
  5160   5281   
         5282  +  assert( pNC->ncFlags & NC_UAggInfo );
  5161   5283     switch( pExpr->op ){
  5162   5284       case TK_AGG_COLUMN:
  5163   5285       case TK_COLUMN: {
  5164   5286         testcase( pExpr->op==TK_AGG_COLUMN );
  5165   5287         testcase( pExpr->op==TK_COLUMN );
  5166   5288         /* Check to see if the column is in one of the tables in the FROM
  5167   5289         ** clause of the aggregate query */

Changes to src/global.c.

   236    236      0,                         /* xVdbeBranch */
   237    237      0,                         /* pVbeBranchArg */
   238    238   #endif
   239    239   #ifndef SQLITE_UNTESTABLE
   240    240      0,                         /* xTestCallback */
   241    241   #endif
   242    242      0,                         /* bLocaltimeFault */
   243         -   0x7ffffffe                 /* iOnceResetThreshold */
          243  +   0x7ffffffe,                /* iOnceResetThreshold */
          244  +   SQLITE_DEFAULT_SORTERREF_SIZE   /* szSorterRef */
   244    245   };
   245    246   
   246    247   /*
   247    248   ** Hash table for global functions - functions common to all
   248    249   ** database connections.  After initialization, this table is
   249    250   ** read-only.
   250    251   */

Changes to src/insert.c.

   484    484   **      D: cleanup
   485    485   */
   486    486   void sqlite3Insert(
   487    487     Parse *pParse,        /* Parser context */
   488    488     SrcList *pTabList,    /* Name of table into which we are inserting */
   489    489     Select *pSelect,      /* A SELECT statement to use as the data source */
   490    490     IdList *pColumn,      /* Column names corresponding to IDLIST. */
   491         -  int onError           /* How to handle constraint errors */
          491  +  int onError,          /* How to handle constraint errors */
          492  +  Upsert *pUpsert       /* ON CONFLICT clauses for upsert, or NULL */
   492    493   ){
   493    494     sqlite3 *db;          /* The main database structure */
   494    495     Table *pTab;          /* The table to insert into.  aka TABLE */
   495    496     int i, j;             /* Loop counters */
   496    497     Vdbe *v;              /* Generate code into this virtual machine */
   497    498     Index *pIdx;          /* For looping over indices of the table */
   498    499     int nColumn;          /* Number of columns in the data */
................................................................................
   779    780     if( pColumn!=0 && nColumn!=pColumn->nId ){
   780    781       sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId);
   781    782       goto insert_cleanup;
   782    783     }
   783    784       
   784    785     /* Initialize the count of rows to be inserted
   785    786     */
   786         -  if( db->flags & SQLITE_CountRows ){
          787  +  if( (db->flags & SQLITE_CountRows)!=0
          788  +   && !pParse->nested
          789  +   && !pParse->pTriggerTab
          790  +  ){
   787    791       regRowCount = ++pParse->nMem;
   788    792       sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
   789    793     }
   790    794   
   791    795     /* If this is not a view, open the table and and all indices */
   792    796     if( !isView ){
   793    797       int nIdx;
................................................................................
   799    803       }
   800    804       for(i=0, pIdx=pTab->pIndex; i<nIdx; pIdx=pIdx->pNext, i++){
   801    805         assert( pIdx );
   802    806         aRegIdx[i] = ++pParse->nMem;
   803    807         pParse->nMem += pIdx->nColumn;
   804    808       }
   805    809     }
          810  +#ifndef SQLITE_OMIT_UPSERT
          811  +  if( pUpsert ){
          812  +    pTabList->a[0].iCursor = iDataCur;
          813  +    pUpsert->pUpsertSrc = pTabList;
          814  +    pUpsert->regData = regData;
          815  +    pUpsert->iDataCur = iDataCur;
          816  +    pUpsert->iIdxCur = iIdxCur;
          817  +    if( pUpsert->pUpsertTarget ){
          818  +      sqlite3UpsertAnalyzeTarget(pParse, pTabList, pUpsert);
          819  +    }
          820  +  }
          821  +#endif
          822  +
   806    823   
   807    824     /* This is the top of the main insertion loop */
   808    825     if( useTempTable ){
   809    826       /* This block codes the top of loop only.  The complete loop is the
   810    827       ** following pseudocode (template 4):
   811    828       **
   812    829       **         rewind temp table, if empty goto D
................................................................................
  1001   1018         sqlite3MayAbort(pParse);
  1002   1019       }else
  1003   1020   #endif
  1004   1021       {
  1005   1022         int isReplace;    /* Set to true if constraints may cause a replace */
  1006   1023         int bUseSeek;     /* True to use OPFLAG_SEEKRESULT */
  1007   1024         sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
  1008         -          regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0
         1025  +          regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert
  1009   1026         );
  1010   1027         sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0);
  1011   1028   
  1012   1029         /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE
  1013   1030         ** constraints or (b) there are no triggers and this table is not a
  1014   1031         ** parent table in a foreign key constraint. It is safe to set the
  1015   1032         ** flag in the second case as if any REPLACE constraint is hit, an
................................................................................
  1024   1041             regIns, aRegIdx, 0, appendFlag, bUseSeek
  1025   1042         );
  1026   1043       }
  1027   1044     }
  1028   1045   
  1029   1046     /* Update the count of rows that are inserted
  1030   1047     */
  1031         -  if( (db->flags & SQLITE_CountRows)!=0 ){
         1048  +  if( regRowCount ){
  1032   1049       sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1);
  1033   1050     }
  1034   1051   
  1035   1052     if( pTrigger ){
  1036   1053       /* Code AFTER triggers */
  1037   1054       sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, 
  1038   1055           pTab, regData-2-pTab->nCol, onError, endOfLoop);
................................................................................
  1061   1078     }
  1062   1079   
  1063   1080     /*
  1064   1081     ** Return the number of rows inserted. If this routine is 
  1065   1082     ** generating code because of a call to sqlite3NestedParse(), do not
  1066   1083     ** invoke the callback function.
  1067   1084     */
  1068         -  if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){
         1085  +  if( regRowCount ){
  1069   1086       sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1);
  1070   1087       sqlite3VdbeSetNumCols(v, 1);
  1071   1088       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC);
  1072   1089     }
  1073   1090   
  1074   1091   insert_cleanup:
  1075   1092     sqlite3SrcListDelete(db, pTabList);
  1076   1093     sqlite3ExprListDelete(db, pList);
         1094  +  sqlite3UpsertDelete(db, pUpsert);
  1077   1095     sqlite3SelectDelete(db, pSelect);
  1078   1096     sqlite3IdListDelete(db, pColumn);
  1079   1097     sqlite3DbFree(db, aRegIdx);
  1080   1098   }
  1081   1099   
  1082   1100   /* Make sure "isView" and other macros defined above are undefined. Otherwise
  1083   1101   ** they may interfere with compilation of other functions in this file
................................................................................
  1140   1158     }
  1141   1159     testcase( w.eCode==0 );
  1142   1160     testcase( w.eCode==CKCNSTRNT_COLUMN );
  1143   1161     testcase( w.eCode==CKCNSTRNT_ROWID );
  1144   1162     testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) );
  1145   1163     return !w.eCode;
  1146   1164   }
         1165  +
         1166  +/*
         1167  +** An instance of the ConstraintAddr object remembers the byte-code addresses
         1168  +** for sections of the constraint checks that deal with uniqueness constraints
         1169  +** on the rowid and on the upsert constraint.
         1170  +**
         1171  +** This information is passed into checkReorderConstraintChecks() to insert
         1172  +** some OP_Goto operations so that the rowid and upsert constraints occur
         1173  +** in the correct order relative to other constraints.
         1174  +*/
         1175  +typedef struct ConstraintAddr ConstraintAddr;
         1176  +struct ConstraintAddr {
         1177  +  int ipkTop;          /* Subroutine for rowid constraint check */
         1178  +  int upsertTop;       /* Label for upsert constraint check subroutine */
         1179  +  int upsertTop2;      /* Copy of upsertTop not cleared by the call */
         1180  +  int upsertBtm;       /* upsert constraint returns to this label */
         1181  +  int ipkBtm;          /* Return opcode rowid constraint check */
         1182  +};
         1183  +
         1184  +/*
         1185  +** Generate any OP_Goto operations needed to cause constraints to be
         1186  +** run that haven't already been run.
         1187  +*/
         1188  +static void reorderConstraintChecks(Vdbe *v, ConstraintAddr *p){
         1189  +  if( p->upsertTop ){
         1190  +    testcase( sqlite3VdbeLabelHasBeenResolved(v, p->upsertTop) );
         1191  +    sqlite3VdbeGoto(v, p->upsertTop);
         1192  +    VdbeComment((v, "call upsert subroutine"));
         1193  +    sqlite3VdbeResolveLabel(v, p->upsertBtm);
         1194  +    p->upsertTop = 0;
         1195  +  }
         1196  +  if( p->ipkTop ){
         1197  +    sqlite3VdbeGoto(v, p->ipkTop);
         1198  +    VdbeComment((v, "call rowid unique-check subroutine"));
         1199  +    sqlite3VdbeJumpHere(v, p->ipkBtm);
         1200  +    p->ipkTop = 0;
         1201  +  }
         1202  +}
  1147   1203   
  1148   1204   /*
  1149   1205   ** Generate code to do constraint checks prior to an INSERT or an UPDATE
  1150   1206   ** on table pTab.
  1151   1207   **
  1152   1208   ** The regNewData parameter is the first register in a range that contains
  1153   1209   ** the data to be inserted or the data after the update.  There will be
................................................................................
  1236   1292     int iIdxCur,         /* First index cursor */
  1237   1293     int regNewData,      /* First register in a range holding values to insert */
  1238   1294     int regOldData,      /* Previous content.  0 for INSERTs */
  1239   1295     u8 pkChng,           /* Non-zero if the rowid or PRIMARY KEY changed */
  1240   1296     u8 overrideError,    /* Override onError to this if not OE_Default */
  1241   1297     int ignoreDest,      /* Jump to this label on an OE_Ignore resolution */
  1242   1298     int *pbMayReplace,   /* OUT: Set to true if constraint may cause a replace */
  1243         -  int *aiChng          /* column i is unchanged if aiChng[i]<0 */
         1299  +  int *aiChng,         /* column i is unchanged if aiChng[i]<0 */
         1300  +  Upsert *pUpsert      /* ON CONFLICT clauses, if any.  NULL otherwise */
  1244   1301   ){
  1245   1302     Vdbe *v;             /* VDBE under constrution */
  1246   1303     Index *pIdx;         /* Pointer to one of the indices */
  1247   1304     Index *pPk = 0;      /* The PRIMARY KEY index */
  1248   1305     sqlite3 *db;         /* Database connection */
  1249   1306     int i;               /* loop counter */
  1250   1307     int ix;              /* Index loop counter */
  1251   1308     int nCol;            /* Number of columns */
  1252   1309     int onError;         /* Conflict resolution strategy */
  1253   1310     int addr1;           /* Address of jump instruction */
  1254   1311     int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */
  1255   1312     int nPkField;        /* Number of fields in PRIMARY KEY. 1 for ROWID tables */
  1256         -  int ipkTop = 0;      /* Top of the rowid change constraint check */
  1257         -  int ipkBottom = 0;   /* Bottom of the rowid change constraint check */
         1313  +  ConstraintAddr sAddr;/* Address information for constraint reordering */
         1314  +  Index *pUpIdx = 0;   /* Index to which to apply the upsert */
  1258   1315     u8 isUpdate;         /* True if this is an UPDATE operation */
  1259   1316     u8 bAffinityDone = 0;  /* True if the OP_Affinity operation has been run */
         1317  +  int upsertBypass = 0;  /* Address of Goto to bypass upsert subroutine */
  1260   1318   
  1261   1319     isUpdate = regOldData!=0;
  1262   1320     db = pParse->db;
  1263   1321     v = sqlite3GetVdbe(pParse);
  1264   1322     assert( v!=0 );
  1265   1323     assert( pTab->pSelect==0 );  /* This table is not a VIEW */
  1266   1324     nCol = pTab->nCol;
         1325  +  memset(&sAddr, 0, sizeof(sAddr));
  1267   1326     
  1268   1327     /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for
  1269   1328     ** normal rowid tables.  nPkField is the number of key fields in the 
  1270   1329     ** pPk index or 1 for a rowid table.  In other words, nPkField is the
  1271   1330     ** number of fields in the true primary key of the table. */
  1272   1331     if( HasRowid(pTab) ){
  1273   1332       pPk = 0;
................................................................................
  1358   1417                                 P5_ConstraintCheck);
  1359   1418         }
  1360   1419         sqlite3VdbeResolveLabel(v, allOk);
  1361   1420       }
  1362   1421       pParse->iSelfTab = 0;
  1363   1422     }
  1364   1423   #endif /* !defined(SQLITE_OMIT_CHECK) */
         1424  +
         1425  +  /* UNIQUE and PRIMARY KEY constraints should be handled in the following
         1426  +  ** order:
         1427  +  **
         1428  +  **   (1)  OE_Abort, OE_Fail, OE_Rollback, OE_Ignore
         1429  +  **   (2)  OE_Update
         1430  +  **   (3)  OE_Replace
         1431  +  **
         1432  +  ** OE_Fail and OE_Ignore must happen before any changes are made.
         1433  +  ** OE_Update guarantees that only a single row will change, so it
         1434  +  ** must happen before OE_Replace.  Technically, OE_Abort and OE_Rollback
         1435  +  ** could happen in any order, but they are grouped up front for
         1436  +  ** convenience.
         1437  +  **
         1438  +  ** Constraint checking code is generated in this order:
         1439  +  **   (A)  The rowid constraint
         1440  +  **   (B)  Unique index constraints that do not have OE_Replace as their
         1441  +  **        default conflict resolution strategy
         1442  +  **   (C)  Unique index that do use OE_Replace by default.
         1443  +  **
         1444  +  ** The ordering of (2) and (3) is accomplished by making sure the linked
         1445  +  ** list of indexes attached to a table puts all OE_Replace indexes last
         1446  +  ** in the list.  See sqlite3CreateIndex() for where that happens.
         1447  +  */
         1448  +
         1449  +  if( pUpsert ){
         1450  +    if( pUpsert->pUpsertTarget==0 ){
         1451  +      /* An ON CONFLICT DO NOTHING clause, without a constraint-target.
         1452  +      ** Make all unique constraint resolution be OE_Ignore */
         1453  +      assert( pUpsert->pUpsertSet==0 );
         1454  +      overrideError = OE_Ignore;
         1455  +      pUpsert = 0;
         1456  +    }else if( (pUpIdx = pUpsert->pUpsertIdx)!=0 ){
         1457  +      /* If the constraint-target is on some column other than
         1458  +      ** then ROWID, then we might need to move the UPSERT around
         1459  +      ** so that it occurs in the correct order. */
         1460  +      sAddr.upsertTop = sAddr.upsertTop2 = sqlite3VdbeMakeLabel(v);
         1461  +      sAddr.upsertBtm = sqlite3VdbeMakeLabel(v);
         1462  +    }
         1463  +  }
  1365   1464   
  1366   1465     /* If rowid is changing, make sure the new rowid does not previously
  1367   1466     ** exist in the table.
  1368   1467     */
  1369   1468     if( pkChng && pPk==0 ){
  1370   1469       int addrRowidOk = sqlite3VdbeMakeLabel(v);
  1371   1470   
................................................................................
  1372   1471       /* Figure out what action to take in case of a rowid collision */
  1373   1472       onError = pTab->keyConf;
  1374   1473       if( overrideError!=OE_Default ){
  1375   1474         onError = overrideError;
  1376   1475       }else if( onError==OE_Default ){
  1377   1476         onError = OE_Abort;
  1378   1477       }
         1478  +
         1479  +    /* figure out whether or not upsert applies in this case */
         1480  +    if( pUpsert && pUpsert->pUpsertIdx==0 ){
         1481  +      if( pUpsert->pUpsertSet==0 ){
         1482  +        onError = OE_Ignore;  /* DO NOTHING is the same as INSERT OR IGNORE */
         1483  +      }else{
         1484  +        onError = OE_Update;  /* DO UPDATE */
         1485  +      }
         1486  +    }
         1487  +
         1488  +    /* If the response to a rowid conflict is REPLACE but the response
         1489  +    ** to some other UNIQUE constraint is FAIL or IGNORE, then we need
         1490  +    ** to defer the running of the rowid conflict checking until after
         1491  +    ** the UNIQUE constraints have run.
         1492  +    */
         1493  +    assert( OE_Update>OE_Replace );
         1494  +    assert( OE_Ignore<OE_Replace );
         1495  +    assert( OE_Fail<OE_Replace );
         1496  +    assert( OE_Abort<OE_Replace );
         1497  +    assert( OE_Rollback<OE_Replace );
         1498  +    if( onError>=OE_Replace
         1499  +     && (pUpsert || onError!=overrideError)
         1500  +     && pTab->pIndex
         1501  +    ){
         1502  +      sAddr.ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1;
         1503  +    }
  1379   1504   
  1380   1505       if( isUpdate ){
  1381   1506         /* pkChng!=0 does not mean that the rowid has changed, only that
  1382   1507         ** it might have changed.  Skip the conflict logic below if the rowid
  1383   1508         ** is unchanged. */
  1384   1509         sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData);
  1385   1510         sqlite3VdbeChangeP5(v, SQLITE_NOTNULL);
  1386   1511         VdbeCoverage(v);
  1387   1512       }
  1388   1513   
  1389         -    /* If the response to a rowid conflict is REPLACE but the response
  1390         -    ** to some other UNIQUE constraint is FAIL or IGNORE, then we need
  1391         -    ** to defer the running of the rowid conflict checking until after
  1392         -    ** the UNIQUE constraints have run.
  1393         -    */
  1394         -    if( onError==OE_Replace && overrideError!=OE_Replace ){
  1395         -      for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
  1396         -        if( pIdx->onError==OE_Ignore || pIdx->onError==OE_Fail ){
  1397         -          ipkTop = sqlite3VdbeAddOp0(v, OP_Goto);
  1398         -          break;
  1399         -        }
  1400         -      }
  1401         -    }
  1402         -
  1403   1514       /* Check to see if the new rowid already exists in the table.  Skip
  1404   1515       ** the following conflict logic if it does not. */
         1516  +    VdbeNoopComment((v, "uniqueness check for ROWID"));
  1405   1517       sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData);
  1406   1518       VdbeCoverage(v);
  1407   1519   
  1408         -    /* Generate code that deals with a rowid collision */
  1409   1520       switch( onError ){
  1410   1521         default: {
  1411   1522           onError = OE_Abort;
  1412   1523           /* Fall thru into the next case */
  1413   1524         }
  1414   1525         case OE_Rollback:
  1415   1526         case OE_Abort:
  1416   1527         case OE_Fail: {
         1528  +        testcase( onError==OE_Rollback );
         1529  +        testcase( onError==OE_Abort );
         1530  +        testcase( onError==OE_Fail );
  1417   1531           sqlite3RowidConstraint(pParse, onError, pTab);
  1418   1532           break;
  1419   1533         }
  1420   1534         case OE_Replace: {
  1421   1535           /* If there are DELETE triggers on this table and the
  1422   1536           ** recursive-triggers flag is set, call GenerateRowDelete() to
  1423   1537           ** remove the conflicting row from the table. This will fire
................................................................................
  1446   1560           }
  1447   1561           if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){
  1448   1562             sqlite3MultiWrite(pParse);
  1449   1563             sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur,
  1450   1564                                      regNewData, 1, 0, OE_Replace, 1, -1);
  1451   1565           }else{
  1452   1566   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
  1453         -          if( HasRowid(pTab) ){
  1454         -            /* This OP_Delete opcode fires the pre-update-hook only. It does
  1455         -            ** not modify the b-tree. It is more efficient to let the coming
  1456         -            ** OP_Insert replace the existing entry than it is to delete the
  1457         -            ** existing entry and then insert a new one. */
  1458         -            sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP);
  1459         -            sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
  1460         -          }
         1567  +          assert( HasRowid(pTab) );
         1568  +          /* This OP_Delete opcode fires the pre-update-hook only. It does
         1569  +          ** not modify the b-tree. It is more efficient to let the coming
         1570  +          ** OP_Insert replace the existing entry than it is to delete the
         1571  +          ** existing entry and then insert a new one. */
         1572  +          sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP);
         1573  +          sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
  1461   1574   #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
  1462   1575             if( pTab->pIndex ){
  1463   1576               sqlite3MultiWrite(pParse);
  1464   1577               sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1);
  1465   1578             }
  1466   1579           }
  1467   1580           seenReplace = 1;
  1468   1581           break;
  1469   1582         }
         1583  +#ifndef SQLITE_OMIT_UPSERT
         1584  +      case OE_Update: {
         1585  +        sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, 0, iDataCur);
         1586  +        /* Fall through */
         1587  +      }
         1588  +#endif
  1470   1589         case OE_Ignore: {
  1471         -        /*assert( seenReplace==0 );*/
         1590  +        testcase( onError==OE_Ignore );
  1472   1591           sqlite3VdbeGoto(v, ignoreDest);
  1473   1592           break;
  1474   1593         }
  1475   1594       }
  1476   1595       sqlite3VdbeResolveLabel(v, addrRowidOk);
  1477         -    if( ipkTop ){
  1478         -      ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto);
  1479         -      sqlite3VdbeJumpHere(v, ipkTop);
         1596  +    if( sAddr.ipkTop ){
         1597  +      sAddr.ipkBtm = sqlite3VdbeAddOp0(v, OP_Goto);
         1598  +      sqlite3VdbeJumpHere(v, sAddr.ipkTop-1);
  1480   1599       }
  1481   1600     }
  1482   1601   
  1483   1602     /* Test all UNIQUE constraints by creating entries for each UNIQUE
  1484   1603     ** index and making sure that duplicate entries do not already exist.
  1485   1604     ** Compute the revised record entries for indices as we go.
  1486   1605     **
................................................................................
  1490   1609     for(ix=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, ix++){
  1491   1610       int regIdx;          /* Range of registers hold conent for pIdx */
  1492   1611       int regR;            /* Range of registers holding conflicting PK */
  1493   1612       int iThisCur;        /* Cursor for this UNIQUE index */
  1494   1613       int addrUniqueOk;    /* Jump here if the UNIQUE constraint is satisfied */
  1495   1614   
  1496   1615       if( aRegIdx[ix]==0 ) continue;  /* Skip indices that do not change */
         1616  +    if( pUpIdx==pIdx ){
         1617  +      addrUniqueOk = sAddr.upsertBtm;
         1618  +      upsertBypass = sqlite3VdbeGoto(v, 0);
         1619  +      VdbeComment((v, "Skip upsert subroutine"));
         1620  +      sqlite3VdbeResolveLabel(v, sAddr.upsertTop2);
         1621  +    }else{
         1622  +      addrUniqueOk = sqlite3VdbeMakeLabel(v);
         1623  +    }
         1624  +    VdbeNoopComment((v, "uniqueness check for %s", pIdx->zName));
  1497   1625       if( bAffinityDone==0 ){
  1498   1626         sqlite3TableAffinity(v, pTab, regNewData+1);
  1499   1627         bAffinityDone = 1;
  1500   1628       }
  1501   1629       iThisCur = iIdxCur+ix;
  1502         -    addrUniqueOk = sqlite3VdbeMakeLabel(v);
         1630  +
  1503   1631   
  1504   1632       /* Skip partial indices for which the WHERE clause is not true */
  1505   1633       if( pIdx->pPartIdxWhere ){
  1506   1634         sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]);
  1507   1635         pParse->iSelfTab = -(regNewData+1);
  1508   1636         sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk,
  1509   1637                               SQLITE_JUMPIFNULL);
................................................................................
  1554   1682         continue;  /* pIdx is not a UNIQUE index */
  1555   1683       }
  1556   1684       if( overrideError!=OE_Default ){
  1557   1685         onError = overrideError;
  1558   1686       }else if( onError==OE_Default ){
  1559   1687         onError = OE_Abort;
  1560   1688       }
         1689  +
         1690  +    /* Figure out if the upsert clause applies to this index */
         1691  +    if( pUpIdx==pIdx ){
         1692  +      if( pUpsert->pUpsertSet==0 ){
         1693  +        onError = OE_Ignore;  /* DO NOTHING is the same as INSERT OR IGNORE */
         1694  +      }else{
         1695  +        onError = OE_Update;  /* DO UPDATE */
         1696  +      }
         1697  +    }
         1698  +
         1699  +    /* Invoke subroutines to handle IPK replace and upsert prior to running
         1700  +    ** the first REPLACE constraint check. */
         1701  +    if( onError==OE_Replace ){
         1702  +      testcase( sAddr.ipkTop );
         1703  +      testcase( sAddr.upsertTop
         1704  +             && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) );
         1705  +      reorderConstraintChecks(v, &sAddr);
         1706  +    }
  1561   1707   
  1562   1708       /* Collision detection may be omitted if all of the following are true:
  1563   1709       **   (1) The conflict resolution algorithm is REPLACE
  1564   1710       **   (2) The table is a WITHOUT ROWID table
  1565   1711       **   (3) There are no secondary indexes on the table
  1566   1712       **   (4) No delete triggers need to be fired if there is a conflict
  1567   1713       **   (5) No FK constraint counters need to be updated if a conflict occurs.
................................................................................
  1637   1783             }
  1638   1784           }
  1639   1785         }
  1640   1786       }
  1641   1787   
  1642   1788       /* Generate code that executes if the new index entry is not unique */
  1643   1789       assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail
  1644         -        || onError==OE_Ignore || onError==OE_Replace );
         1790  +        || onError==OE_Ignore || onError==OE_Replace || onError==OE_Update );
  1645   1791       switch( onError ){
  1646   1792         case OE_Rollback:
  1647   1793         case OE_Abort:
  1648   1794         case OE_Fail: {
         1795  +        testcase( onError==OE_Rollback );
         1796  +        testcase( onError==OE_Abort );
         1797  +        testcase( onError==OE_Fail );
  1649   1798           sqlite3UniqueConstraint(pParse, onError, pIdx);
  1650   1799           break;
  1651   1800         }
         1801  +#ifndef SQLITE_OMIT_UPSERT
         1802  +      case OE_Update: {
         1803  +        sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, pIdx, iIdxCur+ix);
         1804  +        /* Fall through */
         1805  +      }
         1806  +#endif
  1652   1807         case OE_Ignore: {
         1808  +        testcase( onError==OE_Ignore );
  1653   1809           sqlite3VdbeGoto(v, ignoreDest);
  1654   1810           break;
  1655   1811         }
  1656   1812         default: {
  1657   1813           Trigger *pTrigger = 0;
  1658   1814           assert( onError==OE_Replace );
  1659   1815           sqlite3MultiWrite(pParse);
................................................................................
  1663   1819           sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur,
  1664   1820               regR, nPkField, 0, OE_Replace,
  1665   1821               (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur);
  1666   1822           seenReplace = 1;
  1667   1823           break;
  1668   1824         }
  1669   1825       }
  1670         -    sqlite3VdbeResolveLabel(v, addrUniqueOk);
         1826  +    if( pUpIdx==pIdx ){
         1827  +      sqlite3VdbeJumpHere(v, upsertBypass);
         1828  +    }else{
         1829  +      sqlite3VdbeResolveLabel(v, addrUniqueOk);
         1830  +    }
  1671   1831       sqlite3ExprCachePop(pParse);
  1672   1832       if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField);
         1833  +
  1673   1834     }
  1674         -  if( ipkTop ){
  1675         -    sqlite3VdbeGoto(v, ipkTop+1);
  1676         -    sqlite3VdbeJumpHere(v, ipkBottom);
  1677         -  }
         1835  +  testcase( sAddr.ipkTop!=0 );
         1836  +  testcase( sAddr.upsertTop
         1837  +         && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) );
         1838  +  reorderConstraintChecks(v, &sAddr);
  1678   1839     
  1679   1840     *pbMayReplace = seenReplace;
  1680   1841     VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace));
  1681   1842   }
  1682   1843   
  1683   1844   #ifdef SQLITE_ENABLE_NULL_TRIM
  1684   1845   /*

Changes to src/main.c.

   638    638       }
   639    639   
   640    640       case SQLITE_CONFIG_STMTJRNL_SPILL: {
   641    641         sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int);
   642    642         break;
   643    643       }
   644    644   
          645  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          646  +    case SQLITE_CONFIG_SORTERREF_SIZE: {
          647  +      int iVal = va_arg(ap, int);
          648  +      if( iVal<0 ){
          649  +        iVal = SQLITE_DEFAULT_SORTERREF_SIZE;
          650  +      }
          651  +      sqlite3GlobalConfig.szSorterRef = (u32)iVal;
          652  +      break;
          653  +    }
          654  +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */
          655  +
   645    656       default: {
   646    657         rc = SQLITE_ERROR;
   647    658         break;
   648    659       }
   649    660     }
   650    661     va_end(ap);
   651    662     return rc;
................................................................................
  1472   1483   }
  1473   1484   
  1474   1485   /*
  1475   1486   ** This routine implements a busy callback that sleeps and tries
  1476   1487   ** again until a timeout value is reached.  The timeout value is
  1477   1488   ** an integer number of milliseconds passed in as the first
  1478   1489   ** argument.
         1490  +**
         1491  +** Return non-zero to retry the lock.  Return zero to stop trying
         1492  +** and cause SQLite to return SQLITE_BUSY.
  1479   1493   */
  1480   1494   static int sqliteDefaultBusyCallback(
  1481         - void *ptr,               /* Database connection */
  1482         - int count                /* Number of times table has been busy */
         1495  +  void *ptr,               /* Database connection */
         1496  +  int count,               /* Number of times table has been busy */
         1497  +  sqlite3_file *pFile      /* The file on which the lock occurred */
  1483   1498   ){
  1484   1499   #if SQLITE_OS_WIN || HAVE_USLEEP
         1500  +  /* This case is for systems that have support for sleeping for fractions of
         1501  +  ** a second.  Examples:  All windows systems, unix systems with usleep() */
  1485   1502     static const u8 delays[] =
  1486   1503        { 1, 2, 5, 10, 15, 20, 25, 25,  25,  50,  50, 100 };
  1487   1504     static const u8 totals[] =
  1488   1505        { 0, 1, 3,  8, 18, 33, 53, 78, 103, 128, 178, 228 };
  1489   1506   # define NDELAY ArraySize(delays)
  1490   1507     sqlite3 *db = (sqlite3 *)ptr;
  1491         -  int timeout = db->busyTimeout;
         1508  +  int tmout = db->busyTimeout;
  1492   1509     int delay, prior;
  1493   1510   
         1511  +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
         1512  +  if( sqlite3OsFileControl(pFile,SQLITE_FCNTL_LOCK_TIMEOUT,&tmout)==SQLITE_OK ){
         1513  +    if( count ){
         1514  +      tmout = 0;
         1515  +      sqlite3OsFileControl(pFile, SQLITE_FCNTL_LOCK_TIMEOUT, &tmout);
         1516  +      return 0;
         1517  +    }else{
         1518  +      return 1;
         1519  +    }
         1520  +  }
         1521  +#else
         1522  +  UNUSED_PARAMETER(pFile);
         1523  +#endif
  1494   1524     assert( count>=0 );
  1495   1525     if( count < NDELAY ){
  1496   1526       delay = delays[count];
  1497   1527       prior = totals[count];
  1498   1528     }else{
  1499   1529       delay = delays[NDELAY-1];
  1500   1530       prior = totals[NDELAY-1] + delay*(count-(NDELAY-1));
  1501   1531     }
  1502         -  if( prior + delay > timeout ){
  1503         -    delay = timeout - prior;
         1532  +  if( prior + delay > tmout ){
         1533  +    delay = tmout - prior;
  1504   1534       if( delay<=0 ) return 0;
  1505   1535     }
  1506   1536     sqlite3OsSleep(db->pVfs, delay*1000);
  1507   1537     return 1;
  1508   1538   #else
         1539  +  /* This case for unix systems that lack usleep() support.  Sleeping
         1540  +  ** must be done in increments of whole seconds */
  1509   1541     sqlite3 *db = (sqlite3 *)ptr;
  1510         -  int timeout = ((sqlite3 *)ptr)->busyTimeout;
  1511         -  if( (count+1)*1000 > timeout ){
         1542  +  int tmout = ((sqlite3 *)ptr)->busyTimeout;
         1543  +  UNUSED_PARAMETER(pFile);
         1544  +  if( (count+1)*1000 > tmout ){
  1512   1545       return 0;
  1513   1546     }
  1514   1547     sqlite3OsSleep(db->pVfs, 1000000);
  1515   1548     return 1;
  1516   1549   #endif
  1517   1550   }
  1518   1551   
  1519   1552   /*
  1520   1553   ** Invoke the given busy handler.
  1521   1554   **
  1522         -** This routine is called when an operation failed with a lock.
         1555  +** This routine is called when an operation failed to acquire a
         1556  +** lock on VFS file pFile.
         1557  +**
  1523   1558   ** If this routine returns non-zero, the lock is retried.  If it
  1524   1559   ** returns 0, the operation aborts with an SQLITE_BUSY error.
  1525   1560   */
  1526         -int sqlite3InvokeBusyHandler(BusyHandler *p){
         1561  +int sqlite3InvokeBusyHandler(BusyHandler *p, sqlite3_file *pFile){
  1527   1562     int rc;
  1528         -  if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0;
  1529         -  rc = p->xFunc(p->pArg, p->nBusy);
         1563  +  if( p->xBusyHandler==0 || p->nBusy<0 ) return 0;
         1564  +  if( p->bExtraFileArg ){
         1565  +    /* Add an extra parameter with the pFile pointer to the end of the
         1566  +    ** callback argument list */
         1567  +    int (*xTra)(void*,int,sqlite3_file*);
         1568  +    xTra = (int(*)(void*,int,sqlite3_file*))p->xBusyHandler;
         1569  +    rc = xTra(p->pBusyArg, p->nBusy, pFile);
         1570  +  }else{
         1571  +    /* Legacy style busy handler callback */
         1572  +    rc = p->xBusyHandler(p->pBusyArg, p->nBusy);
         1573  +  }
  1530   1574     if( rc==0 ){
  1531   1575       p->nBusy = -1;
  1532   1576     }else{
  1533   1577       p->nBusy++;
  1534   1578     }
  1535   1579     return rc; 
  1536   1580   }
................................................................................
  1544   1588     int (*xBusy)(void*,int),
  1545   1589     void *pArg
  1546   1590   ){
  1547   1591   #ifdef SQLITE_ENABLE_API_ARMOR
  1548   1592     if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
  1549   1593   #endif
  1550   1594     sqlite3_mutex_enter(db->mutex);
  1551         -  db->busyHandler.xFunc = xBusy;
  1552         -  db->busyHandler.pArg = pArg;
         1595  +  db->busyHandler.xBusyHandler = xBusy;
         1596  +  db->busyHandler.pBusyArg = pArg;
  1553   1597     db->busyHandler.nBusy = 0;
         1598  +  db->busyHandler.bExtraFileArg = 0;
  1554   1599     db->busyTimeout = 0;
  1555   1600     sqlite3_mutex_leave(db->mutex);
  1556   1601     return SQLITE_OK;
  1557   1602   }
  1558   1603   
  1559   1604   #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
  1560   1605   /*
................................................................................
  1594   1639   ** specified number of milliseconds before returning 0.
  1595   1640   */
  1596   1641   int sqlite3_busy_timeout(sqlite3 *db, int ms){
  1597   1642   #ifdef SQLITE_ENABLE_API_ARMOR
  1598   1643     if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
  1599   1644   #endif
  1600   1645     if( ms>0 ){
  1601         -    sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db);
         1646  +    sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback,
         1647  +                             (void*)db);
  1602   1648       db->busyTimeout = ms;
         1649  +    db->busyHandler.bExtraFileArg = 1;
  1603   1650     }else{
  1604   1651       sqlite3_busy_handler(db, 0, 0);
  1605   1652     }
  1606   1653     return SQLITE_OK;
  1607   1654   }
  1608   1655   
  1609   1656   /*
................................................................................
  3588   3635         rc = SQLITE_OK;
  3589   3636       }else if( op==SQLITE_FCNTL_VFS_POINTER ){
  3590   3637         *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager);
  3591   3638         rc = SQLITE_OK;
  3592   3639       }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){
  3593   3640         *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager);
  3594   3641         rc = SQLITE_OK;
  3595         -    }else if( fd->pMethods ){
  3596         -      rc = sqlite3OsFileControl(fd, op, pArg);
  3597   3642       }else{
  3598         -      rc = SQLITE_NOTFOUND;
         3643  +      rc = sqlite3OsFileControl(fd, op, pArg);
  3599   3644       }
  3600   3645       sqlite3BtreeLeave(pBtree);
  3601   3646     }
  3602   3647     sqlite3_mutex_leave(db->mutex);
  3603   3648     return rc;
  3604   3649   }
  3605   3650   

Changes to src/memdb.c.

     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13         -** This file implements in-memory VFS.  A database is held as a contiguous
           13  +** This file implements an in-memory VFS. A database is held as a contiguous
    14     14   ** block of memory.
    15     15   **
    16     16   ** This file also implements interface sqlite3_serialize() and
    17     17   ** sqlite3_deserialize().
    18     18   */
    19     19   #ifdef SQLITE_ENABLE_DESERIALIZE
    20     20   #include "sqliteInt.h"

Changes to src/os.c.

   121    121   ** and we need to know about the failures.  Use sqlite3OsFileControlHint()
   122    122   ** when simply tossing information over the wall to the VFS and we do not
   123    123   ** really care if the VFS receives and understands the information since it
   124    124   ** is only a hint and can be safely ignored.  The sqlite3OsFileControlHint()
   125    125   ** routine has no return value since the return value would be meaningless.
   126    126   */
   127    127   int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){
          128  +  if( id->pMethods==0 ) return SQLITE_NOTFOUND;
   128    129   #ifdef SQLITE_TEST
   129         -  if( op!=SQLITE_FCNTL_COMMIT_PHASETWO ){
          130  +  if( op!=SQLITE_FCNTL_COMMIT_PHASETWO
          131  +   && op!=SQLITE_FCNTL_LOCK_TIMEOUT
          132  +  ){
   130    133       /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite
   131    134       ** is using a regular VFS, it is called after the corresponding
   132    135       ** transaction has been committed. Injecting a fault at this point
   133    136       ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM
   134    137       ** but the transaction is committed anyway.
   135    138       **
   136    139       ** The core must call OsFileControl() though, not OsFileControlHint(),
................................................................................
   139    142       ** to the user.  */
   140    143       DO_OS_MALLOC_TEST(id);
   141    144     }
   142    145   #endif
   143    146     return id->pMethods->xFileControl(id, op, pArg);
   144    147   }
   145    148   void sqlite3OsFileControlHint(sqlite3_file *id, int op, void *pArg){
   146         -  (void)id->pMethods->xFileControl(id, op, pArg);
          149  +  if( id->pMethods ) (void)id->pMethods->xFileControl(id, op, pArg);
   147    150   }
   148    151   
   149    152   int sqlite3OsSectorSize(sqlite3_file *id){
   150    153     int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize;
   151    154     return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE);
   152    155   }
   153    156   int sqlite3OsDeviceCharacteristics(sqlite3_file *id){

Changes to src/os_unix.c.

   225    225     int deviceCharacteristics;          /* Precomputed device characteristics */
   226    226   #if SQLITE_ENABLE_LOCKING_STYLE
   227    227     int openFlags;                      /* The flags specified at open() */
   228    228   #endif
   229    229   #if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__)
   230    230     unsigned fsFlags;                   /* cached details from statfs() */
   231    231   #endif
          232  +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
          233  +  unsigned iBusyTimeout;              /* Wait this many millisec on locks */
          234  +#endif
   232    235   #if OS_VXWORKS
   233    236     struct vxworksFileId *pId;          /* Unique file ID */
   234    237   #endif
   235    238   #ifdef SQLITE_DEBUG
   236    239     /* The next group of variables are used to track whether or not the
   237    240     ** transaction counter in bytes 24-27 of database files are updated
   238    241     ** whenever any part of the database changes.  An assertion fault will
................................................................................
  1471   1474     
  1472   1475     unixLeaveMutex();
  1473   1476     OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved));
  1474   1477   
  1475   1478     *pResOut = reserved;
  1476   1479     return rc;
  1477   1480   }
         1481  +
         1482  +/*
         1483  +** Set a posix-advisory-lock.
         1484  +**
         1485  +** There are two versions of this routine.  If compiled with
         1486  +** SQLITE_ENABLE_SETLK_TIMEOUT then the routine has an extra parameter
         1487  +** which is a pointer to a unixFile.  If the unixFile->iBusyTimeout
         1488  +** value is set, then it is the number of milliseconds to wait before
         1489  +** failing the lock.  The iBusyTimeout value is always reset back to
         1490  +** zero on each call.
         1491  +**
         1492  +** If SQLITE_ENABLE_SETLK_TIMEOUT is not defined, then do a non-blocking
         1493  +** attempt to set the lock.
         1494  +*/
         1495  +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT
         1496  +# define osSetPosixAdvisoryLock(h,x,t) osFcntl(h,F_SETLK,x)
         1497  +#else
         1498  +static int osSetPosixAdvisoryLock(
         1499  +  int h,                /* The file descriptor on which to take the lock */
         1500  +  struct flock *pLock,  /* The description of the lock */
         1501  +  unixFile *pFile       /* Structure holding timeout value */
         1502  +){
         1503  +  int rc = osFcntl(h,F_SETLK,pLock);
         1504  +  while( rc<0 && pFile->iBusyTimeout>0 ){
         1505  +    /* On systems that support some kind of blocking file lock with a timeout,
         1506  +    ** make appropriate changes here to invoke that blocking file lock.  On
         1507  +    ** generic posix, however, there is no such API.  So we simply try the
         1508  +    ** lock once every millisecond until either the timeout expires, or until
         1509  +    ** the lock is obtained. */
         1510  +    usleep(1000);
         1511  +    rc = osFcntl(h,F_SETLK,pLock);
         1512  +    pFile->iBusyTimeout--;
         1513  +  }
         1514  +  return rc;
         1515  +}
         1516  +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */
         1517  +
  1478   1518   
  1479   1519   /*
  1480   1520   ** Attempt to set a system-lock on the file pFile.  The lock is 
  1481   1521   ** described by pLock.
  1482   1522   **
  1483   1523   ** If the pFile was opened read/write from unix-excl, then the only lock
  1484   1524   ** ever obtained is an exclusive lock, and it is obtained exactly once
................................................................................
  1504   1544       if( pInode->bProcessLock==0 ){
  1505   1545         struct flock lock;
  1506   1546         assert( pInode->nLock==0 );
  1507   1547         lock.l_whence = SEEK_SET;
  1508   1548         lock.l_start = SHARED_FIRST;
  1509   1549         lock.l_len = SHARED_SIZE;
  1510   1550         lock.l_type = F_WRLCK;
  1511         -      rc = osFcntl(pFile->h, F_SETLK, &lock);
         1551  +      rc = osSetPosixAdvisoryLock(pFile->h, &lock, pFile);
  1512   1552         if( rc<0 ) return rc;
  1513   1553         pInode->bProcessLock = 1;
  1514   1554         pInode->nLock++;
  1515   1555       }else{
  1516   1556         rc = 0;
  1517   1557       }
  1518   1558     }else{
  1519         -    rc = osFcntl(pFile->h, F_SETLK, pLock);
         1559  +    rc = osSetPosixAdvisoryLock(pFile->h, pLock, pFile);
  1520   1560     }
  1521   1561     return rc;
  1522   1562   }
  1523   1563   
  1524   1564   /*
  1525   1565   ** Lock the file with the lock specified by parameter eFileLock - one
  1526   1566   ** of the following:
................................................................................
  3880   3920         }
  3881   3921         return SQLITE_OK;
  3882   3922       }
  3883   3923       case SQLITE_FCNTL_HAS_MOVED: {
  3884   3924         *(int*)pArg = fileHasMoved(pFile);
  3885   3925         return SQLITE_OK;
  3886   3926       }
         3927  +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
         3928  +    case SQLITE_FCNTL_LOCK_TIMEOUT: {
         3929  +      pFile->iBusyTimeout = *(int*)pArg;
         3930  +      return SQLITE_OK;
         3931  +    }
         3932  +#endif
  3887   3933   #if SQLITE_MAX_MMAP_SIZE>0
  3888   3934       case SQLITE_FCNTL_MMAP_SIZE: {
  3889   3935         i64 newLimit = *(i64*)pArg;
  3890   3936         int rc = SQLITE_OK;
  3891   3937         if( newLimit>sqlite3GlobalConfig.mxMmap ){
  3892   3938           newLimit = sqlite3GlobalConfig.mxMmap;
  3893   3939         }
................................................................................
  4202   4248   
  4203   4249     if( pShmNode->h>=0 ){
  4204   4250       /* Initialize the locking parameters */
  4205   4251       f.l_type = lockType;
  4206   4252       f.l_whence = SEEK_SET;
  4207   4253       f.l_start = ofst;
  4208   4254       f.l_len = n;
  4209         -    rc = osFcntl(pShmNode->h, F_SETLK, &f);
         4255  +    rc = osSetPosixAdvisoryLock(pShmNode->h, &f, pFile);
  4210   4256       rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY;
  4211   4257     }
  4212   4258   
  4213   4259     /* Update the global lock state and do debug tracing */
  4214   4260   #ifdef SQLITE_DEBUG
  4215   4261     { u16 mask;
  4216   4262     OSTRACE(("SHM-LOCK "));

Changes to src/pager.c.

  2176   2176       ** successfully committed, but the EXCLUSIVE lock is still held on the
  2177   2177       ** file. So it is safe to truncate the database file to its minimum
  2178   2178       ** required size.  */
  2179   2179       assert( pPager->eLock==EXCLUSIVE_LOCK );
  2180   2180       rc = pager_truncate(pPager, pPager->dbSize);
  2181   2181     }
  2182   2182   
  2183         -  if( rc==SQLITE_OK && bCommit && isOpen(pPager->fd) ){
         2183  +  if( rc==SQLITE_OK && bCommit ){
  2184   2184       rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_COMMIT_PHASETWO, 0);
  2185   2185       if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
  2186   2186     }
  2187   2187   
  2188   2188     if( !pPager->exclusiveMode 
  2189   2189      && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0))
  2190   2190     ){
................................................................................
  2995   2995     }
  2996   2996     /* Following a rollback, the database file should be back in its original
  2997   2997     ** state prior to the start of the transaction, so invoke the
  2998   2998     ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the
  2999   2999     ** assertion that the transaction counter was modified.
  3000   3000     */
  3001   3001   #ifdef SQLITE_DEBUG
  3002         -  if( pPager->fd->pMethods ){
  3003         -    sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0);
  3004         -  }
         3002  +  sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0);
  3005   3003   #endif
  3006   3004   
  3007   3005     /* If this playback is happening automatically as a result of an IO or 
  3008   3006     ** malloc error that occurred after the change-counter was updated but 
  3009   3007     ** before the transaction was committed, then the change-counter 
  3010   3008     ** modification may just have been reverted. If this happens in exclusive 
  3011   3009     ** mode, then subsequent transactions performed by the connection will not
................................................................................
  3762   3760   **   SHARED_LOCK   -> EXCLUSIVE_LOCK   | No
  3763   3761   **   RESERVED_LOCK -> EXCLUSIVE_LOCK   | Yes
  3764   3762   **
  3765   3763   ** If the busy-handler callback returns non-zero, the lock is 
  3766   3764   ** retried. If it returns zero, then the SQLITE_BUSY error is
  3767   3765   ** returned to the caller of the pager API function.
  3768   3766   */
  3769         -void sqlite3PagerSetBusyhandler(
         3767  +void sqlite3PagerSetBusyHandler(
  3770   3768     Pager *pPager,                       /* Pager object */
  3771   3769     int (*xBusyHandler)(void *),         /* Pointer to busy-handler function */
  3772   3770     void *pBusyHandlerArg                /* Argument to pass to xBusyHandler */
  3773   3771   ){
         3772  +  void **ap;
  3774   3773     pPager->xBusyHandler = xBusyHandler;
  3775   3774     pPager->pBusyHandlerArg = pBusyHandlerArg;
  3776         -
  3777         -  if( isOpen(pPager->fd) ){
  3778         -    void **ap = (void **)&pPager->xBusyHandler;
  3779         -    assert( ((int(*)(void *))(ap[0]))==xBusyHandler );
  3780         -    assert( ap[1]==pBusyHandlerArg );
  3781         -    sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap);
  3782         -  }
         3775  +  ap = (void **)&pPager->xBusyHandler;
         3776  +  assert( ((int(*)(void *))(ap[0]))==xBusyHandler );
         3777  +  assert( ap[1]==pBusyHandlerArg );
         3778  +  sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap);
  3783   3779   }
  3784   3780   
  3785   3781   /*
  3786   3782   ** Change the page size used by the Pager object. The new page size 
  3787   3783   ** is passed in *pPageSize.
  3788   3784   **
  3789   3785   ** If the pager is in the error state when this function is called, it
................................................................................
  5769   5765   }
  5770   5766   void sqlite3PagerUnrefPageOne(DbPage *pPg){
  5771   5767     Pager *pPager;
  5772   5768     assert( pPg!=0 );
  5773   5769     assert( pPg->pgno==1 );
  5774   5770     assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */
  5775   5771     pPager = pPg->pPager;
         5772  +  sqlite3PagerResetLockTimeout(pPager);
  5776   5773     sqlite3PcacheRelease(pPg);
  5777   5774     pagerUnlockIfUnused(pPager);
  5778   5775   }
  5779   5776   
  5780   5777   /*
  5781   5778   ** This function is called at the start of every write transaction.
  5782   5779   ** There must already be a RESERVED or EXCLUSIVE lock on the database 
................................................................................
  6367   6364   ** or pages with the Pager.noSync flag set.
  6368   6365   **
  6369   6366   ** If successful, or if called on a pager for which it is a no-op, this
  6370   6367   ** function returns SQLITE_OK. Otherwise, an IO error code is returned.
  6371   6368   */
  6372   6369   int sqlite3PagerSync(Pager *pPager, const char *zMaster){
  6373   6370     int rc = SQLITE_OK;
  6374         -
  6375         -  if( isOpen(pPager->fd) ){
  6376         -    void *pArg = (void*)zMaster;
  6377         -    rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg);
  6378         -    if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
  6379         -  }
         6371  +  void *pArg = (void*)zMaster;
         6372  +  rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg);
         6373  +  if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
  6380   6374     if( rc==SQLITE_OK && !pPager->noSync ){
  6381   6375       assert( !MEMDB );
  6382   6376       rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
  6383   6377     }
  6384   6378     return rc;
  6385   6379   }
  6386   6380   
................................................................................
  7121   7115   ** with the pager.  This might return NULL if the file has
  7122   7116   ** not yet been opened.
  7123   7117   */
  7124   7118   sqlite3_file *sqlite3PagerFile(Pager *pPager){
  7125   7119     return pPager->fd;
  7126   7120   }
  7127   7121   
         7122  +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
         7123  +/*
         7124  +** Reset the lock timeout for pager.
         7125  +*/
         7126  +void sqlite3PagerResetLockTimeout(Pager *pPager){
         7127  +  int x = 0;
         7128  +  sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCK_TIMEOUT, &x);
         7129  +}
         7130  +#endif
         7131  +
  7128   7132   /*
  7129   7133   ** Return the file handle for the journal file (if it exists).
  7130   7134   ** This will be either the rollback journal or the WAL file.
  7131   7135   */
  7132   7136   sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){
  7133   7137   #if SQLITE_OMIT_WAL
  7134   7138     return pPager->jfd;
................................................................................
  7581   7585     if( pPager->pWal ){
  7582   7586       rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode,
  7583   7587           (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler),
  7584   7588           pPager->pBusyHandlerArg,
  7585   7589           pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace,
  7586   7590           pnLog, pnCkpt
  7587   7591       );
         7592  +    sqlite3PagerResetLockTimeout(pPager);
  7588   7593     }
  7589   7594     return rc;
  7590   7595   }
  7591   7596   
  7592   7597   int sqlite3PagerWalCallback(Pager *pPager){
  7593   7598     return sqlite3WalCallback(pPager->pWal);
  7594   7599   }

Changes to src/pager.h.

   122    122     int,
   123    123     void(*)(DbPage*)
   124    124   );
   125    125   int sqlite3PagerClose(Pager *pPager, sqlite3*);
   126    126   int sqlite3PagerReadFileheader(Pager*, int, unsigned char*);
   127    127   
   128    128   /* Functions used to configure a Pager object. */
   129         -void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *);
          129  +void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *);
   130    130   int sqlite3PagerSetPagesize(Pager*, u32*, int);
   131    131   #ifdef SQLITE_HAS_CODEC
   132    132   void sqlite3PagerAlignReserve(Pager*,Pager*);
   133    133   #endif
   134    134   int sqlite3PagerMaxPageCount(Pager*, int);
   135    135   void sqlite3PagerSetCachesize(Pager*, int);
   136    136   int sqlite3PagerSetSpillsize(Pager*, int);
................................................................................
   209    209   sqlite3_file *sqlite3PagerJrnlFile(Pager*);
   210    210   const char *sqlite3PagerJournalname(Pager*);
   211    211   void *sqlite3PagerTempSpace(Pager*);
   212    212   int sqlite3PagerIsMemdb(Pager*);
   213    213   void sqlite3PagerCacheStat(Pager *, int, int, int *);
   214    214   void sqlite3PagerClearCache(Pager*);
   215    215   int sqlite3SectorSize(sqlite3_file *);
          216  +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
          217  +void sqlite3PagerResetLockTimeout(Pager *pPager);
          218  +#else
          219  +# define sqlite3PagerResetLockTimeout(X)
          220  +#endif
   216    221   
   217    222   /* Functions used to truncate the database file. */
   218    223   void sqlite3PagerTruncateImage(Pager*,Pgno);
   219    224   
   220    225   void sqlite3PagerRekey(DbPage*, Pgno, u16);
   221    226   
   222    227   #ifndef SQLITE_OMIT_CONCURRENT

Changes to src/parse.y.

    20     20   
    21     21   // The type of the data attached to each token is Token.  This is also the
    22     22   // default type for non-terminals.
    23     23   //
    24     24   %token_type {Token}
    25     25   %default_type {Token}
    26     26   
    27         -// The generated parser function takes a 4th argument as follows:
    28         -%extra_argument {Parse *pParse}
           27  +// An extra argument to the constructor for the parser, which is available
           28  +// to all actions.
           29  +%extra_context {Parse *pParse}
    29     30   
    30     31   // This code runs whenever there is a syntax error
    31     32   //
    32     33   %syntax_error {
    33     34     UNUSED_PARAMETER(yymajor);  /* Silence some compiler warnings */
    34     35     if( TOKEN.z[0] ){
    35     36       sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
................................................................................
   117    118   } // end %include
   118    119   
   119    120   // Input is a single SQL command
   120    121   input ::= cmdlist.
   121    122   cmdlist ::= cmdlist ecmd.
   122    123   cmdlist ::= ecmd.
   123    124   ecmd ::= SEMI.
   124         -ecmd ::= explain cmdx SEMI.
   125         -explain ::= .
          125  +ecmd ::= cmdx SEMI.
   126    126   %ifndef SQLITE_OMIT_EXPLAIN
          127  +ecmd ::= explain cmdx.
   127    128   explain ::= EXPLAIN.              { pParse->explain = 1; }
   128    129   explain ::= EXPLAIN QUERY PLAN.   { pParse->explain = 2; }
   129    130   %endif  SQLITE_OMIT_EXPLAIN
   130    131   cmdx ::= cmd.           { sqlite3FinishCoding(pParse); }
   131    132   
   132    133   ///////////////////// Begin and end transactions. ////////////////////////////
   133    134   //
................................................................................
   217    218   
   218    219   // The following directive causes tokens ABORT, AFTER, ASC, etc. to
   219    220   // fallback to ID if they will not parse as their original value.
   220    221   // This obviates the need for the "id" nonterminal.
   221    222   //
   222    223   %fallback ID
   223    224     ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
   224         -  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
          225  +  CONFLICT DATABASE DEFERRED DESC DETACH DO
          226  +  EACH END EXCLUSIVE EXPLAIN FAIL FOR
   225    227     IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
   226    228     QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
   227    229     ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
   228    230   %ifdef SQLITE_OMIT_COMPOUND_SELECT
   229    231     EXCEPT INTERSECT UNION
   230    232   %endif SQLITE_OMIT_COMPOUND_SELECT
   231    233     REINDEX RENAME CTIME_KW IF
................................................................................
   251    253   %right ESCAPE.
   252    254   %left BITAND BITOR LSHIFT RSHIFT.
   253    255   %left PLUS MINUS.
   254    256   %left STAR SLASH REM.
   255    257   %left CONCAT.
   256    258   %left COLLATE.
   257    259   %right BITNOT.
          260  +%nonassoc ON.
   258    261   
   259    262   // An IDENTIFIER can be a generic identifier, or one of several
   260    263   // keywords.  Any non-standard keyword can also be an identifier.
   261    264   //
   262    265   %token_class id  ID|INDEXED.
   263    266   
   264    267   // And "ids" is an identifer-or-string.
................................................................................
   475    478         ){
   476    479           sqlite3ErrorMsg(pParse, "too many terms in compound SELECT");
   477    480         }
   478    481       }
   479    482     }
   480    483   }
   481    484   
   482         -select(A) ::= with(W) selectnowith(X). {
          485  +select(A) ::= WITH wqlist(W) selectnowith(X). {
          486  +  Select *p = X;
          487  +  if( p ){
          488  +    p->pWith = W;
          489  +    parserDoubleLinkSelect(pParse, p);
          490  +  }else{
          491  +    sqlite3WithDelete(pParse->db, W);
          492  +  }
          493  +  A = p;
          494  +}
          495  +select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X). {
   483    496     Select *p = X;
   484    497     if( p ){
   485    498       p->pWith = W;
   486    499       parserDoubleLinkSelect(pParse, p);
   487    500     }else{
   488    501       sqlite3WithDelete(pParse->db, W);
   489    502     }
   490         -  A = p; /*A-overwrites-W*/
          503  +  A = p;
          504  +}
          505  +select(A) ::= selectnowith(X). {
          506  +  Select *p = X;
          507  +  if( p ){
          508  +    parserDoubleLinkSelect(pParse, p);
          509  +  }
          510  +  A = p; /*A-overwrites-X*/
   491    511   }
   492    512   
   493    513   selectnowith(A) ::= oneselect(A).
   494    514   %ifndef SQLITE_OMIT_COMPOUND_SELECT
   495    515   selectnowith(A) ::= selectnowith(A) multiselect_op(Y) oneselect(Z).  {
   496    516     Select *pRhs = Z;
   497    517     Select *pLhs = A;
................................................................................
   677    697   
   678    698   %type dbnm {Token}
   679    699   dbnm(A) ::= .          {A.z=0; A.n=0;}
   680    700   dbnm(A) ::= DOT nm(X). {A = X;}
   681    701   
   682    702   %type fullname {SrcList*}
   683    703   %destructor fullname {sqlite3SrcListDelete(pParse->db, $$);}
   684         -fullname(A) ::= nm(X) dbnm(Y).  
          704  +fullname(A) ::= nm(X).  
          705  +   {A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/}
          706  +fullname(A) ::= nm(X) DOT nm(Y).  
          707  +   {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/}
          708  +
          709  +%type xfullname {SrcList*}
          710  +%destructor xfullname {sqlite3SrcListDelete(pParse->db, $$);}
          711  +xfullname(A) ::= nm(X).  
          712  +   {A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/}
          713  +xfullname(A) ::= nm(X) DOT nm(Y).  
   685    714      {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/}
          715  +xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z).  {
          716  +   A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/
          717  +   if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z);
          718  +}
          719  +xfullname(A) ::= nm(X) AS nm(Z). {  
          720  +   A = sqlite3SrcListAppend(pParse->db,0,&X,0); /*A-overwrites-X*/
          721  +   if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z);
          722  +}
   686    723   
   687    724   %type joinop {int}
   688    725   joinop(X) ::= COMMA|JOIN.              { X = JT_INNER; }
   689    726   joinop(X) ::= JOIN_KW(A) JOIN.
   690    727                     {X = sqlite3JoinType(pParse,&A,0,0);  /*X-overwrites-A*/}
   691    728   joinop(X) ::= JOIN_KW(A) nm(B) JOIN.
   692    729                     {X = sqlite3JoinType(pParse,&A,&B,0); /*X-overwrites-A*/}
   693    730   joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN.
   694    731                     {X = sqlite3JoinType(pParse,&A,&B,&C);/*X-overwrites-A*/}
   695    732   
          733  +// There is a parsing abiguity in an upsert statement that uses a
          734  +// SELECT on the RHS of a the INSERT:
          735  +//
          736  +//      INSERT INTO tab SELECT * FROM aaa JOIN bbb ON CONFLICT ...
          737  +//                                        here ----^^
          738  +//
          739  +// When the ON token is encountered, the parser does not know if it is
          740  +// the beginning of an ON CONFLICT clause, or the beginning of an ON
          741  +// clause associated with the JOIN.  The conflict is resolved in favor
          742  +// of the JOIN.  If an ON CONFLICT clause is intended, insert a dummy
          743  +// WHERE clause in between, like this:
          744  +//
          745  +//      INSERT INTO tab SELECT * FROM aaa JOIN bbb WHERE true ON CONFLICT ...
          746  +//
          747  +// The [AND] and [OR] precedence marks in the rules for on_opt cause the
          748  +// ON in this context to always be interpreted as belonging to the JOIN.
          749  +//
   696    750   %type on_opt {Expr*}
   697    751   %destructor on_opt {sqlite3ExprDelete(pParse->db, $$);}
   698         -on_opt(N) ::= ON expr(E).   {N = E;}
   699         -on_opt(N) ::= .             {N = 0;}
          752  +on_opt(N) ::= ON expr(E).  {N = E;}
          753  +on_opt(N) ::= .     [OR]   {N = 0;}
   700    754   
   701    755   // Note that this block abuses the Token type just a little. If there is
   702    756   // no "INDEXED BY" clause, the returned token is empty (z==0 && n==0). If
   703    757   // there is an INDEXED BY clause, then the token is populated as per normal,
   704    758   // with z pointing to the token data and n containing the number of bytes
   705    759   // in the token.
   706    760   //
................................................................................
   773    827                            {A = sqlite3PExpr(pParse,TK_LIMIT,X,Y);}
   774    828   limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). 
   775    829                            {A = sqlite3PExpr(pParse,TK_LIMIT,Y,X);}
   776    830   
   777    831   /////////////////////////// The DELETE statement /////////////////////////////
   778    832   //
   779    833   %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   780         -cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
          834  +cmd ::= with DELETE FROM xfullname(X) indexed_opt(I) where_opt(W) 
   781    835           orderby_opt(O) limit_opt(L). {
   782         -  sqlite3WithPush(pParse, C, 1);
   783    836     sqlite3SrcListIndexedBy(pParse, X, &I);
   784    837     sqlite3DeleteFrom(pParse,X,W,O,L);
   785    838   }
   786    839   %endif
   787    840   %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   788         -cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
   789         -  sqlite3WithPush(pParse, C, 1);
          841  +cmd ::= with DELETE FROM xfullname(X) indexed_opt(I) where_opt(W). {
   790    842     sqlite3SrcListIndexedBy(pParse, X, &I);
   791    843     sqlite3DeleteFrom(pParse,X,W,0,0);
   792    844   }
   793    845   %endif
   794    846   
   795    847   %type where_opt {Expr*}
   796    848   %destructor where_opt {sqlite3ExprDelete(pParse->db, $$);}
................................................................................
   797    849   
   798    850   where_opt(A) ::= .                    {A = 0;}
   799    851   where_opt(A) ::= WHERE expr(X).       {A = X;}
   800    852   
   801    853   ////////////////////////// The UPDATE command ////////////////////////////////
   802    854   //
   803    855   %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   804         -cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
          856  +cmd ::= with UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y)
   805    857           where_opt(W) orderby_opt(O) limit_opt(L).  {
   806         -  sqlite3WithPush(pParse, C, 1);
   807    858     sqlite3SrcListIndexedBy(pParse, X, &I);
   808    859     sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   809         -  sqlite3Update(pParse,X,Y,W,R,O,L);
          860  +  sqlite3Update(pParse,X,Y,W,R,O,L,0);
   810    861   }
   811    862   %endif
   812    863   %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   813         -cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
          864  +cmd ::= with UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y)
   814    865           where_opt(W).  {
   815         -  sqlite3WithPush(pParse, C, 1);
   816    866     sqlite3SrcListIndexedBy(pParse, X, &I);
   817    867     sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   818         -  sqlite3Update(pParse,X,Y,W,R,0,0);
          868  +  sqlite3Update(pParse,X,Y,W,R,0,0,0);
   819    869   }
   820    870   %endif
   821    871   
   822    872   %type setlist {ExprList*}
   823    873   %destructor setlist {sqlite3ExprListDelete(pParse->db, $$);}
   824    874   
   825    875   setlist(A) ::= setlist(A) COMMA nm(X) EQ expr(Y). {
................................................................................
   835    885   }
   836    886   setlist(A) ::= LP idlist(X) RP EQ expr(Y). {
   837    887     A = sqlite3ExprListAppendVector(pParse, 0, X, Y);
   838    888   }
   839    889   
   840    890   ////////////////////////// The INSERT command /////////////////////////////////
   841    891   //
   842         -cmd ::= with(W) insert_cmd(R) INTO fullname(X) idlist_opt(F) select(S). {
   843         -  sqlite3WithPush(pParse, W, 1);
   844         -  sqlite3Insert(pParse, X, S, F, R);
          892  +cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) select(S)
          893  +        upsert(U). {
          894  +  sqlite3Insert(pParse, X, S, F, R, U);
   845    895   }
   846         -cmd ::= with(W) insert_cmd(R) INTO fullname(X) idlist_opt(F) DEFAULT VALUES.
          896  +cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) DEFAULT VALUES.
   847    897   {
   848         -  sqlite3WithPush(pParse, W, 1);
   849         -  sqlite3Insert(pParse, X, 0, F, R);
          898  +  sqlite3Insert(pParse, X, 0, F, R, 0);
   850    899   }
   851    900   
          901  +%type upsert {Upsert*}
          902  +
          903  +// Because upsert only occurs at the tip end of the INSERT rule for cmd,
          904  +// there is never a case where the value of the upsert pointer will not
          905  +// be destroyed by the cmd action.  So comment-out the destructor to
          906  +// avoid unreachable code.
          907  +//%destructor upsert {sqlite3UpsertDelete(pParse->db,$$);}
          908  +upsert(A) ::= . { A = 0; }
          909  +upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW)
          910  +              DO UPDATE SET setlist(Z) where_opt(W).
          911  +              { A = sqlite3UpsertNew(pParse->db,T,TW,Z,W);}
          912  +upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW) DO NOTHING.
          913  +              { A = sqlite3UpsertNew(pParse->db,T,TW,0,0); }
          914  +upsert(A) ::= ON CONFLICT DO NOTHING.
          915  +              { A = sqlite3UpsertNew(pParse->db,0,0,0,0); }
          916  +
   852    917   %type insert_cmd {int}
   853    918   insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
   854    919   insert_cmd(A) ::= REPLACE.            {A = OE_Replace;}
   855    920   
   856    921   %type idlist_opt {IdList*}
   857    922   %destructor idlist_opt {sqlite3IdListDelete(pParse->db, $$);}
   858    923   %type idlist {IdList*}
................................................................................
  1390   1455   // UPDATE 
  1391   1456   trigger_cmd(A) ::=
  1392   1457      UPDATE(B) orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z) scanpt(E).  
  1393   1458      {A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R, B.z, E);}
  1394   1459   
  1395   1460   // INSERT
  1396   1461   trigger_cmd(A) ::= scanpt(B) insert_cmd(R) INTO
  1397         -                      trnm(X) idlist_opt(F) select(S) scanpt(Z).
  1398         -   {A = sqlite3TriggerInsertStep(pParse->db,&X,F,S,R,B,Z);/*A-overwrites-R*/}
  1399         -
         1462  +                      trnm(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). {
         1463  +   A = sqlite3TriggerInsertStep(pParse->db,&X,F,S,R,U,B,Z);/*A-overwrites-R*/
         1464  +}
  1400   1465   // DELETE
  1401   1466   trigger_cmd(A) ::= DELETE(B) FROM trnm(X) tridxby where_opt(Y) scanpt(E).
  1402   1467      {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y, B.z, E);}
  1403   1468   
  1404   1469   // SELECT
  1405   1470   trigger_cmd(A) ::= scanpt(B) select(X) scanpt(E).
  1406   1471      {A = sqlite3TriggerSelectStep(pParse->db, X, B, E); /*A-overwrites-X*/}
................................................................................
  1499   1564   anylist ::= .
  1500   1565   anylist ::= anylist LP anylist RP.
  1501   1566   anylist ::= anylist ANY.
  1502   1567   %endif  SQLITE_OMIT_VIRTUALTABLE
  1503   1568   
  1504   1569   
  1505   1570   //////////////////////// COMMON TABLE EXPRESSIONS ////////////////////////////
  1506         -%type with {With*}
  1507   1571   %type wqlist {With*}
  1508         -%destructor with {sqlite3WithDelete(pParse->db, $$);}
  1509   1572   %destructor wqlist {sqlite3WithDelete(pParse->db, $$);}
  1510   1573   
  1511         -with(A) ::= . {A = 0;}
         1574  +with ::= .
  1512   1575   %ifndef SQLITE_OMIT_CTE
  1513         -with(A) ::= WITH wqlist(W).              { A = W; }
  1514         -with(A) ::= WITH RECURSIVE wqlist(W).    { A = W; }
         1576  +with ::= WITH wqlist(W).              { sqlite3WithPush(pParse, W, 1); }
         1577  +with ::= WITH RECURSIVE wqlist(W).    { sqlite3WithPush(pParse, W, 1); }
  1515   1578   
  1516   1579   wqlist(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
  1517   1580     A = sqlite3WithAdd(pParse, 0, &X, Y, Z); /*A-overwrites-X*/
  1518   1581   }
  1519   1582   wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
  1520   1583     A = sqlite3WithAdd(pParse, A, &X, Y, Z);
  1521   1584   }
  1522   1585   %endif  SQLITE_OMIT_CTE

Changes to src/resolve.c.

   187    187     int cntTab = 0;                   /* Number of matching table names */
   188    188     int nSubquery = 0;                /* How many levels of subquery */
   189    189     sqlite3 *db = pParse->db;         /* The database connection */
   190    190     struct SrcList_item *pItem;       /* Use for looping over pSrcList items */
   191    191     struct SrcList_item *pMatch = 0;  /* The matching pSrcList item */
   192    192     NameContext *pTopNC = pNC;        /* First namecontext in the list */
   193    193     Schema *pSchema = 0;              /* Schema of the expression */
   194         -  int isTrigger = 0;                /* True if resolved to a trigger column */
          194  +  int eNewExprOp = TK_COLUMN;       /* New value for pExpr->op on success */
   195    195     Table *pTab = 0;                  /* Table hold the row */
   196    196     Column *pCol;                     /* A column of pTab */
   197    197   
   198    198     assert( pNC );     /* the name context cannot be NULL. */
   199    199     assert( zCol );    /* The Z in X.Y.Z cannot be NULL */
   200    200     assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
   201    201   
................................................................................
   292    292           if( (pMatch->fg.jointype & JT_LEFT)!=0 ){
   293    293             ExprSetProperty(pExpr, EP_CanBeNull);
   294    294           }
   295    295           pSchema = pExpr->pTab->pSchema;
   296    296         }
   297    297       } /* if( pSrcList ) */
   298    298   
   299         -#ifndef SQLITE_OMIT_TRIGGER
          299  +#if !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT)
   300    300       /* If we have not already resolved the name, then maybe 
   301         -    ** it is a new.* or old.* trigger argument reference
          301  +    ** it is a new.* or old.* trigger argument reference.  Or
          302  +    ** maybe it is an excluded.* from an upsert.
   302    303       */
   303         -    if( zDb==0 && zTab!=0 && cntTab==0 && pParse->pTriggerTab!=0 ){
   304         -      int op = pParse->eTriggerOp;
   305         -      assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT );
   306         -      if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){
   307         -        pExpr->iTable = 1;
   308         -        pTab = pParse->pTriggerTab;
   309         -      }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){
   310         -        pExpr->iTable = 0;
   311         -        pTab = pParse->pTriggerTab;
   312         -      }else{
   313         -        pTab = 0;
          304  +    if( zDb==0 && zTab!=0 && cntTab==0 ){
          305  +      pTab = 0;
          306  +#ifndef SQLITE_OMIT_TRIGGER
          307  +      if( pParse->pTriggerTab!=0 ){
          308  +        int op = pParse->eTriggerOp;
          309  +        assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT );
          310  +        if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){
          311  +          pExpr->iTable = 1;
          312  +          pTab = pParse->pTriggerTab;
          313  +        }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){
          314  +          pExpr->iTable = 0;
          315  +          pTab = pParse->pTriggerTab;
          316  +        }
   314    317         }
          318  +#endif /* SQLITE_OMIT_TRIGGER */
          319  +#ifndef SQLITE_OMIT_UPSERT
          320  +      if( (pNC->ncFlags & NC_UUpsert)!=0 ){
          321  +        Upsert *pUpsert = pNC->uNC.pUpsert;
          322  +        if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){
          323  +          pTab = pUpsert->pUpsertSrc->a[0].pTab;
          324  +          pExpr->iTable = 2;
          325  +        }
          326  +      }
          327  +#endif /* SQLITE_OMIT_UPSERT */
   315    328   
   316    329         if( pTab ){ 
   317    330           int iCol;
   318    331           pSchema = pTab->pSchema;
   319    332           cntTab++;
   320    333           for(iCol=0, pCol=pTab->aCol; iCol<pTab->nCol; iCol++, pCol++){
   321    334             if( sqlite3StrICmp(pCol->zName, zCol)==0 ){
................................................................................
   327    340           }
   328    341           if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){
   329    342             /* IMP: R-51414-32910 */
   330    343             iCol = -1;
   331    344           }
   332    345           if( iCol<pTab->nCol ){
   333    346             cnt++;
   334         -          if( iCol<0 ){
   335         -            pExpr->affinity = SQLITE_AFF_INTEGER;
   336         -          }else if( pExpr->iTable==0 ){
   337         -            testcase( iCol==31 );
   338         -            testcase( iCol==32 );
   339         -            pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
   340         -          }else{
   341         -            testcase( iCol==31 );
   342         -            testcase( iCol==32 );
   343         -            pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
          347  +#ifndef SQLITE_OMIT_UPSERT
          348  +          if( pExpr->iTable==2 ){
          349  +            testcase( iCol==(-1) );
          350  +            pExpr->iTable = pNC->uNC.pUpsert->regData + iCol;
          351  +            eNewExprOp = TK_REGISTER;
          352  +          }else
          353  +#endif /* SQLITE_OMIT_UPSERT */
          354  +          {
          355  +#ifndef SQLITE_OMIT_TRIGGER
          356  +            if( iCol<0 ){
          357  +              pExpr->affinity = SQLITE_AFF_INTEGER;
          358  +            }else if( pExpr->iTable==0 ){
          359  +              testcase( iCol==31 );
          360  +              testcase( iCol==32 );
          361  +              pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
          362  +            }else{
          363  +              testcase( iCol==31 );
          364  +              testcase( iCol==32 );
          365  +              pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
          366  +            }
          367  +            pExpr->pTab = pTab;
          368  +            pExpr->iColumn = (i16)iCol;
          369  +            eNewExprOp = TK_TRIGGER;
          370  +#endif /* SQLITE_OMIT_TRIGGER */
   344    371             }
   345         -          pExpr->iColumn = (i16)iCol;
   346         -          pExpr->pTab = pTab;
   347         -          isTrigger = 1;
   348    372           }
   349    373         }
   350    374       }
   351         -#endif /* !defined(SQLITE_OMIT_TRIGGER) */
          375  +#endif /* !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) */
   352    376   
   353    377       /*
   354    378       ** Perhaps the name is a reference to the ROWID
   355    379       */
   356    380       if( cnt==0
   357    381        && cntTab==1
   358    382        && pMatch
................................................................................
   379    403       **
   380    404       ** The ability to use an output result-set column in the WHERE, GROUP BY,
   381    405       ** or HAVING clauses, or as part of a larger expression in the ORDER BY
   382    406       ** clause is not standard SQL.  This is a (goofy) SQLite extension, that
   383    407       ** is supported for backwards compatibility only. Hence, we issue a warning
   384    408       ** on sqlite3_log() whenever the capability is used.
   385    409       */
   386         -    if( (pEList = pNC->pEList)!=0
   387         -     && zTab==0
          410  +    if( (pNC->ncFlags & NC_UEList)!=0
   388    411        && cnt==0
          412  +     && zTab==0
   389    413       ){
          414  +      pEList = pNC->uNC.pEList;
          415  +      assert( pEList!=0 );
   390    416         for(j=0; j<pEList->nExpr; j++){
   391    417           char *zAs = pEList->a[j].zName;
   392    418           if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){
   393    419             Expr *pOrig;
   394    420             assert( pExpr->pLeft==0 && pExpr->pRight==0 );
   395    421             assert( pExpr->x.pList==0 );
   396    422             assert( pExpr->x.pSelect==0 );
................................................................................
   479    505   
   480    506     /* Clean up and return
   481    507     */
   482    508     sqlite3ExprDelete(db, pExpr->pLeft);
   483    509     pExpr->pLeft = 0;
   484    510     sqlite3ExprDelete(db, pExpr->pRight);
   485    511     pExpr->pRight = 0;
   486         -  pExpr->op = (isTrigger ? TK_TRIGGER : TK_COLUMN);
          512  +  pExpr->op = eNewExprOp;
   487    513     ExprSetProperty(pExpr, EP_Leaf);
   488    514   lookupname_end:
   489    515     if( cnt==1 ){
   490    516       assert( pNC!=0 );
   491    517       if( !ExprHasProperty(pExpr, EP_Alias) ){
   492    518         sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList);
   493    519       }
................................................................................
   911    937     pEList = pSelect->pEList;
   912    938   
   913    939     /* Resolve all names in the ORDER BY term expression
   914    940     */
   915    941     memset(&nc, 0, sizeof(nc));
   916    942     nc.pParse = pParse;
   917    943     nc.pSrcList = pSelect->pSrc;
   918         -  nc.pEList = pEList;
   919         -  nc.ncFlags = NC_AllowAgg;
          944  +  nc.uNC.pEList = pEList;
          945  +  nc.ncFlags = NC_AllowAgg|NC_UEList;
   920    946     nc.nErr = 0;
   921    947     db = pParse->db;
   922    948     savedSuppErr = db->suppressErr;
   923    949     db->suppressErr = 1;
   924    950     rc = sqlite3ResolveExprNames(&nc, pE);
   925    951     db->suppressErr = savedSuppErr;
   926    952     if( rc ) return 0;
................................................................................
  1295   1321       ** other expressions in the SELECT statement. This is so that
  1296   1322       ** expressions in the WHERE clause (etc.) can refer to expressions by
  1297   1323       ** aliases in the result set.
  1298   1324       **
  1299   1325       ** Minor point: If this is the case, then the expression will be
  1300   1326       ** re-evaluated for each reference to it.
  1301   1327       */
  1302         -    sNC.pEList = p->pEList;
         1328  +    assert( (sNC.ncFlags & (NC_UAggInfo|NC_UUpsert))==0 );
         1329  +    sNC.uNC.pEList = p->pEList;
         1330  +    sNC.ncFlags |= NC_UEList;
  1303   1331       if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort;
  1304   1332       if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort;
  1305   1333   
  1306   1334       /* Resolve names in table-valued-function arguments */
  1307   1335       for(i=0; i<p->pSrc->nSrc; i++){
  1308   1336         struct SrcList_item *pItem = &p->pSrc->a[i];
  1309   1337         if( pItem->fg.isTabFunc
................................................................................
  1528   1556   ** Any errors cause an error message to be set in pParse.
  1529   1557   */
  1530   1558   void sqlite3ResolveSelfReference(
  1531   1559     Parse *pParse,      /* Parsing context */
  1532   1560     Table *pTab,        /* The table being referenced */
  1533   1561     int type,           /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */
  1534   1562     Expr *pExpr,        /* Expression to resolve.  May be NULL. */
  1535         -  ExprList *pList     /* Expression list to resolve.  May be NUL. */
         1563  +  ExprList *pList     /* Expression list to resolve.  May be NULL. */
  1536   1564   ){
  1537   1565     SrcList sSrc;                   /* Fake SrcList for pParse->pNewTable */
  1538   1566     NameContext sNC;                /* Name context for pParse->pNewTable */
  1539   1567   
  1540   1568     assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr );
  1541   1569     memset(&sNC, 0, sizeof(sNC));
  1542   1570     memset(&sSrc, 0, sizeof(sSrc));

Changes to src/select.c.

    17     17   /*
    18     18   ** Trace output macros
    19     19   */
    20     20   #if SELECTTRACE_ENABLED
    21     21   /***/ int sqlite3SelectTrace = 0;
    22     22   # define SELECTTRACE(K,P,S,X)  \
    23     23     if(sqlite3SelectTrace&(K))   \
    24         -    sqlite3DebugPrintf("%s/%p: ",(S)->zSelName,(S)),\
           24  +    sqlite3DebugPrintf("%s/%d/%p: ",(S)->zSelName,(P)->iSelectId,(S)),\
    25     25       sqlite3DebugPrintf X
    26     26   #else
    27     27   # define SELECTTRACE(K,P,S,X)
    28     28   #endif
    29     29   
    30     30   
    31     31   /*
................................................................................
    40     40     int tabTnct;    /* Ephemeral table used for DISTINCT processing */
    41     41     int addrTnct;   /* Address of OP_OpenEphemeral opcode for tabTnct */
    42     42   };
    43     43   
    44     44   /*
    45     45   ** An instance of the following object is used to record information about
    46     46   ** the ORDER BY (or GROUP BY) clause of query is being coded.
           47  +**
           48  +** The aDefer[] array is used by the sorter-references optimization. For
           49  +** example, assuming there is no index that can be used for the ORDER BY,
           50  +** for the query:
           51  +**
           52  +**     SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10;
           53  +**
           54  +** it may be more efficient to add just the "a" values to the sorter, and
           55  +** retrieve the associated "bigblob" values directly from table t1 as the
           56  +** 10 smallest "a" values are extracted from the sorter.
           57  +**
           58  +** When the sorter-reference optimization is used, there is one entry in the
           59  +** aDefer[] array for each database table that may be read as values are
           60  +** extracted from the sorter.
    47     61   */
    48     62   typedef struct SortCtx SortCtx;
    49     63   struct SortCtx {
    50     64     ExprList *pOrderBy;   /* The ORDER BY (or GROUP BY clause) */
    51     65     int nOBSat;           /* Number of ORDER BY terms satisfied by indices */
    52     66     int iECursor;         /* Cursor number for the sorter */
    53     67     int regReturn;        /* Register holding block-output return address */
    54     68     int labelBkOut;       /* Start label for the block-output subroutine */
    55     69     int addrSortIndex;    /* Address of the OP_SorterOpen or OP_OpenEphemeral */
    56     70     int labelDone;        /* Jump here when done, ex: LIMIT reached */
    57     71     u8 sortFlags;         /* Zero or more SORTFLAG_* bits */
    58     72     u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */
           73  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
           74  +  u8 nDefer;            /* Number of valid entries in aDefer[] */
           75  +  struct DeferredCsr {
           76  +    Table *pTab;        /* Table definition */
           77  +    int iCsr;           /* Cursor number for table */
           78  +    int nKey;           /* Number of PK columns for table pTab (>=1) */
           79  +  } aDefer[4];
           80  +#endif
    59     81   };
    60     82   #define SORTFLAG_UseSorter  0x01   /* Use SorterOpen instead of OpenEphemeral */
    61     83   
    62     84   /*
    63     85   ** Delete all the content of a Select structure.  Deallocate the structure
    64     86   ** itself only if bFree is true.
    65     87   */
................................................................................
   377    399           setJoinExpr(p->x.pList->a[i].pExpr, iTable);
   378    400         }
   379    401       }
   380    402       setJoinExpr(p->pLeft, iTable);
   381    403       p = p->pRight;
   382    404     } 
   383    405   }
          406  +
          407  +/* Undo the work of setJoinExpr().  In the expression tree p, convert every
          408  +** term that is marked with EP_FromJoin and iRightJoinTable==iTable into
          409  +** an ordinary term that omits the EP_FromJoin mark.
          410  +**
          411  +** This happens when a LEFT JOIN is simplified into an ordinary JOIN.
          412  +*/
          413  +static void unsetJoinExpr(Expr *p, int iTable){
          414  +  while( p ){
          415  +    if( ExprHasProperty(p, EP_FromJoin)
          416  +     && (iTable<0 || p->iRightJoinTable==iTable) ){
          417  +      ExprClearProperty(p, EP_FromJoin);
          418  +    }
          419  +    if( p->op==TK_FUNCTION && p->x.pList ){
          420  +      int i;
          421  +      for(i=0; i<p->x.pList->nExpr; i++){
          422  +        unsetJoinExpr(p->x.pList->a[i].pExpr, iTable);
          423  +      }
          424  +    }
          425  +    unsetJoinExpr(p->pLeft, iTable);
          426  +    p = p->pRight;
          427  +  } 
          428  +}
   384    429   
   385    430   /*
   386    431   ** This routine processes the join information for a SELECT statement.
   387    432   ** ON and USING clauses are converted into extra terms of the WHERE clause.
   388    433   ** NATURAL joins also create extra WHERE clause terms.
   389    434   **
   390    435   ** The terms of a FROM clause are contained in the Select.pSrc structure.
................................................................................
   651    696     sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v);
   652    697     sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);
   653    698     sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, iMem, N);
   654    699     sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   655    700     sqlite3ReleaseTempReg(pParse, r1);
   656    701   }
   657    702   
          703  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          704  +/*
          705  +** This function is called as part of inner-loop generation for a SELECT
          706  +** statement with an ORDER BY that is not optimized by an index. It 
          707  +** determines the expressions, if any, that the sorter-reference 
          708  +** optimization should be used for. The sorter-reference optimization
          709  +** is used for SELECT queries like:
          710  +**
          711  +**   SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10
          712  +**
          713  +** If the optimization is used for expression "bigblob", then instead of
          714  +** storing values read from that column in the sorter records, the PK of
          715  +** the row from table t1 is stored instead. Then, as records are extracted from
          716  +** the sorter to return to the user, the required value of bigblob is
          717  +** retrieved directly from table t1. If the values are very large, this 
          718  +** can be more efficient than storing them directly in the sorter records.
          719  +**
          720  +** The ExprList_item.bSorterRef flag is set for each expression in pEList 
          721  +** for which the sorter-reference optimization should be enabled. 
          722  +** Additionally, the pSort->aDefer[] array is populated with entries
          723  +** for all cursors required to evaluate all selected expressions. Finally.
          724  +** output variable (*ppExtra) is set to an expression list containing
          725  +** expressions for all extra PK values that should be stored in the
          726  +** sorter records.
          727  +*/
          728  +static void selectExprDefer(
          729  +  Parse *pParse,                  /* Leave any error here */
          730  +  SortCtx *pSort,                 /* Sorter context */
          731  +  ExprList *pEList,               /* Expressions destined for sorter */
          732  +  ExprList **ppExtra              /* Expressions to append to sorter record */
          733  +){
          734  +  int i;
          735  +  int nDefer = 0;
          736  +  ExprList *pExtra = 0;
          737  +  for(i=0; i<pEList->nExpr; i++){
          738  +    struct ExprList_item *pItem = &pEList->a[i];
          739  +    if( pItem->u.x.iOrderByCol==0 ){
          740  +      Expr *pExpr = pItem->pExpr;
          741  +      Table *pTab = pExpr->pTab;
          742  +      if( pExpr->op==TK_COLUMN && pTab && !IsVirtual(pTab)
          743  +       && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF)
          744  +#if 0
          745  +          && pTab->pSchema && pTab->pSelect==0 && !IsVirtual(pTab)
          746  +#endif
          747  +      ){
          748  +        int j;
          749  +        for(j=0; j<nDefer; j++){
          750  +          if( pSort->aDefer[j].iCsr==pExpr->iTable ) break;
          751  +        }
          752  +        if( j==nDefer ){
          753  +          if( nDefer==ArraySize(pSort->aDefer) ){
          754  +            continue;
          755  +          }else{
          756  +            int nKey = 1;
          757  +            int k;
          758  +            Index *pPk = 0;
          759  +            if( !HasRowid(pTab) ){
          760  +              pPk = sqlite3PrimaryKeyIndex(pTab);
          761  +              nKey = pPk->nKeyCol;
          762  +            }
          763  +            for(k=0; k<nKey; k++){
          764  +              Expr *pNew = sqlite3PExpr(pParse, TK_COLUMN, 0, 0);
          765  +              if( pNew ){
          766  +                pNew->iTable = pExpr->iTable;
          767  +                pNew->pTab = pExpr->pTab;
          768  +                pNew->iColumn = pPk ? pPk->aiColumn[k] : -1;
          769  +                pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew);
          770  +              }
          771  +            }
          772  +            pSort->aDefer[nDefer].pTab = pExpr->pTab;
          773  +            pSort->aDefer[nDefer].iCsr = pExpr->iTable;
          774  +            pSort->aDefer[nDefer].nKey = nKey;
          775  +            nDefer++;
          776  +          }
          777  +        }
          778  +        pItem->bSorterRef = 1;
          779  +      }
          780  +    }
          781  +  }
          782  +  pSort->nDefer = (u8)nDefer;
          783  +  *ppExtra = pExtra;
          784  +}
          785  +#endif
          786  +
   658    787   /*
   659    788   ** This routine generates the code for the inside of the inner loop
   660    789   ** of a SELECT.
   661    790   **
   662    791   ** If srcTab is negative, then the p->pEList expressions
   663    792   ** are evaluated in order to get the data for this row.  If srcTab is
   664    793   ** zero or more, then data is pulled from srcTab and p->pEList is used only 
................................................................................
   723    852     regOrig = regResult = pDest->iSdst;
   724    853     if( srcTab>=0 ){
   725    854       for(i=0; i<nResultCol; i++){
   726    855         sqlite3VdbeAddOp3(v, OP_Column, srcTab, i, regResult+i);
   727    856         VdbeComment((v, "%s", p->pEList->a[i].zName));
   728    857       }
   729    858     }else if( eDest!=SRT_Exists ){
          859  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          860  +    ExprList *pExtra = 0;
          861  +#endif
   730    862       /* If the destination is an EXISTS(...) expression, the actual
   731    863       ** values returned by the SELECT are not required.
   732    864       */
   733    865       u8 ecelFlags;
   734    866       if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){
   735    867         ecelFlags = SQLITE_ECEL_DUP;
   736    868       }else{
................................................................................
   746    878         ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF);
   747    879         for(i=pSort->nOBSat; i<pSort->pOrderBy->nExpr; i++){
   748    880           int j;
   749    881           if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){
   750    882             p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat;
   751    883           }
   752    884         }
          885  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          886  +      selectExprDefer(pParse, pSort, p->pEList, &pExtra);
          887  +      if( pExtra && pParse->db->mallocFailed==0 ){
          888  +        /* If there are any extra PK columns to add to the sorter records,
          889  +        ** allocate extra memory cells and adjust the OpenEphemeral 
          890  +        ** instruction to account for the larger records. This is only
          891  +        ** required if there are one or more WITHOUT ROWID tables with
          892  +        ** composite primary keys in the SortCtx.aDefer[] array.  */
          893  +        VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
          894  +        pOp->p2 += (pExtra->nExpr - pSort->nDefer);
          895  +        pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer);
          896  +        pParse->nMem += pExtra->nExpr;
          897  +      }
          898  +#endif
   753    899         regOrig = 0;
   754    900         assert( eDest==SRT_Set || eDest==SRT_Mem 
   755    901              || eDest==SRT_Coroutine || eDest==SRT_Output );
   756    902       }
   757    903       nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult,
   758    904                                            0,ecelFlags);
          905  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
          906  +    if( pExtra ){
          907  +      nResultCol += sqlite3ExprCodeExprList(
          908  +          pParse, pExtra, regResult + nResultCol, 0, 0
          909  +      );
          910  +      sqlite3ExprListDelete(pParse->db, pExtra);
          911  +    }
          912  +#endif
   759    913     }
   760    914   
   761    915     /* If the DISTINCT keyword was present on the SELECT statement
   762    916     ** and this row has been seen before, then do not make this row
   763    917     ** part of the result.
   764    918     */
   765    919     if( hasDistinct ){
................................................................................
  1209   1363     SortCtx *pSort,   /* Information on the ORDER BY clause */
  1210   1364     int nColumn,      /* Number of columns of data */
  1211   1365     SelectDest *pDest /* Write the sorted results here */
  1212   1366   ){
  1213   1367     Vdbe *v = pParse->pVdbe;                     /* The prepared statement */
  1214   1368     int addrBreak = pSort->labelDone;            /* Jump here to exit loop */
  1215   1369     int addrContinue = sqlite3VdbeMakeLabel(v);  /* Jump here for next cycle */
  1216         -  int addr;
         1370  +  int addr;                       /* Top of output loop. Jump for Next. */
  1217   1371     int addrOnce = 0;
  1218   1372     int iTab;
  1219   1373     ExprList *pOrderBy = pSort->pOrderBy;
  1220   1374     int eDest = pDest->eDest;
  1221   1375     int iParm = pDest->iSDParm;
  1222   1376     int regRow;
  1223   1377     int regRowid;
  1224   1378     int iCol;
  1225         -  int nKey;
         1379  +  int nKey;                       /* Number of key columns in sorter record */
  1226   1380     int iSortTab;                   /* Sorter cursor to read from */
  1227         -  int nSortData;                  /* Trailing values to read from sorter */
  1228   1381     int i;
  1229   1382     int bSeq;                       /* True if sorter record includes seq. no. */
         1383  +  int nRefKey = 0;
  1230   1384     struct ExprList_item *aOutEx = p->pEList->a;
  1231   1385   
  1232   1386     assert( addrBreak<0 );
  1233   1387     if( pSort->labelBkOut ){
  1234   1388       sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
  1235   1389       sqlite3VdbeGoto(v, addrBreak);
  1236   1390       sqlite3VdbeResolveLabel(v, pSort->labelBkOut);
  1237   1391     }
         1392  +
         1393  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1394  +  /* Open any cursors needed for sorter-reference expressions */
         1395  +  for(i=0; i<pSort->nDefer; i++){
         1396  +    Table *pTab = pSort->aDefer[i].pTab;
         1397  +    int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
         1398  +    sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead);
         1399  +    nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey);
         1400  +  }
         1401  +#endif
         1402  +
  1238   1403     iTab = pSort->iECursor;
  1239   1404     if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){
  1240   1405       regRowid = 0;
  1241   1406       regRow = pDest->iSdst;
  1242         -    nSortData = nColumn;
  1243   1407     }else{
  1244   1408       regRowid = sqlite3GetTempReg(pParse);
  1245   1409       regRow = sqlite3GetTempRange(pParse, nColumn);
  1246         -    nSortData = nColumn;
  1247   1410     }
  1248   1411     nKey = pOrderBy->nExpr - pSort->nOBSat;
  1249   1412     if( pSort->sortFlags & SORTFLAG_UseSorter ){
  1250   1413       int regSortOut = ++pParse->nMem;
  1251   1414       iSortTab = pParse->nTab++;
  1252   1415       if( pSort->labelBkOut ){
  1253   1416         addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
  1254   1417       }
  1255         -    sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData);
         1418  +    sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, 
         1419  +        nKey+1+nColumn+nRefKey);
  1256   1420       if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
  1257   1421       addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
  1258   1422       VdbeCoverage(v);
  1259   1423       codeOffset(v, p->iOffset, addrContinue);
  1260   1424       sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab);
  1261   1425       bSeq = 0;
  1262   1426     }else{
  1263   1427       addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
  1264   1428       codeOffset(v, p->iOffset, addrContinue);
  1265   1429       iSortTab = iTab;
  1266   1430       bSeq = 1;
  1267   1431     }
  1268         -  for(i=0, iCol=nKey+bSeq-1; i<nSortData; i++){
         1432  +  for(i=0, iCol=nKey+bSeq-1; i<nColumn; i++){
         1433  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1434  +    if( aOutEx[i].bSorterRef ) continue;
         1435  +#endif
  1269   1436       if( aOutEx[i].u.x.iOrderByCol==0 ) iCol++;
  1270   1437     }
  1271         -  for(i=nSortData-1; i>=0; i--){
  1272         -    int iRead;
  1273         -    if( aOutEx[i].u.x.iOrderByCol ){
  1274         -      iRead = aOutEx[i].u.x.iOrderByCol-1;
  1275         -    }else{
  1276         -      iRead = iCol--;
  1277         -    }
  1278         -    sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
  1279         -    VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));
         1438  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1439  +  if( pSort->nDefer ){
         1440  +    int iKey = iCol+1;
         1441  +    int regKey = sqlite3GetTempRange(pParse, nRefKey);
         1442  +
         1443  +    for(i=0; i<pSort->nDefer; i++){
         1444  +      int iCsr = pSort->aDefer[i].iCsr;
         1445  +      Table *pTab = pSort->aDefer[i].pTab;
         1446  +      int nKey = pSort->aDefer[i].nKey;
         1447  +
         1448  +      sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
         1449  +      if( HasRowid(pTab) ){
         1450  +        sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey);
         1451  +        sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, 
         1452  +            sqlite3VdbeCurrentAddr(v)+1, regKey);
         1453  +      }else{
         1454  +        int k;
         1455  +        int iJmp;
         1456  +        assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey );
         1457  +        for(k=0; k<nKey; k++){
         1458  +          sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey+k);
         1459  +        }
         1460  +        iJmp = sqlite3VdbeCurrentAddr(v);
         1461  +        sqlite3VdbeAddOp4Int(v, OP_SeekGE, iCsr, iJmp+2, regKey, nKey);
         1462  +        sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp+3, regKey, nKey);
         1463  +        sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
         1464  +      }
         1465  +    }
         1466  +    sqlite3ReleaseTempRange(pParse, regKey, nRefKey);
         1467  +  }
         1468  +#endif
         1469  +  for(i=nColumn-1; i>=0; i--){
         1470  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         1471  +    if( aOutEx[i].bSorterRef ){
         1472  +      sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i);
         1473  +    }else
         1474  +#endif
         1475  +    {
         1476  +      int iRead;
         1477  +      if( aOutEx[i].u.x.iOrderByCol ){
         1478  +        iRead = aOutEx[i].u.x.iOrderByCol-1;
         1479  +      }else{
         1480  +        iRead = iCol--;
         1481  +      }
         1482  +      sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
         1483  +      VdbeComment((v, "%s", aOutEx[i].zName?aOutEx[i].zName : aOutEx[i].zSpan));
         1484  +    }
  1280   1485     }
  1281   1486     switch( eDest ){
  1282   1487       case SRT_Table:
  1283   1488       case SRT_EphemTab: {
  1284   1489         sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid);
  1285   1490         sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid);
  1286   1491         sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
................................................................................
  3610   3815       if( pNew==0 ){
  3611   3816         p->pPrior = pPrior;
  3612   3817       }else{
  3613   3818         pNew->pPrior = pPrior;
  3614   3819         if( pPrior ) pPrior->pNext = pNew;
  3615   3820         pNew->pNext = p;
  3616   3821         p->pPrior = pNew;
  3617         -      SELECTTRACE(2,pParse,p,
  3618         -         ("compound-subquery flattener creates %s.%p as peer\n",
  3619         -         pNew->zSelName, pNew));
         3822  +      SELECTTRACE(2,pParse,p,("compound-subquery flattener"
         3823  +                              " creates %s.%p as peer\n",pNew->zSelName, pNew));
  3620   3824       }
  3621   3825       if( db->mallocFailed ) return 1;
  3622   3826     }
  3623   3827   
  3624   3828     /* Begin flattening the iFrom-th entry of the FROM clause 
  3625   3829     ** in the outer query.
  3626   3830     */
................................................................................
  3746   3950         ** (the only way this can happen is if the compound sub-query is
  3747   3951         ** currently part of pSub->pSrc). See ticket [d11a6e908f].  */
  3748   3952         ExprList *pOrderBy = pSub->pOrderBy;
  3749   3953         for(i=0; i<pOrderBy->nExpr; i++){
  3750   3954           pOrderBy->a[i].u.x.iOrderByCol = 0;
  3751   3955         }
  3752   3956         assert( pParent->pOrderBy==0 );
  3753         -      assert( pSub->pPrior==0 );
  3754   3957         pParent->pOrderBy = pOrderBy;
  3755   3958         pSub->pOrderBy = 0;
  3756   3959       }
  3757   3960       pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
  3758   3961       if( isLeftJoin>0 ){
  3759   3962         setJoinExpr(pWhere, iNewParent);
  3760   3963       }
................................................................................
  3830   4033   **           to suppress it. **)
  3831   4034   **
  3832   4035   **   (2) The inner query is the recursive part of a common table expression.
  3833   4036   **
  3834   4037   **   (3) The inner query has a LIMIT clause (since the changes to the WHERE
  3835   4038   **       close would change the meaning of the LIMIT).
  3836   4039   **
  3837         -**   (4) The inner query is the right operand of a LEFT JOIN.  (The caller
  3838         -**       enforces this restriction since this routine does not have enough
  3839         -**       information to know.)
         4040  +**   (4) The inner query is the right operand of a LEFT JOIN and the
         4041  +**       expression to be pushed down does not come from the ON clause
         4042  +**       on that LEFT JOIN.
  3840   4043   **
  3841   4044   **   (5) The WHERE clause expression originates in the ON or USING clause
  3842         -**       of a LEFT JOIN.
         4045  +**       of a LEFT JOIN where iCursor is not the right-hand table of that
         4046  +**       left join.  An example:
         4047  +**
         4048  +**           SELECT *
         4049  +**           FROM (SELECT 1 AS a1 UNION ALL SELECT 2) AS aa
         4050  +**           JOIN (SELECT 1 AS b2 UNION ALL SELECT 2) AS bb ON (a1=b2)
         4051  +**           LEFT JOIN (SELECT 8 AS c3 UNION ALL SELECT 9) AS cc ON (b2=2);
         4052  +**
         4053  +**       The correct answer is three rows:  (1,1,NULL),(2,2,8),(2,2,9).
         4054  +**       But if the (b2=2) term were to be pushed down into the bb subquery,
         4055  +**       then the (1,1,NULL) row would be suppressed.
  3843   4056   **
  3844   4057   ** Return 0 if no changes are made and non-zero if one or more WHERE clause
  3845   4058   ** terms are duplicated into the subquery.
  3846   4059   */
  3847   4060   static int pushDownWhereTerms(
  3848   4061     Parse *pParse,        /* Parse context (for malloc() and error reporting) */
  3849   4062     Select *pSubq,        /* The subquery whose WHERE clause is to be augmented */
  3850   4063     Expr *pWhere,         /* The WHERE clause of the outer query */
  3851         -  int iCursor           /* Cursor number of the subquery */
         4064  +  int iCursor,          /* Cursor number of the subquery */
         4065  +  int isLeftJoin        /* True if pSubq is the right term of a LEFT JOIN */
  3852   4066   ){
  3853   4067     Expr *pNew;
  3854   4068     int nChng = 0;
  3855   4069     if( pWhere==0 ) return 0;
  3856   4070     if( pSubq->selFlags & SF_Recursive ) return 0;  /* restriction (2) */
  3857   4071   
  3858   4072   #ifdef SQLITE_DEBUG
................................................................................
  3868   4082     }
  3869   4083   #endif
  3870   4084   
  3871   4085     if( pSubq->pLimit!=0 ){
  3872   4086       return 0; /* restriction (3) */
  3873   4087     }
  3874   4088     while( pWhere->op==TK_AND ){
  3875         -    nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, iCursor);
         4089  +    nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight,
         4090  +                                iCursor, isLeftJoin);
  3876   4091       pWhere = pWhere->pLeft;
  3877   4092     }
  3878         -  if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction (5) */
         4093  +  if( isLeftJoin
         4094  +   && (ExprHasProperty(pWhere,EP_FromJoin)==0
         4095  +         || pWhere->iRightJoinTable!=iCursor)
         4096  +  ){
         4097  +    return 0; /* restriction (4) */
         4098  +  }
         4099  +  if( ExprHasProperty(pWhere,EP_FromJoin) && pWhere->iRightJoinTable!=iCursor ){
         4100  +    return 0; /* restriction (5) */
         4101  +  }
  3879   4102     if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){
  3880   4103       nChng++;
  3881   4104       while( pSubq ){
  3882   4105         SubstContext x;
  3883   4106         pNew = sqlite3ExprDup(pParse->db, pWhere, 0);
         4107  +      unsetJoinExpr(pNew, -1);
  3884   4108         x.pParse = pParse;
  3885   4109         x.iTable = iCursor;
  3886   4110         x.iNewTable = iCursor;
  3887   4111         x.isLeftJoin = 0;
  3888   4112         x.pEList = pSubq->pEList;
  3889   4113         pNew = substExpr(&x, pNew);
  3890   4114         if( pSubq->selFlags & SF_Aggregate ){
................................................................................
  4331   4555     }
  4332   4556     assert( p->pSrc!=0 );
  4333   4557     if( (selFlags & SF_Expanded)!=0 ){
  4334   4558       return WRC_Prune;
  4335   4559     }
  4336   4560     pTabList = p->pSrc;
  4337   4561     pEList = p->pEList;
  4338         -  if( OK_IF_ALWAYS_TRUE(p->pWith) ){
  4339         -    sqlite3WithPush(pParse, p->pWith, 0);
  4340         -  }
         4562  +  sqlite3WithPush(pParse, p->pWith, 0);
  4341   4563   
  4342   4564     /* Make sure cursor numbers have been assigned to all entries in
  4343   4565     ** the FROM clause of the SELECT statement.
  4344   4566     */
  4345   4567     sqlite3SrcListAssignCursors(pParse, pTabList);
  4346   4568   
  4347   4569     /* Look up every table named in the FROM clause of the select.  If
................................................................................
  4908   5130       );
  4909   5131     }
  4910   5132   }
  4911   5133   #else
  4912   5134   # define explainSimpleCount(a,b,c)
  4913   5135   #endif
  4914   5136   
  4915         -/*
  4916         -** Context object for havingToWhereExprCb().
  4917         -*/
  4918         -struct HavingToWhereCtx {
  4919         -  Expr **ppWhere;
  4920         -  ExprList *pGroupBy;
  4921         -};
  4922         -
  4923   5137   /*
  4924   5138   ** sqlite3WalkExpr() callback used by havingToWhere().
  4925   5139   **
  4926   5140   ** If the node passed to the callback is a TK_AND node, return 
  4927   5141   ** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes.
  4928   5142   **
  4929   5143   ** Otherwise, return WRC_Prune. In this case, also check if the 
  4930   5144   ** sub-expression matches the criteria for being moved to the WHERE
  4931   5145   ** clause. If so, add it to the WHERE clause and replace the sub-expression
  4932   5146   ** within the HAVING expression with a constant "1".
  4933   5147   */
  4934   5148   static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){
  4935   5149     if( pExpr->op!=TK_AND ){
  4936         -    struct HavingToWhereCtx *p = pWalker->u.pHavingCtx;
  4937         -    if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, p->pGroupBy) ){
         5150  +    Select *pS = pWalker->u.pSelect;
         5151  +    if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) ){
  4938   5152         sqlite3 *db = pWalker->pParse->db;
  4939   5153         Expr *pNew = sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[1], 0);
  4940   5154         if( pNew ){
  4941         -        Expr *pWhere = *(p->ppWhere);
         5155  +        Expr *pWhere = pS->pWhere;
  4942   5156           SWAP(Expr, *pNew, *pExpr);
  4943   5157           pNew = sqlite3ExprAnd(db, pWhere, pNew);
  4944         -        *(p->ppWhere) = pNew;
         5158  +        pS->pWhere = pNew;
         5159  +        pWalker->eCode = 1;
  4945   5160         }
  4946   5161       }
  4947   5162       return WRC_Prune;
  4948   5163     }
  4949   5164     return WRC_Continue;
  4950   5165   }
  4951   5166   
................................................................................
  4960   5175   **
  4961   5176   **   SELECT * FROM <tables> WHERE a=? AND b=? GROUP BY b HAVING c=?
  4962   5177   **
  4963   5178   ** A term of the HAVING expression is eligible for transfer if it consists
  4964   5179   ** entirely of constants and expressions that are also GROUP BY terms that
  4965   5180   ** use the "BINARY" collation sequence.
  4966   5181   */
  4967         -static void havingToWhere(
  4968         -  Parse *pParse,
  4969         -  ExprList *pGroupBy,
  4970         -  Expr *pHaving, 
  4971         -  Expr **ppWhere
  4972         -){
  4973         -  struct HavingToWhereCtx sCtx;
         5182  +static void havingToWhere(Parse *pParse, Select *p){
  4974   5183     Walker sWalker;
  4975         -
  4976         -  sCtx.ppWhere = ppWhere;
  4977         -  sCtx.pGroupBy = pGroupBy;
  4978         -
  4979   5184     memset(&sWalker, 0, sizeof(sWalker));
  4980   5185     sWalker.pParse = pParse;
  4981   5186     sWalker.xExprCallback = havingToWhereExprCb;
  4982         -  sWalker.u.pHavingCtx = &sCtx;
  4983         -  sqlite3WalkExpr(&sWalker, pHaving);
         5187  +  sWalker.u.pSelect = p;
         5188  +  sqlite3WalkExpr(&sWalker, p->pHaving);
         5189  +#if SELECTTRACE_ENABLED
         5190  +  if( sWalker.eCode && (sqlite3SelectTrace & 0x100)!=0 ){
         5191  +    SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n"));
         5192  +    sqlite3TreeViewSelect(0, p, 0);
         5193  +  }
         5194  +#endif
  4984   5195   }
  4985   5196   
  4986   5197   /*
  4987   5198   ** Check to see if the pThis entry of pTabList is a self-join of a prior view.
  4988   5199   ** If it is, then return the SrcList_item for the prior view.  If it is not,
  4989   5200   ** then return 0.
  4990   5201   */
................................................................................
  5137   5348     db = pParse->db;
  5138   5349     if( p==0 || db->mallocFailed || pParse->nErr ){
  5139   5350       return 1;
  5140   5351     }
  5141   5352     if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1;
  5142   5353     memset(&sAggInfo, 0, sizeof(sAggInfo));
  5143   5354   #if SELECTTRACE_ENABLED
  5144         -  SELECTTRACE(1,pParse,p, ("begin processing:\n"));
         5355  +#ifndef SQLITE_OMIT_EXPLAIN
         5356  +  p->iSelectId = pParse->iSelectId;
         5357  +#endif
         5358  +  SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->iSelectId));
  5145   5359     if( sqlite3SelectTrace & 0x100 ){
  5146   5360       sqlite3TreeViewSelect(0, p, 0);
  5147   5361     }
  5148   5362   #endif
  5149   5363   
  5150   5364     assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo );
  5151   5365     assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo );
................................................................................
  5168   5382     pTabList = p->pSrc;
  5169   5383     if( pParse->nErr || db->mallocFailed ){
  5170   5384       goto select_end;
  5171   5385     }
  5172   5386     assert( p->pEList!=0 );
  5173   5387     isAgg = (p->selFlags & SF_Aggregate)!=0;
  5174   5388   #if SELECTTRACE_ENABLED
  5175         -  if( sqlite3SelectTrace & 0x100 ){
  5176         -    SELECTTRACE(0x100,pParse,p, ("after name resolution:\n"));
         5389  +  if( sqlite3SelectTrace & 0x104 ){
         5390  +    SELECTTRACE(0x104,pParse,p, ("after name resolution:\n"));
  5177   5391       sqlite3TreeViewSelect(0, p, 0);
  5178   5392     }
  5179   5393   #endif
  5180   5394   
  5181   5395     /* Get a pointer the VDBE under construction, allocating a new VDBE if one
  5182   5396     ** does not already exist */
  5183   5397     v = sqlite3GetVdbe(pParse);
  5184   5398     if( v==0 ) goto select_end;
  5185   5399     if( pDest->eDest==SRT_Output ){
  5186   5400       generateColumnNames(pParse, p);
  5187   5401     }
  5188   5402   
  5189         -  /* Try to flatten subqueries in the FROM clause up into the main query
         5403  +  /* Try to various optimizations (flattening subqueries, and strength
         5404  +  ** reduction of join operators) in the FROM clause up into the main query
  5190   5405     */
  5191   5406   #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
  5192   5407     for(i=0; !p->pPrior && i<pTabList->nSrc; i++){
  5193   5408       struct SrcList_item *pItem = &pTabList->a[i];
  5194   5409       Select *pSub = pItem->pSelect;
  5195   5410       Table *pTab = pItem->pTab;
         5411  +
         5412  +    /* Convert LEFT JOIN into JOIN if there are terms of the right table
         5413  +    ** of the LEFT JOIN used in the WHERE clause.
         5414  +    */
         5415  +    if( (pItem->fg.jointype & JT_LEFT)!=0
         5416  +     && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor)
         5417  +     && OptimizationEnabled(db, SQLITE_SimplifyJoin)
         5418  +    ){
         5419  +      SELECTTRACE(0x100,pParse,p,
         5420  +                ("LEFT-JOIN simplifies to JOIN on term %d\n",i));
         5421  +      pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER);
         5422  +      unsetJoinExpr(p->pWhere, pItem->iCursor);
         5423  +    }
         5424  +
         5425  +    /* No futher action if this term of the FROM clause is no a subquery */
  5196   5426       if( pSub==0 ) continue;
  5197   5427   
  5198   5428       /* Catch mismatch in the declared columns of a view and the number of
  5199   5429       ** columns in the SELECT on the RHS */
  5200   5430       if( pTab->nCol!=pSub->pEList->nExpr ){
  5201   5431         sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d",
  5202   5432                         pTab->nCol, pTab->zName, pSub->pEList->nExpr);
................................................................................
  5254   5484   
  5255   5485   #ifndef SQLITE_OMIT_COMPOUND_SELECT
  5256   5486     /* Handle compound SELECT statements using the separate multiSelect()
  5257   5487     ** procedure.
  5258   5488     */
  5259   5489     if( p->pPrior ){
  5260   5490       rc = multiSelect(pParse, p, pDest);
  5261         -    explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  5262   5491   #if SELECTTRACE_ENABLED
  5263         -    SELECTTRACE(1,pParse,p,("end compound-select processing\n"));
         5492  +    SELECTTRACE(0x1,pParse,p,("end compound-select processing\n"));
         5493  +    if( pParse->iSelectId==0 && (sqlite3SelectTrace & 0x2000)!=0 ){
         5494  +      sqlite3TreeViewSelect(0, p, 0);
         5495  +    }
  5264   5496   #endif
         5497  +    explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  5265   5498       return rc;
  5266   5499     }
  5267   5500   #endif
  5268   5501   
  5269   5502     /* For each term in the FROM clause, do two things:
  5270   5503     ** (1) Authorized unreferenced tables
  5271   5504     ** (2) Generate code for all sub-queries
................................................................................
  5329   5562       ** an exact limit.
  5330   5563       */
  5331   5564       pParse->nHeight += sqlite3SelectExprHeight(p);
  5332   5565   
  5333   5566       /* Make copies of constant WHERE-clause terms in the outer query down
  5334   5567       ** inside the subquery.  This can help the subquery to run more efficiently.
  5335   5568       */
  5336         -    if( (pItem->fg.jointype & JT_OUTER)==0
  5337         -     && OptimizationEnabled(db, SQLITE_PushDown)
  5338         -     && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor)
         5569  +    if( OptimizationEnabled(db, SQLITE_PushDown)
         5570  +     && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor,
         5571  +                           (pItem->fg.jointype & JT_OUTER)!=0)
  5339   5572       ){
  5340   5573   #if SELECTTRACE_ENABLED
  5341   5574         if( sqlite3SelectTrace & 0x100 ){
  5342   5575           SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n"));
  5343   5576           sqlite3TreeViewSelect(0, p, 0);
  5344   5577         }
  5345   5578   #endif
................................................................................
  5636   5869       /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in
  5637   5870       ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the
  5638   5871       ** SELECT statement.
  5639   5872       */
  5640   5873       memset(&sNC, 0, sizeof(sNC));
  5641   5874       sNC.pParse = pParse;
  5642   5875       sNC.pSrcList = pTabList;
  5643         -    sNC.pAggInfo = &sAggInfo;
         5876  +    sNC.uNC.pAggInfo = &sAggInfo;
         5877  +    VVA_ONLY( sNC.ncFlags = NC_UAggInfo; )
  5644   5878       sAggInfo.mnReg = pParse->nMem+1;
  5645   5879       sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
  5646   5880       sAggInfo.pGroupBy = pGroupBy;
  5647   5881       sqlite3ExprAnalyzeAggList(&sNC, pEList);
  5648   5882       sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy);
  5649   5883       if( pHaving ){
  5650   5884         if( pGroupBy ){
  5651   5885           assert( pWhere==p->pWhere );
  5652         -        havingToWhere(pParse, pGroupBy, pHaving, &p->pWhere);
         5886  +        assert( pHaving==p->pHaving );
         5887  +        assert( pGroupBy==p->pGroupBy );
         5888  +        havingToWhere(pParse, p);
  5653   5889           pWhere = p->pWhere;
  5654   5890         }
  5655   5891         sqlite3ExprAnalyzeAggregates(&sNC, pHaving);
  5656   5892       }
  5657   5893       sAggInfo.nAccumulator = sAggInfo.nColumn;
  5658   5894       if( p->pGroupBy==0 && p->pHaving==0 && sAggInfo.nFunc==1 ){
  5659   5895         minMaxFlag = minMaxQuery(db, sAggInfo.aFunc[0].pExpr, &pMinMaxOrderBy);
................................................................................
  6023   6259   
  6024   6260     /* If there is an ORDER BY clause, then we need to sort the results
  6025   6261     ** and send them to the callback one by one.
  6026   6262     */
  6027   6263     if( sSort.pOrderBy ){
  6028   6264       explainTempTable(pParse,
  6029   6265                        sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY");
         6266  +    assert( p->pEList==pEList );
  6030   6267       generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest);
  6031   6268     }
  6032   6269   
  6033   6270     /* Jump here to skip this query
  6034   6271     */
  6035   6272     sqlite3VdbeResolveLabel(v, iEnd);
  6036   6273   
................................................................................
  6038   6275     ** set the return code to 1. Otherwise 0. */
  6039   6276     rc = (pParse->nErr>0);
  6040   6277   
  6041   6278     /* Control jumps to here if an error is encountered above, or upon
  6042   6279     ** successful coding of the SELECT.
  6043   6280     */
  6044   6281   select_end:
  6045         -  explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  6046   6282     sqlite3ExprListDelete(db, pMinMaxOrderBy);
  6047   6283     sqlite3DbFree(db, sAggInfo.aCol);
  6048   6284     sqlite3DbFree(db, sAggInfo.aFunc);
  6049   6285   #if SELECTTRACE_ENABLED
  6050         -  SELECTTRACE(1,pParse,p,("end processing\n"));
         6286  +  SELECTTRACE(0x1,pParse,p,("end processing\n"));
         6287  +  if( pParse->iSelectId==0 && (sqlite3SelectTrace & 0x2000)!=0 ){
         6288  +    sqlite3TreeViewSelect(0, p, 0);
         6289  +  }
  6051   6290   #endif
         6291  +  explainSetInteger(pParse->iSelectId, iRestoreSelectId);
  6052   6292     return rc;
  6053   6293   }

Changes to src/shell.c.in.

   128    128   #if defined(_WIN32) || defined(WIN32)
   129    129   # include <io.h>
   130    130   # include <fcntl.h>
   131    131   # define isatty(h) _isatty(h)
   132    132   # ifndef access
   133    133   #  define access(f,m) _access((f),(m))
   134    134   # endif
          135  +# ifndef unlink
          136  +#  define unlink _unlink
          137  +# endif
   135    138   # undef popen
   136    139   # define popen _popen
   137    140   # undef pclose
   138    141   # define pclose _pclose
   139    142   #else
   140    143    /* Make sure isatty() has a prototype. */
   141    144    extern int isatty(int);
................................................................................
   428    431   /*
   429    432   ** Render output like fprintf().  This should not be used on anything that
   430    433   ** includes string formatting (e.g. "%s").
   431    434   */
   432    435   #if !defined(raw_printf)
   433    436   # define raw_printf fprintf
   434    437   #endif
          438  +
          439  +/* Indicate out-of-memory and exit. */
          440  +static void shell_out_of_memory(void){
          441  +  raw_printf(stderr,"Error: out of memory\n");
          442  +  exit(1);
          443  +}
   435    444   
   436    445   /*
   437    446   ** Write I/O traces to the following stream.
   438    447   */
   439    448   #ifdef SQLITE_ENABLE_IOTRACE
   440    449   static FILE *iotrace = 0;
   441    450   #endif
................................................................................
   998   1007   
   999   1008   typedef struct ExpertInfo ExpertInfo;
  1000   1009   struct ExpertInfo {
  1001   1010     sqlite3expert *pExpert;
  1002   1011     int bVerbose;
  1003   1012   };
  1004   1013   
         1014  +/* A single line in the EQP output */
         1015  +typedef struct EQPGraphRow EQPGraphRow;
         1016  +struct EQPGraphRow {
         1017  +  int iSelectId;        /* The SelectID for this row */
         1018  +  EQPGraphRow *pNext;   /* Next row in sequence */
         1019  +  char zText[1];        /* Text to display for this row */
         1020  +};
         1021  +
         1022  +/* All EQP output is collected into an instance of the following */
         1023  +typedef struct EQPGraph EQPGraph;
         1024  +struct EQPGraph {
         1025  +  EQPGraphRow *pRow;    /* Linked list of all rows of the EQP output */
         1026  +  EQPGraphRow *pLast;   /* Last element of the pRow list */
         1027  +  char zPrefix[100];    /* Graph prefix */
         1028  +};
         1029  +
  1005   1030   /*
  1006   1031   ** State information about the database connection is contained in an
  1007   1032   ** instance of the following structure.
  1008   1033   */
  1009   1034   typedef struct ShellState ShellState;
  1010   1035   struct ShellState {
  1011   1036     sqlite3 *db;           /* The database */
  1012   1037     u8 autoExplain;        /* Automatically turn on .explain mode */
  1013   1038     u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  1014   1039     u8 statsOn;            /* True to display memory stats before each finalize */
  1015   1040     u8 scanstatsOn;        /* True to display scan stats before each finalize */
  1016   1041     u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  1017   1042     u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
         1043  +  u8 nEqpLevel;          /* Depth of the EQP output graph */
         1044  +  unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  1018   1045     int outCount;          /* Revert to stdout when reaching zero */
  1019   1046     int cnt;               /* Number of records displayed so far */
  1020   1047     FILE *out;             /* Write results here */
  1021   1048     FILE *traceOut;        /* Output for sqlite3_trace() */
  1022   1049     int nErr;              /* Number of errors seen */
  1023   1050     int mode;              /* An output mode setting */
  1024   1051     int modePrior;         /* Saved mode */
................................................................................
  1044   1071     char *zFreeOnClose;         /* Filename to free when closing */
  1045   1072     const char *zVfs;           /* Name of VFS to use */
  1046   1073     sqlite3_stmt *pStmt;   /* Current statement if any. */
  1047   1074     FILE *pLog;            /* Write log output here */
  1048   1075     int *aiIndent;         /* Array of indents used in MODE_Explain */
  1049   1076     int nIndent;           /* Size of array aiIndent[] */
  1050   1077     int iIndent;           /* Index of current op in aiIndent[] */
         1078  +  EQPGraph sGraph;       /* Information for the graphical EXPLAIN QUERY PLAN */
  1051   1079   #if defined(SQLITE_ENABLE_SESSION)
  1052   1080     int nSession;             /* Number of active sessions */
  1053   1081     OpenSession aSession[4];  /* Array of sessions.  [0] is in focus. */
  1054   1082   #endif
  1055   1083     ExpertInfo expert;        /* Valid if previous command was ".expert OPT..." */
  1056   1084   };
  1057   1085   
................................................................................
  1100   1128   #define MODE_Insert   5  /* Generate SQL "insert" statements */
  1101   1129   #define MODE_Quote    6  /* Quote values as for SQL */
  1102   1130   #define MODE_Tcl      7  /* Generate ANSI-C or TCL quoted elements */
  1103   1131   #define MODE_Csv      8  /* Quote strings, numbers are plain */
  1104   1132   #define MODE_Explain  9  /* Like MODE_Column, but do not truncate data */
  1105   1133   #define MODE_Ascii   10  /* Use ASCII unit and record separators (0x1F/0x1E) */
  1106   1134   #define MODE_Pretty  11  /* Pretty-print schemas */
         1135  +#define MODE_EQP     12  /* Converts EXPLAIN QUERY PLAN output into a graph */
  1107   1136   
  1108   1137   static const char *modeDescr[] = {
  1109   1138     "line",
  1110   1139     "column",
  1111   1140     "list",
  1112   1141     "semi",
  1113   1142     "html",
................................................................................
  1114   1143     "insert",
  1115   1144     "quote",
  1116   1145     "tcl",
  1117   1146     "csv",
  1118   1147     "explain",
  1119   1148     "ascii",
  1120   1149     "prettyprint",
         1150  +  "eqp"
  1121   1151   };
  1122   1152   
  1123   1153   /*
  1124   1154   ** These are the column/row/line separators used by the various
  1125   1155   ** import/export modes.
  1126   1156   */
  1127   1157   #define SEP_Column    "|"
................................................................................
  1662   1692       if( z[i]=='\n' ) return 1;
  1663   1693       if( IsSpace(z[i]) ) continue;
  1664   1694       if( z[i]=='-' && z[i+1]=='-' ) return 1;
  1665   1695       return 0;
  1666   1696     }
  1667   1697     return 1;
  1668   1698   }
  1669         -    
         1699  +
         1700  +/*
         1701  +** Add a new entry to the EXPLAIN QUERY PLAN data
         1702  +*/
         1703  +static void eqp_append(ShellState *p, int iSelectId, const char *zText){
         1704  +  EQPGraphRow *pNew;
         1705  +  int nText = strlen30(zText);
         1706  +  pNew = sqlite3_malloc64( sizeof(*pNew) + nText );
         1707  +  if( pNew==0 ) shell_out_of_memory();
         1708  +  pNew->iSelectId = iSelectId;
         1709  +  memcpy(pNew->zText, zText, nText+1);
         1710  +  pNew->pNext = 0;
         1711  +  if( p->sGraph.pLast ){
         1712  +    p->sGraph.pLast->pNext = pNew;
         1713  +  }else{
         1714  +    p->sGraph.pRow = pNew;
         1715  +  }
         1716  +  p->sGraph.pLast = pNew;
         1717  +}
         1718  +
         1719  +/*
         1720  +** Free and reset the EXPLAIN QUERY PLAN data that has been collected
         1721  +** in p->sGraph.
         1722  +*/
         1723  +static void eqp_reset(ShellState *p){
         1724  +  EQPGraphRow *pRow, *pNext;
         1725  +  for(pRow = p->sGraph.pRow; pRow; pRow = pNext){
         1726  +    pNext = pRow->pNext;
         1727  +    sqlite3_free(pRow);
         1728  +  }
         1729  +  memset(&p->sGraph, 0, sizeof(p->sGraph));
         1730  +}
         1731  +
         1732  +/* Return the next EXPLAIN QUERY PLAN line with iSelectId that occurs after
         1733  +** pOld, or return the first such line if pOld is NULL
         1734  +*/
         1735  +static EQPGraphRow *eqp_next_row(ShellState *p, int iSelectId, EQPGraphRow *pOld){
         1736  +  EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow;
         1737  +  while( pRow && pRow->iSelectId!=iSelectId ) pRow = pRow->pNext;
         1738  +  return pRow;
         1739  +}
         1740  +
         1741  +/* Render a single level of the graph shell having iSelectId.  Called
         1742  +** recursively to render sublevels.
         1743  +*/
         1744  +static void eqp_render_level(ShellState *p, int iSelectId){
         1745  +  EQPGraphRow *pRow, *pNext;
         1746  +  int i;
         1747  +  int n = strlen30(p->sGraph.zPrefix);
         1748  +  char *z;
         1749  +  for(pRow = eqp_next_row(p, iSelectId, 0); pRow; pRow = pNext){
         1750  +    pNext = eqp_next_row(p, iSelectId, pRow);
         1751  +    z = pRow->zText;
         1752  +    utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z);
         1753  +    if( n<sizeof(p->sGraph.zPrefix)-7 && (z = strstr(z, " SUBQUER"))!=0 ){
         1754  +      memcpy(&p->sGraph.zPrefix[n], pNext ? "|  " : "   ", 4);
         1755  +      if( strncmp(z, " SUBQUERY ", 9)==0 && (i = atoi(z+10))>iSelectId ){
         1756  +        eqp_render_level(p, i);
         1757  +      }else if( strncmp(z, " SUBQUERIES ", 12)==0 ){
         1758  +        i = atoi(z+12);
         1759  +        if( i>iSelectId ){
         1760  +          utf8_printf(p->out, "%s|--SUBQUERY %d\n", p->sGraph.zPrefix, i);
         1761  +          memcpy(&p->sGraph.zPrefix[n+3],"|  ",4);
         1762  +          eqp_render_level(p, i);
         1763  +        }
         1764  +        z = strstr(z, " AND ");
         1765  +        if( z && (i = atoi(z+5))>iSelectId ){
         1766  +          p->sGraph.zPrefix[n+3] = 0;
         1767  +          utf8_printf(p->out, "%s`--SUBQUERY %d\n", p->sGraph.zPrefix, i);
         1768  +          memcpy(&p->sGraph.zPrefix[n+3],"   ",4);
         1769  +          eqp_render_level(p, i);
         1770  +        }
         1771  +      }
         1772  +      p->sGraph.zPrefix[n] = 0;
         1773  +    }
         1774  +  }
         1775  +}
         1776  +
         1777  +/*
         1778  +** Display and reset the EXPLAIN QUERY PLAN data
         1779  +*/
         1780  +static void eqp_render(ShellState *p){
         1781  +  EQPGraphRow *pRow = p->sGraph.pRow;
         1782  +  if( pRow ){
         1783  +    if( pRow->zText[0]=='-' ){
         1784  +      if( pRow->pNext==0 ){
         1785  +        eqp_reset(p);
         1786  +        return;
         1787  +      }
         1788  +      utf8_printf(p->out, "%s\n", pRow->zText+3);
         1789  +      p->sGraph.pRow = pRow->pNext;
         1790  +      sqlite3_free(pRow);
         1791  +    }else{
         1792  +      utf8_printf(p->out, "QUERY PLAN\n");
         1793  +    }
         1794  +    p->sGraph.zPrefix[0] = 0;
         1795  +    eqp_render_level(p, 0);
         1796  +    eqp_reset(p);
         1797  +  }
         1798  +}
  1670   1799   
  1671   1800   /*
  1672   1801   ** This is the callback routine that the shell
  1673   1802   ** invokes for each row of a query result.
  1674   1803   */
  1675   1804   static int shell_callback(
  1676   1805     void *pArg,
................................................................................
  2013   2142         for(i=0; i<nArg; i++){
  2014   2143           if( i>0 ) utf8_printf(p->out, "%s", p->colSeparator);
  2015   2144           utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue);
  2016   2145         }
  2017   2146         utf8_printf(p->out, "%s", p->rowSeparator);
  2018   2147         break;
  2019   2148       }
         2149  +    case MODE_EQP: {
         2150  +      eqp_append(p, atoi(azArg[0]), azArg[3]);
         2151  +      break;
         2152  +    }
  2020   2153     }
  2021   2154     return 0;
  2022   2155   }
  2023   2156   
  2024   2157   /*
  2025   2158   ** This is the callback routine that the SQLite library
  2026   2159   ** invokes for each row of a query result.
................................................................................
  2111   2244       p->zDestTable = 0;
  2112   2245     }
  2113   2246     if( zName==0 ) return;
  2114   2247     cQuote = quoteChar(zName);
  2115   2248     n = strlen30(zName);
  2116   2249     if( cQuote ) n += n+2;
  2117   2250     z = p->zDestTable = malloc( n+1 );
  2118         -  if( z==0 ){
  2119         -    raw_printf(stderr,"Error: out of memory\n");
  2120         -    exit(1);
  2121         -  }
         2251  +  if( z==0 ) shell_out_of_memory();
  2122   2252     n = 0;
  2123   2253     if( cQuote ) z[n++] = cQuote;
  2124   2254     for(i=0; zName[i]; i++){
  2125   2255       z[n++] = zName[i];
  2126   2256       if( zName[i]==cQuote ) z[n++] = cQuote;
  2127   2257     }
  2128   2258     if( cQuote ) z[n++] = cQuote;
................................................................................
  2854   2984           if( pArg->autoEQP>=AUTOEQP_trigger ){
  2855   2985             sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
  2856   2986           }
  2857   2987           zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
  2858   2988           rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
  2859   2989           if( rc==SQLITE_OK ){
  2860   2990             while( sqlite3_step(pExplain)==SQLITE_ROW ){
  2861         -            raw_printf(pArg->out,"--EQP-- %d,",sqlite3_column_int(pExplain, 0));
  2862         -            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 1));
  2863         -            raw_printf(pArg->out,"%d,", sqlite3_column_int(pExplain, 2));
  2864         -            utf8_printf(pArg->out,"%s\n", sqlite3_column_text(pExplain, 3));
         2991  +            const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
         2992  +            int iSelectId = sqlite3_column_int(pExplain, 0);
         2993  +            if( zEQPLine[0]=='-' ) eqp_render(pArg);
         2994  +            eqp_append(pArg, iSelectId, zEQPLine);
  2865   2995             }
         2996  +          eqp_render(pArg);
  2866   2997           }
  2867   2998           sqlite3_finalize(pExplain);
  2868   2999           sqlite3_free(zEQP);
  2869   3000           if( pArg->autoEQP>=AUTOEQP_full ){
  2870   3001             /* Also do an EXPLAIN for ".eqp full" mode */
  2871   3002             zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
  2872   3003             rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
................................................................................
  2886   3017             sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  2887   3018           }
  2888   3019           restore_debug_trace_modes();
  2889   3020         }
  2890   3021   
  2891   3022         if( pArg ){
  2892   3023           pArg->cMode = pArg->mode;
  2893         -        if( pArg->autoExplain
  2894         -         && sqlite3_column_count(pStmt)==8
  2895         -         && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0
  2896         -        ){
  2897         -          pArg->cMode = MODE_Explain;
         3024  +        if( pArg->autoExplain ){
         3025  +          if( sqlite3_column_count(pStmt)==8
         3026  +           && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0
         3027  +          ){
         3028  +            pArg->cMode = MODE_Explain;
         3029  +          }
         3030  +          if( sqlite3_column_count(pStmt)==4
         3031  +           && sqlite3_strlike("EXPLAIN QUERY PLAN%", zStmtSql,0)==0 ){
         3032  +            pArg->cMode = MODE_EQP;
         3033  +          }
  2898   3034           }
  2899   3035   
  2900   3036           /* If the shell is currently in ".explain" mode, gather the extra
  2901   3037           ** data required to add indents to the output.*/
  2902   3038           if( pArg->cMode==MODE_Explain ){
  2903   3039             explain_data_prepare(pArg, pStmt);
  2904   3040           }
  2905   3041         }
  2906   3042   
  2907   3043         exec_prepared_stmt(pArg, pStmt);
  2908   3044         explain_data_delete(pArg);
         3045  +      eqp_render(pArg);
  2909   3046   
  2910   3047         /* print usage stats if stats on */
  2911   3048         if( pArg && pArg->statsOn ){
  2912   3049           display_stats(db, pArg, 0);
  2913   3050         }
  2914   3051   
  2915   3052         /* print loop-counters if required */
................................................................................
  2979   3116     rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  2980   3117     sqlite3_free(zSql);
  2981   3118     if( rc ) return 0;
  2982   3119     while( sqlite3_step(pStmt)==SQLITE_ROW ){
  2983   3120       if( nCol>=nAlloc-2 ){
  2984   3121         nAlloc = nAlloc*2 + nCol + 10;
  2985   3122         azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
  2986         -      if( azCol==0 ){
  2987         -        raw_printf(stderr, "Error: out of memory\n");
  2988         -        exit(1);
  2989         -      }
         3123  +      if( azCol==0 ) shell_out_of_memory();
  2990   3124       }
  2991   3125       azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
  2992   3126       if( sqlite3_column_int(pStmt, 5) ){
  2993   3127         nPK++;
  2994   3128         if( nPK==1
  2995   3129          && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2),
  2996   3130                             "INTEGER")==0
................................................................................
  3477   3611   
  3478   3612   /*
  3479   3613   ** Make sure the database is open.  If it is not, then open it.  If
  3480   3614   ** the database fails to open, print an error message and exit.
  3481   3615   */
  3482   3616   static void open_db(ShellState *p, int keepAlive){
  3483   3617     if( p->db==0 ){
  3484         -    sqlite3_initialize();
  3485   3618       if( p->openMode==SHELL_OPEN_UNSPEC && access(p->zDbFilename,0)==0 ){
  3486   3619         p->openMode = (u8)deduceDatabaseType(p->zDbFilename, 0);
  3487   3620       }
  3488   3621       switch( p->openMode ){
  3489   3622         case SHELL_OPEN_APPENDVFS: {
  3490   3623           sqlite3_open_v2(p->zDbFilename, &p->db, 
  3491   3624              SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
................................................................................
  3726   3859       if( f==0 ){
  3727   3860         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
  3728   3861       }
  3729   3862     }
  3730   3863     return f;
  3731   3864   }
  3732   3865   
  3733         -#if !defined(SQLITE_UNTESTABLE)
  3734   3866   #if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
  3735   3867   /*
  3736   3868   ** A routine for handling output from sqlite3_trace().
  3737   3869   */
  3738   3870   static int sql_trace_callback(
  3739   3871     unsigned mType,
  3740   3872     void *pArg,
................................................................................
  3748   3880       const char *z = (const char*)pX;
  3749   3881       int i = strlen30(z);
  3750   3882       while( i>0 && z[i-1]==';' ){ i--; }
  3751   3883       utf8_printf(f, "%.*s;\n", i, z);
  3752   3884     }
  3753   3885     return 0;
  3754   3886   }
  3755         -#endif
  3756   3887   #endif
  3757   3888   
  3758   3889   /*
  3759   3890   ** A no-op routine that runs with the ".breakpoint" doc-command.  This is
  3760   3891   ** a useful spot to set a debugger breakpoint.
  3761   3892   */
  3762   3893   static void test_breakpoint(void){
................................................................................
  3782   3913   };
  3783   3914   
  3784   3915   /* Append a single byte to z[] */
  3785   3916   static void import_append_char(ImportCtx *p, int c){
  3786   3917     if( p->n+1>=p->nAlloc ){
  3787   3918       p->nAlloc += p->nAlloc + 100;
  3788   3919       p->z = sqlite3_realloc64(p->z, p->nAlloc);
  3789         -    if( p->z==0 ){
  3790         -      raw_printf(stderr, "out of memory\n");
  3791         -      exit(1);
  3792         -    }
         3920  +    if( p->z==0 ) shell_out_of_memory();
  3793   3921     }
  3794   3922     p->z[p->n++] = (char)c;
  3795   3923   }
  3796   3924   
  3797   3925   /* Read a single field of CSV text.  Compatible with rfc4180 and extended
  3798   3926   ** with the option of having a separator other than ",".
  3799   3927   **
................................................................................
  3946   4074       utf8_printf(stderr, "Error %d: %s on [%s]\n",
  3947   4075               sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
  3948   4076               zQuery);
  3949   4077       goto end_data_xfer;
  3950   4078     }
  3951   4079     n = sqlite3_column_count(pQuery);
  3952   4080     zInsert = sqlite3_malloc64(200 + nTable + n*3);
  3953         -  if( zInsert==0 ){
  3954         -    raw_printf(stderr, "out of memory\n");
  3955         -    goto end_data_xfer;
  3956         -  }
         4081  +  if( zInsert==0 ) shell_out_of_memory();
  3957   4082     sqlite3_snprintf(200+nTable,zInsert,
  3958   4083                      "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable);
  3959   4084     i = strlen30(zInsert);
  3960   4085     for(j=1; j<n; j++){
  3961   4086       memcpy(zInsert+i, ",?", 2);
  3962   4087       i += 2;
  3963   4088     }
................................................................................
  4290   4415   */
  4291   4416   static int shellDatabaseError(sqlite3 *db){
  4292   4417     const char *zErr = sqlite3_errmsg(db);
  4293   4418     utf8_printf(stderr, "Error: %s\n", zErr);
  4294   4419     return 1;
  4295   4420   }
  4296   4421   
  4297         -/*
  4298         -** Print an out-of-memory message to stderr and return 1.
  4299         -*/
  4300         -static int shellNomemError(void){
  4301         -  raw_printf(stderr, "Error: out of memory\n");
  4302         -  return 1;
  4303         -}
  4304         -
  4305   4422   /*
  4306   4423   ** Compare the pattern in zGlob[] against the text in z[].  Return TRUE
  4307   4424   ** if they match and FALSE (0) if they do not match.
  4308   4425   **
  4309   4426   ** Globbing rules:
  4310   4427   **
  4311   4428   **      '*'       Matches any sequence of zero or more characters.
................................................................................
  5332   5449       if( bUpdate==0 ){
  5333   5450         rc = arExecSql(pAr, zDrop);
  5334   5451         if( rc!=SQLITE_OK ) goto end_ar_transaction;
  5335   5452       }
  5336   5453       rc = arExecSql(pAr, zCreate);
  5337   5454     }
  5338   5455     for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
  5339         -    char *zSql = sqlite3_mprintf(zInsertFmt[pAr->bZip], zTab,
         5456  +    char *zSql2 = sqlite3_mprintf(zInsertFmt[pAr->bZip], zTab,
  5340   5457           pAr->bVerbose ? "shell_putsnl(name)" : "name",
  5341   5458           pAr->azArg[i], pAr->zDir);
  5342         -    rc = arExecSql(pAr, zSql);
  5343         -    sqlite3_free(zSql);
         5459  +    rc = arExecSql(pAr, zSql2);
         5460  +    sqlite3_free(zSql2);
  5344   5461     }
  5345   5462   end_ar_transaction:
  5346   5463     if( rc!=SQLITE_OK ){
  5347   5464       arExecSql(pAr, "ROLLBACK TO ar; RELEASE ar;");
  5348   5465     }else{
  5349   5466       rc = arExecSql(pAr, "RELEASE ar;");
  5350   5467       if( pAr->bZip && pAr->zFile ){
................................................................................
  5991   6108         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
  5992   6109         return 1;
  5993   6110       }
  5994   6111       sCtx.cColSep = p->colSeparator[0];
  5995   6112       sCtx.cRowSep = p->rowSeparator[0];
  5996   6113       zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
  5997   6114       if( zSql==0 ){
  5998         -      raw_printf(stderr, "Error: out of memory\n");
  5999   6115         xCloser(sCtx.in);
  6000         -      return 1;
         6116  +      shell_out_of_memory();
  6001   6117       }
  6002   6118       nByte = strlen30(zSql);
  6003   6119       rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  6004   6120       import_append_char(&sCtx, 0);    /* To ensure sCtx.z is allocated */
  6005   6121       if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){
  6006   6122         char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable);
  6007   6123         char cSep = '(';
................................................................................
  6038   6154       }
  6039   6155       nCol = sqlite3_column_count(pStmt);
  6040   6156       sqlite3_finalize(pStmt);
  6041   6157       pStmt = 0;
  6042   6158       if( nCol==0 ) return 0; /* no columns, no error */
  6043   6159       zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 );
  6044   6160       if( zSql==0 ){
  6045         -      raw_printf(stderr, "Error: out of memory\n");
  6046   6161         xCloser(sCtx.in);
  6047         -      return 1;
         6162  +      shell_out_of_memory();
  6048   6163       }
  6049   6164       sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
  6050   6165       j = strlen30(zSql);
  6051   6166       for(i=1; i<nCol; i++){
  6052   6167         zSql[j++] = ',';
  6053   6168         zSql[j++] = '?';
  6054   6169       }
................................................................................
  6116   6231   #ifndef SQLITE_UNTESTABLE
  6117   6232     if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){
  6118   6233       char *zSql;
  6119   6234       char *zCollist = 0;
  6120   6235       sqlite3_stmt *pStmt;
  6121   6236       int tnum = 0;
  6122   6237       int i;
  6123         -    if( nArg!=3 ){
  6124         -      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n");
         6238  +    if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){
         6239  +      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n"
         6240  +                          "       .imposter off\n");
  6125   6241         rc = 1;
  6126   6242         goto meta_command_exit;
  6127   6243       }
  6128   6244       open_db(p, 0);
         6245  +    if( nArg==2 ){
         6246  +      sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 1);
         6247  +      goto meta_command_exit;
         6248  +    }
  6129   6249       zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_master"
  6130   6250                              " WHERE name='%q' AND type='index'", azArg[1]);
  6131   6251       sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  6132   6252       sqlite3_free(zSql);
  6133   6253       if( sqlite3_step(pStmt)==SQLITE_ROW ){
  6134   6254         tnum = sqlite3_column_int(pStmt, 0);
  6135   6255       }
................................................................................
  6614   6734         }else{
  6615   6735           raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n");
  6616   6736           rc = 1;
  6617   6737           goto meta_command_exit;
  6618   6738         }
  6619   6739       }
  6620   6740       if( zName!=0 ){
  6621         -      int isMaster = sqlite3_strlike(zName, "sqlite_master", 0)==0;
  6622         -      if( isMaster || sqlite3_strlike(zName,"sqlite_temp_master",0)==0 ){
         6741  +      int isMaster = sqlite3_strlike(zName, "sqlite_master", '\\')==0;
         6742  +      if( isMaster || sqlite3_strlike(zName,"sqlite_temp_master", '\\')==0 ){
  6623   6743           char *new_argv[2], *new_colv[2];
  6624   6744           new_argv[0] = sqlite3_mprintf(
  6625   6745                         "CREATE TABLE %s (\n"
  6626   6746                         "  type text,\n"
  6627   6747                         "  name text,\n"
  6628   6748                         "  tbl_name text,\n"
  6629   6749                         "  rootpage integer,\n"
................................................................................
  6675   6795              " UNION ALL SELECT shell_module_schema(name),"
  6676   6796              " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", 0);
  6677   6797         }
  6678   6798   #endif
  6679   6799         appendText(&sSelect, ") WHERE ", 0);
  6680   6800         if( zName ){
  6681   6801           char *zQarg = sqlite3_mprintf("%Q", zName);
         6802  +        int bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 ||
         6803  +                    strchr(zName, '[') != 0;
  6682   6804           if( strchr(zName, '.') ){
  6683   6805             appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0);
  6684   6806           }else{
  6685   6807             appendText(&sSelect, "lower(tbl_name)", 0);
  6686   6808           }
  6687         -        appendText(&sSelect, strchr(zName, '*') ? " GLOB " : " LIKE ", 0);
         6809  +        appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0);
  6688   6810           appendText(&sSelect, zQarg, 0);
         6811  +        if( !bGlob ){
         6812  +          appendText(&sSelect, " ESCAPE '\\' ", 0);
         6813  +        }
  6689   6814           appendText(&sSelect, " AND ", 0);
  6690   6815           sqlite3_free(zQarg);
  6691   6816         }
  6692   6817         appendText(&sSelect, "type!='meta' AND sql IS NOT NULL"
  6693   6818                              " ORDER BY snum, rowid", 0);
  6694   6819         if( bDebug ){
  6695   6820           utf8_printf(p->out, "SQL: %s;\n", sSelect.z);
................................................................................
  7096   7221         }else if( zLike ){
  7097   7222           raw_printf(stderr, "Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n");
  7098   7223           rc = 1;
  7099   7224           goto meta_command_exit;
  7100   7225         }else{
  7101   7226           zLike = z;
  7102   7227           bSeparate = 1;
  7103         -        if( sqlite3_strlike("sqlite_%", zLike, 0)==0 ) bSchema = 1;
         7228  +        if( sqlite3_strlike("sqlite\\_%", zLike, '\\')==0 ) bSchema = 1;
  7104   7229         }
  7105   7230       }
  7106   7231       if( bSchema ){
  7107   7232         zSql = "SELECT lower(name) FROM sqlite_master"
  7108   7233                " WHERE type='table' AND coalesce(rootpage,0)>1"
  7109   7234                " UNION ALL SELECT 'sqlite_master'"
  7110   7235                " ORDER BY 1 collate nocase";
................................................................................
  7301   7426         sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC);
  7302   7427       }
  7303   7428       while( sqlite3_step(pStmt)==SQLITE_ROW ){
  7304   7429         if( nRow>=nAlloc ){
  7305   7430           char **azNew;
  7306   7431           int n2 = nAlloc*2 + 10;
  7307   7432           azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2);
  7308         -        if( azNew==0 ){
  7309         -          rc = shellNomemError();
  7310         -          break;
  7311         -        }
         7433  +        if( azNew==0 ) shell_out_of_memory();
  7312   7434           nAlloc = n2;
  7313   7435           azResult = azNew;
  7314   7436         }
  7315   7437         azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
  7316         -      if( 0==azResult[nRow] ){
  7317         -        rc = shellNomemError();
  7318         -        break;
  7319         -      }
         7438  +      if( 0==azResult[nRow] ) shell_out_of_memory();
  7320   7439         nRow++;
  7321   7440       }
  7322   7441       if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
  7323   7442         rc = shellDatabaseError(p->db);
  7324   7443       }
  7325   7444   
  7326   7445       /* Pretty-print the contents of array azResult[] to the output */
................................................................................
  7883   8002       if( line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){
  7884   8003         memcpy(zLine,";",2);
  7885   8004       }
  7886   8005       nLine = strlen30(zLine);
  7887   8006       if( nSql+nLine+2>=nAlloc ){
  7888   8007         nAlloc = nSql+nLine+100;
  7889   8008         zSql = realloc(zSql, nAlloc);
  7890         -      if( zSql==0 ){
  7891         -        raw_printf(stderr, "Error: out of memory\n");
  7892         -        exit(1);
  7893         -      }
         8009  +      if( zSql==0 ) shell_out_of_memory();
  7894   8010       }
  7895   8011       nSqlPrior = nSql;
  7896   8012       if( nSql==0 ){
  7897   8013         int i;
  7898   8014         for(i=0; zLine[i] && IsSpace(zLine[i]); i++){}
  7899   8015         assert( nAlloc>0 && zSql!=0 );
  7900   8016         memcpy(zSql, zLine+i, nLine+1-i);
................................................................................
  8015   8131     if (sqliterc == NULL) {
  8016   8132       home_dir = find_home_dir(0);
  8017   8133       if( home_dir==0 ){
  8018   8134         raw_printf(stderr, "-- warning: cannot find home directory;"
  8019   8135                         " cannot read ~/.sqliterc\n");
  8020   8136         return;
  8021   8137       }
  8022         -    sqlite3_initialize();
  8023   8138       zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir);
  8024   8139       sqliterc = zBuf;
  8025   8140     }
  8026   8141     in = fopen(sqliterc,"rb");
  8027   8142     if( in ){
  8028   8143       if( stdin_is_interactive ){
  8029   8144         utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc);
................................................................................
  8066   8181   #endif
  8067   8182     "   -newline SEP         set output row separator. Default: '\\n'\n"
  8068   8183     "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
  8069   8184     "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
  8070   8185     "   -quote               set output mode to 'quote'\n"
  8071   8186     "   -readonly            open the database read-only\n"
  8072   8187     "   -separator SEP       set output column separator. Default: '|'\n"
         8188  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         8189  +  "   -sorterref SIZE      sorter references threshold size\n"
         8190  +#endif
  8073   8191     "   -stats               print memory stats before each finalize\n"
  8074   8192     "   -version             show SQLite version\n"
  8075   8193     "   -vfs NAME            use NAME as the default VFS\n"
  8076   8194   #ifdef SQLITE_ENABLE_VFSTRACE
  8077   8195     "   -vfstrace            enable tracing of all VFS calls\n"
  8078   8196   #endif
  8079   8197   #ifdef SQLITE_HAVE_ZLIB
................................................................................
  8088   8206     if( showDetail ){
  8089   8207       utf8_printf(stderr, "OPTIONS include:\n%s", zOptions);
  8090   8208     }else{
  8091   8209       raw_printf(stderr, "Use the -help option for additional information\n");
  8092   8210     }
  8093   8211     exit(1);
  8094   8212   }
         8213  +
         8214  +/*
         8215  +** Internal check:  Verify that the SQLite is uninitialized.  Print a
         8216  +** error message if it is initialized.
         8217  +*/
         8218  +static void verify_uninitialized(void){
         8219  +  if( sqlite3_config(-1)==SQLITE_MISUSE ){
         8220  +    utf8_printf(stdout, "WARNING: attempt to configuration SQLite after"
         8221  +                        " initialization.\n");
         8222  +  }
         8223  +}
  8095   8224   
  8096   8225   /*
  8097   8226   ** Initialize the state information in data
  8098   8227   */
  8099   8228   static void main_init(ShellState *data) {
  8100   8229     memset(data, 0, sizeof(*data));
  8101   8230     data->normalMode = data->cMode = data->mode = MODE_List;
  8102   8231     data->autoExplain = 1;
  8103   8232     memcpy(data->colSeparator,SEP_Column, 2);
  8104   8233     memcpy(data->rowSeparator,SEP_Row, 2);
  8105   8234     data->showHeader = 0;
  8106   8235     data->shellFlgs = SHFLG_Lookaside;
         8236  +  verify_uninitialized();
  8107   8237     sqlite3_config(SQLITE_CONFIG_URI, 1);
  8108   8238     sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data);
  8109   8239     sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  8110   8240     sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> ");
  8111   8241     sqlite3_snprintf(sizeof(continuePrompt), continuePrompt,"   ...> ");
  8112   8242   }
  8113   8243   
................................................................................
  8163   8293     const char *zInitFile = 0;
  8164   8294     int i;
  8165   8295     int rc = 0;
  8166   8296     int warnInmemoryDb = 0;
  8167   8297     int readStdin = 1;
  8168   8298     int nCmd = 0;
  8169   8299     char **azCmd = 0;
         8300  +  const char *zVfs = 0;           /* Value of -vfs command-line option */
  8170   8301   
  8171   8302     setBinaryMode(stdin, 0);
  8172   8303     setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
  8173   8304     stdin_is_interactive = isatty(0);
  8174   8305     stdout_is_console = isatty(1);
  8175   8306   
  8176   8307   #if USE_SYSTEM_SQLITE+0!=1
................................................................................
  8187   8318     ** do this.  But we want to run an sqlite3_shutdown() afterwards so that
  8188   8319     ** subsequent sqlite3_config() calls will work.  So copy all results into
  8189   8320     ** memory that does not come from the SQLite memory allocator.
  8190   8321     */
  8191   8322   #if !SQLITE_SHELL_IS_UTF8
  8192   8323     sqlite3_initialize();
  8193   8324     argv = malloc(sizeof(argv[0])*argc);
  8194         -  if( argv==0 ){
  8195         -    raw_printf(stderr, "out of memory\n");
  8196         -    exit(1);
  8197         -  }
         8325  +  if( argv==0 ) shell_out_of_memory();
  8198   8326     for(i=0; i<argc; i++){
  8199   8327       char *z = sqlite3_win32_unicode_to_utf8(wargv[i]);
  8200   8328       int n;
  8201         -    if( z==0 ){
  8202         -      raw_printf(stderr, "out of memory\n");
  8203         -      exit(1);
  8204         -    }
         8329  +    if( z==0 ) shell_out_of_memory();
  8205   8330       n = (int)strlen(z);
  8206   8331       argv[i] = malloc( n+1 );
  8207         -    if( argv[i]==0 ){
  8208         -      raw_printf(stderr, "out of memory\n");
  8209         -      exit(1);
  8210         -    }
         8332  +    if( argv[i]==0 ) shell_out_of_memory();
  8211   8333       memcpy(argv[i], z, n+1);
  8212   8334       sqlite3_free(z);
  8213   8335     }
  8214   8336     sqlite3_shutdown();
  8215   8337   #endif
  8216   8338   
  8217   8339     assert( argc>=1 && argv && argv[0] );
................................................................................
  8239   8361   #endif
  8240   8362   
  8241   8363     /* Do an initial pass through the command-line argument to locate
  8242   8364     ** the name of the database file, the name of the initialization file,
  8243   8365     ** the size of the alternative malloc heap,
  8244   8366     ** and the first command to execute.
  8245   8367     */
         8368  +  verify_uninitialized();
  8246   8369     for(i=1; i<argc; i++){
  8247   8370       char *z;
  8248   8371       z = argv[i];
  8249   8372       if( z[0]!='-' ){
  8250   8373         if( data.zDbFilename==0 ){
  8251   8374           data.zDbFilename = z;
  8252   8375         }else{
  8253   8376           /* Excesss arguments are interpreted as SQL (or dot-commands) and
  8254   8377           ** mean that nothing is read from stdin */
  8255   8378           readStdin = 0;
  8256   8379           nCmd++;
  8257   8380           azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd);
  8258         -        if( azCmd==0 ){
  8259         -          raw_printf(stderr, "out of memory\n");
  8260         -          exit(1);
  8261         -        }
         8381  +        if( azCmd==0 ) shell_out_of_memory();
  8262   8382           azCmd[nCmd-1] = z;
  8263   8383         }
  8264   8384       }
  8265   8385       if( z[1]=='-' ) z++;
  8266   8386       if( strcmp(z,"-separator")==0
  8267   8387        || strcmp(z,"-nullvalue")==0
  8268   8388        || strcmp(z,"-newline")==0
................................................................................
  8321   8441       }else if( strcmp(z,"-multiplex")==0 ){
  8322   8442         extern int sqlite3_multiple_initialize(const char*,int);
  8323   8443         sqlite3_multiplex_initialize(0, 1);
  8324   8444   #endif
  8325   8445       }else if( strcmp(z,"-mmap")==0 ){
  8326   8446         sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
  8327   8447         sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz);
         8448  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         8449  +    }else if( strcmp(z,"-sorterref")==0 ){
         8450  +      sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
         8451  +      sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz);
         8452  +#endif
  8328   8453       }else if( strcmp(z,"-vfs")==0 ){
  8329         -      sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i));
  8330         -      if( pVfs ){
  8331         -        sqlite3_vfs_register(pVfs, 1);
  8332         -      }else{
  8333         -        utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
  8334         -        exit(1);
  8335         -      }
         8454  +      zVfs = cmdline_option_value(argc, argv, ++i);
  8336   8455   #ifdef SQLITE_HAVE_ZLIB
  8337   8456       }else if( strcmp(z,"-zip")==0 ){
  8338   8457         data.openMode = SHELL_OPEN_ZIPFILE;
  8339   8458   #endif
  8340   8459       }else if( strcmp(z,"-append")==0 ){
  8341   8460         data.openMode = SHELL_OPEN_APPENDVFS;
  8342   8461       }else if( strcmp(z,"-readonly")==0 ){
................................................................................
  8345   8464       }else if( strncmp(z, "-A",2)==0 ){
  8346   8465         /* All remaining command-line arguments are passed to the ".archive"
  8347   8466         ** command, so ignore them */
  8348   8467         break;
  8349   8468   #endif
  8350   8469       }
  8351   8470     }
         8471  +  verify_uninitialized();
         8472  +
         8473  +
         8474  +  /* All the sqlite3_config() calls have now been made. So it is safe
         8475  +  ** to call sqlite3_initialize() and process any command line -vfs option. */
         8476  +  sqlite3_initialize();
         8477  +  if( zVfs ){
         8478  +    sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs);
         8479  +    if( pVfs ){
         8480  +      sqlite3_vfs_register(pVfs, 1);
         8481  +    }else{
         8482  +      utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
         8483  +      exit(1);
         8484  +    }
         8485  +  }
         8486  +
  8352   8487     if( data.zDbFilename==0 ){
  8353   8488   #ifndef SQLITE_OMIT_MEMORYDB
  8354   8489       data.zDbFilename = ":memory:";
  8355   8490       warnInmemoryDb = argc==1;
  8356   8491   #else
  8357   8492       utf8_printf(stderr,"%s: Error: no database filename specified\n", Argv0);
  8358   8493       return 1;
................................................................................
  8402   8537         memcpy(data.colSeparator,",",2);
  8403   8538   #ifdef SQLITE_HAVE_ZLIB
  8404   8539       }else if( strcmp(z,"-zip")==0 ){
  8405   8540         data.openMode = SHELL_OPEN_ZIPFILE;
  8406   8541   #endif
  8407   8542       }else if( strcmp(z,"-append")==0 ){
  8408   8543         data.openMode = SHELL_OPEN_APPENDVFS;
         8544  +    }else if( strcmp(z,"-readonly")==0 ){
         8545  +      data.openMode = SHELL_OPEN_READONLY;
  8409   8546       }else if( strcmp(z,"-ascii")==0 ){
  8410   8547         data.mode = MODE_Ascii;
  8411   8548         sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
  8412   8549                          SEP_Unit);
  8413   8550         sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,
  8414   8551                          SEP_Record);
  8415   8552       }else if( strcmp(z,"-separator")==0 ){
................................................................................
  8455   8592         i++;
  8456   8593       }else if( strcmp(z,"-pagecache")==0 ){
  8457   8594         i+=2;
  8458   8595       }else if( strcmp(z,"-lookaside")==0 ){
  8459   8596         i+=2;
  8460   8597       }else if( strcmp(z,"-mmap")==0 ){
  8461   8598         i++;
         8599  +#ifdef SQLITE_ENABLE_SORTER_REFERENCES
         8600  +    }else if( strcmp(z,"-sorterref")==0 ){
         8601  +      i++;
         8602  +#endif
  8462   8603       }else if( strcmp(z,"-vfs")==0 ){
  8463   8604         i++;
  8464   8605   #ifdef SQLITE_ENABLE_VFSTRACE
  8465   8606       }else if( strcmp(z,"-vfstrace")==0 ){
  8466   8607         i++;
  8467   8608   #endif
  8468   8609   #ifdef SQLITE_ENABLE_MULTIPLEX

Changes to src/sqlite.h.in.

  1060   1060   ** The [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE] opcode causes all write
  1061   1061   ** operations since the previous successful call to 
  1062   1062   ** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be rolled back.
  1063   1063   ** ^This file control takes the file descriptor out of batch write mode
  1064   1064   ** so that all subsequent write operations are independent.
  1065   1065   ** ^SQLite will never invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE without
  1066   1066   ** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE].
         1067  +**
         1068  +** <li>[[SQLITE_FCNTL_LOCK_TIMEOUT]]
         1069  +** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode causes attempts to obtain
         1070  +** a file lock using the xLock or xShmLock methods of the VFS to wait
         1071  +** for up to M milliseconds before failing, where M is the single 
         1072  +** unsigned integer parameter.
  1067   1073   ** </ul>
  1068   1074   */
  1069   1075   #define SQLITE_FCNTL_LOCKSTATE               1
  1070   1076   #define SQLITE_FCNTL_GET_LOCKPROXYFILE       2
  1071   1077   #define SQLITE_FCNTL_SET_LOCKPROXYFILE       3
  1072   1078   #define SQLITE_FCNTL_LAST_ERRNO              4
  1073   1079   #define SQLITE_FCNTL_SIZE_HINT               5
................................................................................
  1094   1100   #define SQLITE_FCNTL_VFS_POINTER            27
  1095   1101   #define SQLITE_FCNTL_JOURNAL_POINTER        28
  1096   1102   #define SQLITE_FCNTL_WIN32_GET_HANDLE       29
  1097   1103   #define SQLITE_FCNTL_PDB                    30
  1098   1104   #define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE     31
  1099   1105   #define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE    32
  1100   1106   #define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE  33
         1107  +#define SQLITE_FCNTL_LOCK_TIMEOUT           34
  1101   1108   
  1102   1109   /* deprecated names */
  1103   1110   #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
  1104   1111   #define SQLITE_SET_LOCKPROXYFILE      SQLITE_FCNTL_SET_LOCKPROXYFILE
  1105   1112   #define SQLITE_LAST_ERRNO             SQLITE_FCNTL_LAST_ERRNO
  1106   1113   
  1107   1114   
................................................................................
  1919   1926   ** Or if the threshold is -1, statement journals are always held
  1920   1927   ** exclusively in memory.
  1921   1928   ** Since many statement journals never become large, setting the spill
  1922   1929   ** threshold to a value such as 64KiB can greatly reduce the amount of
  1923   1930   ** I/O required to support statement rollback.
  1924   1931   ** The default value for this setting is controlled by the
  1925   1932   ** [SQLITE_STMTJRNL_SPILL] compile-time option.
         1933  +**
         1934  +** [[SQLITE_CONFIG_SORTERREF_SIZE]]
         1935  +** <dt>SQLITE_CONFIG_SORTERREF_SIZE
         1936  +** <dd>The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter
         1937  +** of type (int) - the new value of the sorter-reference size threshold.
         1938  +** Usually, when SQLite uses an external sort to order records according
         1939  +** to an ORDER BY clause, all fields required by the caller are present in the
         1940  +** sorted records. However, if SQLite determines based on the declared type
         1941  +** of a table column that its values are likely to be very large - larger
         1942  +** than the configured sorter-reference size threshold - then a reference
         1943  +** is stored in each sorted record and the required column values loaded
         1944  +** from the database as records are returned in sorted order. The default
         1945  +** value for this option is to never use this optimization. Specifying a 
         1946  +** negative value for this option restores the default behaviour.
         1947  +** This option is only available if SQLite is compiled with the
         1948  +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option.
  1926   1949   ** </dl>
  1927   1950   */
  1928   1951   #define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
  1929   1952   #define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
  1930   1953   #define SQLITE_CONFIG_SERIALIZED    3  /* nil */
  1931   1954   #define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
  1932   1955   #define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
................................................................................
  1948   1971   #define SQLITE_CONFIG_SQLLOG       21  /* xSqllog, void* */
  1949   1972   #define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
  1950   1973   #define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
  1951   1974   #define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
  1952   1975   #define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
  1953   1976   #define SQLITE_CONFIG_STMTJRNL_SPILL      26  /* int nByte */
  1954   1977   #define SQLITE_CONFIG_SMALL_MALLOC        27  /* boolean */
         1978  +#define SQLITE_CONFIG_SORTERREF_SIZE      28  /* int nByte */
  1955   1979   
  1956   1980   /*
  1957   1981   ** CAPI3REF: Database Connection Configuration Options
  1958   1982   **
  1959   1983   ** These constants are the available integer configuration options that
  1960   1984   ** can be passed as the second argument to the [sqlite3_db_config()] interface.
  1961   1985   **
................................................................................
  8832   8856   */
  8833   8857   #define SQLITE_SERIALIZE_NOCOPY 0x001   /* Do no memory allocations */
  8834   8858   
  8835   8859   /*
  8836   8860   ** CAPI3REF: Deserialize a database
  8837   8861   **
  8838   8862   ** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the 
  8839         -** [database connection] D to disconnection from database S and then
         8863  +** [database connection] D to disconnect from database S and then
  8840   8864   ** reopen S as an in-memory database based on the serialization contained
  8841   8865   ** in P.  The serialized database P is N bytes in size.  M is the size of
  8842   8866   ** the buffer P, which might be larger than N.  If M is larger than N, and
  8843   8867   ** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is
  8844   8868   ** permitted to add content to the in-memory database as long as the total
  8845   8869   ** size does not exceed M bytes.
  8846   8870   **

Changes to src/sqliteInt.h.

   633    633   ** The default value of "20" was choosen to minimize the run-time of the
   634    634   ** speedtest1 test program with options: --shrink-memory --reprepare
   635    635   */
   636    636   #ifndef SQLITE_DEFAULT_PCACHE_INITSZ
   637    637   # define SQLITE_DEFAULT_PCACHE_INITSZ 20
   638    638   #endif
   639    639   
          640  +/*
          641  +** Default value for the SQLITE_CONFIG_SORTERREF_SIZE option.
          642  +*/
          643  +#ifndef SQLITE_DEFAULT_SORTERREF_SIZE
          644  +# define SQLITE_DEFAULT_SORTERREF_SIZE 0x7fffffff
          645  +#endif
          646  +
   640    647   /*
   641    648   ** The compile-time options SQLITE_MMAP_READWRITE and 
   642    649   ** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another.
   643    650   ** You must choose one or the other (or neither) but not both.
   644    651   */
   645    652   #if defined(SQLITE_MMAP_READWRITE) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE)
   646    653   #error Cannot use both SQLITE_MMAP_READWRITE and SQLITE_ENABLE_BATCH_ATOMIC_WRITE
................................................................................
   957    964   ** The sqlite.busyHandler member of the sqlite struct contains the busy
   958    965   ** callback for the database handle. Each pager opened via the sqlite
   959    966   ** handle is passed a pointer to sqlite.busyHandler. The busy-handler
   960    967   ** callback is currently invoked only from within pager.c.
   961    968   */
   962    969   typedef struct BusyHandler BusyHandler;
   963    970   struct BusyHandler {
   964         -  int (*xFunc)(void *,int);  /* The busy callback */
   965         -  void *pArg;                /* First arg to busy callback */
   966         -  int nBusy;                 /* Incremented with each busy call */
          971  +  int (*xBusyHandler)(void *,int);  /* The busy callback */
          972  +  void *pBusyArg;                   /* First arg to busy callback */
          973  +  int nBusy;                        /* Incremented with each busy call */
          974  +  u8 bExtraFileArg;                 /* Include sqlite3_file as callback arg */
   967    975   };
   968    976   
   969    977   /*
   970    978   ** Name of the master database table.  The master database table
   971    979   ** is a special table that holds the names and attributes of all
   972    980   ** user tables and indices.
   973    981   */
................................................................................
  1090   1098   typedef struct TableLock TableLock;
  1091   1099   typedef struct Token Token;
  1092   1100   typedef struct TreeView TreeView;
  1093   1101   typedef struct Trigger Trigger;
  1094   1102   typedef struct TriggerPrg TriggerPrg;
  1095   1103   typedef struct TriggerStep TriggerStep;
  1096   1104   typedef struct UnpackedRecord UnpackedRecord;
         1105  +typedef struct Upsert Upsert;
  1097   1106   typedef struct VTable VTable;
  1098   1107   typedef struct VtabCtx VtabCtx;
  1099   1108   typedef struct Walker Walker;
  1100   1109   typedef struct WhereInfo WhereInfo;
  1101   1110   typedef struct With With;
  1102   1111   
  1103   1112   /* A VList object records a mapping between parameters/variables/wildcards
................................................................................
  1531   1540   #define SQLITE_Transitive     0x0080   /* Transitive constraints */
  1532   1541   #define SQLITE_OmitNoopJoin   0x0100   /* Omit unused tables in joins */
  1533   1542   #define SQLITE_CountOfView    0x0200   /* The count-of-view optimization */
  1534   1543   #define SQLITE_CursorHints    0x0400   /* Add OP_CursorHint opcodes */
  1535   1544   #define SQLITE_Stat34         0x0800   /* Use STAT3 or STAT4 data */
  1536   1545      /* TH3 expects the Stat34  ^^^^^^ value to be 0x0800.  Don't change it */
  1537   1546   #define SQLITE_PushDown       0x1000   /* The push-down optimization */
         1547  +#define SQLITE_SimplifyJoin   0x2000   /* Convert LEFT JOIN to JOIN */
  1538   1548   #define SQLITE_AllOpts        0xffff   /* All optimizations */
  1539   1549   
  1540   1550   /*
  1541   1551   ** Macros for testing whether or not optimizations are enabled or disabled.
  1542   1552   */
  1543   1553   #define OptimizationDisabled(db, mask)  (((db)->dbOptFlags&(mask))!=0)
  1544   1554   #define OptimizationEnabled(db, mask)   (((db)->dbOptFlags&(mask))==0)
................................................................................
  1755   1765   
  1756   1766   /* Allowed values for Column.colFlags:
  1757   1767   */
  1758   1768   #define COLFLAG_PRIMKEY  0x0001    /* Column is part of the primary key */
  1759   1769   #define COLFLAG_HIDDEN   0x0002    /* A hidden column in a virtual table */
  1760   1770   #define COLFLAG_HASTYPE  0x0004    /* Type name follows column name */
  1761   1771   #define COLFLAG_UNIQUE   0x0008    /* Column def contains "UNIQUE" or "PK" */
         1772  +#define COLFLAG_SORTERREF 0x0010   /* Use sorter-refs with this column */
  1762   1773   
  1763   1774   /*
  1764   1775   ** A "Collating Sequence" is defined by an instance of the following
  1765   1776   ** structure. Conceptually, a collating sequence consists of a name and
  1766   1777   ** a comparison routine that defines the order of that sequence.
  1767   1778   **
  1768   1779   ** If CollSeq.xCmp is NULL, it means that the
................................................................................
  2042   2053   */
  2043   2054   #define OE_None     0   /* There is no constraint to check */
  2044   2055   #define OE_Rollback 1   /* Fail the operation and rollback the transaction */
  2045   2056   #define OE_Abort    2   /* Back out changes but do no rollback transaction */
  2046   2057   #define OE_Fail     3   /* Stop the operation but leave all prior changes */
  2047   2058   #define OE_Ignore   4   /* Ignore the error. Do not do the INSERT or UPDATE */
  2048   2059   #define OE_Replace  5   /* Delete existing record, then do INSERT or UPDATE */
  2049         -
  2050         -#define OE_Restrict 6   /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */
  2051         -#define OE_SetNull  7   /* Set the foreign key value to NULL */
  2052         -#define OE_SetDflt  8   /* Set the foreign key value to its default */
  2053         -#define OE_Cascade  9   /* Cascade the changes */
  2054         -
  2055         -#define OE_Default  10  /* Do whatever the default action is */
         2060  +#define OE_Update   6   /* Process as a DO UPDATE in an upsert */
         2061  +#define OE_Restrict 7   /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */
         2062  +#define OE_SetNull  8   /* Set the foreign key value to NULL */
         2063  +#define OE_SetDflt  9   /* Set the foreign key value to its default */
         2064  +#define OE_Cascade  10  /* Cascade the changes */
         2065  +#define OE_Default  11  /* Do whatever the default action is */
  2056   2066   
  2057   2067   
  2058   2068   /*
  2059   2069   ** An instance of the following structure is passed as the first
  2060   2070   ** argument to sqlite3VdbeKeyCompare and is used to control the
  2061   2071   ** comparison of the two index keys.
  2062   2072   **
................................................................................
  2495   2505       Expr *pExpr;            /* The parse tree for this expression */
  2496   2506       char *zName;            /* Token associated with this expression */
  2497   2507       char *zSpan;            /* Original text of the expression */
  2498   2508       u8 sortOrder;           /* 1 for DESC or 0 for ASC */
  2499   2509       unsigned done :1;       /* A flag to indicate when processing is finished */
  2500   2510       unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */
  2501   2511       unsigned reusable :1;   /* Constant expression is reusable */
         2512  +    unsigned bSorterRef :1; /* Defer evaluation until after sorting */
  2502   2513       union {
  2503   2514         struct {
  2504   2515           u16 iOrderByCol;      /* For ORDER BY, column number in result set */
  2505   2516           u16 iAlias;           /* Index into Parse.aAlias[] for zName */
  2506   2517         } x;
  2507   2518         int iConstExprReg;      /* Register in which Expr value is cached */
  2508   2519       } u;
................................................................................
  2678   2689   ** NameContext in the parent query.  Thus the process of scanning the
  2679   2690   ** NameContext list corresponds to searching through successively outer
  2680   2691   ** subqueries looking for a match.
  2681   2692   */
  2682   2693   struct NameContext {
  2683   2694     Parse *pParse;       /* The parser */
  2684   2695     SrcList *pSrcList;   /* One or more tables used to resolve names */
  2685         -  ExprList *pEList;    /* Optional list of result-set columns */
  2686         -  AggInfo *pAggInfo;   /* Information about aggregates at this level */
         2696  +  union {
         2697  +    ExprList *pEList;    /* Optional list of result-set columns */
         2698  +    AggInfo *pAggInfo;   /* Information about aggregates at this level */
         2699  +    Upsert *pUpsert;     /* ON CONFLICT clause information from an upsert */
         2700  +  } uNC;
  2687   2701     NameContext *pNext;  /* Next outer name context.  NULL for outermost */
  2688   2702     int nRef;            /* Number of names resolved by this context */
  2689   2703     int nErr;            /* Number of errors encountered while resolving names */
  2690   2704     u16 ncFlags;         /* Zero or more NC_* flags defined below */
  2691   2705   };
  2692   2706   
  2693   2707   /*
................................................................................
  2701   2715   #define NC_AllowAgg  0x0001  /* Aggregate functions are allowed here */
  2702   2716   #define NC_PartIdx   0x0002  /* True if resolving a partial index WHERE */
  2703   2717   #define NC_IsCheck   0x0004  /* True if resolving names in a CHECK constraint */
  2704   2718   #define NC_InAggFunc 0x0008  /* True if analyzing arguments to an agg func */
  2705   2719   #define NC_HasAgg    0x0010  /* One or more aggregate functions seen */
  2706   2720   #define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
  2707   2721   #define NC_VarSelect 0x0040  /* A correlated subquery has been seen */
         2722  +#define NC_UEList    0x0080  /* True if uNC.pEList is used */
         2723  +#define NC_UAggInfo  0x0100  /* True if uNC.pAggInfo is used */
         2724  +#define NC_UUpsert   0x0200  /* True if uNC.pUpsert is used */
  2708   2725   #define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */
  2709   2726   #define NC_Complex   0x2000  /* True if a function or subquery seen */
         2727  +
         2728  +/*
         2729  +** An instance of the following object describes a single ON CONFLICT
         2730  +** clause in an upsert.
         2731  +**
         2732  +** The pUpsertTarget field is only set if the ON CONFLICT clause includes
         2733  +** conflict-target clause.  (In "ON CONFLICT(a,b)" the "(a,b)" is the
         2734  +** conflict-target clause.)  The pUpsertTargetWhere is the optional
         2735  +** WHERE clause used to identify partial unique indexes.
         2736  +**
         2737  +** pUpsertSet is the list of column=expr terms of the UPDATE statement. 
         2738  +** The pUpsertSet field is NULL for a ON CONFLICT DO NOTHING.  The
         2739  +** pUpsertWhere is the WHERE clause for the UPDATE and is NULL if the
         2740  +** WHERE clause is omitted.
         2741  +*/
         2742  +struct Upsert {
         2743  +  ExprList *pUpsertTarget;  /* Optional description of conflicting index */
         2744  +  Expr *pUpsertTargetWhere; /* WHERE clause for partial index targets */
         2745  +  ExprList *pUpsertSet;     /* The SET clause from an ON CONFLICT UPDATE */
         2746  +  Expr *pUpsertWhere;       /* WHERE clause for the ON CONFLICT UPDATE */
         2747  +  /* The fields above comprise the parse tree for the upsert clause.
         2748  +  ** The fields below are used to transfer information from the INSERT
         2749  +  ** processing down into the UPDATE processing while generating code.
         2750  +  ** Upsert owns the memory allocated above, but not the memory below. */
         2751  +  Index *pUpsertIdx;        /* Constraint that pUpsertTarget identifies */
         2752  +  SrcList *pUpsertSrc;      /* Table to be updated */
         2753  +  int regData;              /* First register holding array of VALUES */
         2754  +  int iDataCur;             /* Index of the data cursor */
         2755  +  int iIdxCur;              /* Index of the first index cursor */
         2756  +};
  2710   2757   
  2711   2758   /*
  2712   2759   ** An instance of the following structure contains all information
  2713   2760   ** needed to generate code for a single SELECT statement.
  2714   2761   **
  2715   2762   ** nLimit is set to -1 if there is no LIMIT clause.  nOffset is set to 0.
  2716   2763   ** If there is a LIMIT clause, the parser sets nLimit to the value of the
................................................................................
  2732   2779     ExprList *pEList;      /* The fields of the result */
  2733   2780     u8 op;                 /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */
  2734   2781     LogEst nSelectRow;     /* Estimated number of result rows */
  2735   2782     u32 selFlags;          /* Various SF_* values */
  2736   2783     int iLimit, iOffset;   /* Memory registers holding LIMIT & OFFSET counters */
  2737   2784   #if SELECTTRACE_ENABLED
  2738   2785     char zSelName[12];     /* Symbolic name of this SELECT use for debugging */
         2786  +  u32 iSelectId;         /* EXPLAIN QUERY PLAN select ID */
  2739   2787   #endif
  2740   2788     int addrOpenEphm[2];   /* OP_OpenEphem opcodes related to this select */
  2741   2789     SrcList *pSrc;         /* The FROM clause */
  2742   2790     Expr *pWhere;          /* The WHERE clause */
  2743   2791     ExprList *pGroupBy;    /* The GROUP BY clause */
  2744   2792     Expr *pHaving;         /* The HAVING clause */
  2745   2793     ExprList *pOrderBy;    /* The ORDER BY clause */
................................................................................
  3203   3251   struct TriggerStep {
  3204   3252     u8 op;               /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */
  3205   3253     u8 orconf;           /* OE_Rollback etc. */
  3206   3254     Trigger *pTrig;      /* The trigger that this step is a part of */
  3207   3255     Select *pSelect;     /* SELECT statement or RHS of INSERT INTO SELECT ... */
  3208   3256     char *zTarget;       /* Target table for DELETE, UPDATE, INSERT */
  3209   3257     Expr *pWhere;        /* The WHERE clause for DELETE or UPDATE steps */
  3210         -  ExprList *pExprList; /* SET clause for UPDATE. */
         3258  +  ExprList *pExprList; /* SET clause for UPDATE */
  3211   3259     IdList *pIdList;     /* Column names for INSERT */
         3260  +  Upsert *pUpsert;     /* Upsert clauses on an INSERT */
  3212   3261     char *zSpan;         /* Original SQL text of this command */
  3213   3262     TriggerStep *pNext;  /* Next in the link-list */
  3214   3263     TriggerStep *pLast;  /* Last element in link-list. Valid for 1st elem only */
  3215   3264   };
  3216   3265   
  3217   3266   /*
  3218   3267   ** The following structure contains information used by the sqliteFix...
................................................................................
  3316   3365     void *pVdbeBranchArg;                                     /* 1st argument */
  3317   3366   #endif
  3318   3367   #ifndef SQLITE_UNTESTABLE
  3319   3368     int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
  3320   3369   #endif
  3321   3370     int bLocaltimeFault;              /* True to fail localtime() calls */
  3322   3371     int iOnceResetThreshold;          /* When to reset OP_Once counters */
         3372  +  u32 szSorterRef;                  /* Min size in bytes to use sorter-refs */
  3323   3373   };
  3324   3374   
  3325   3375   /*
  3326   3376   ** This macro is used inside of assert() statements to indicate that
  3327   3377   ** the assert is only valid on a well-formed database.  Instead of:
  3328   3378   **
  3329   3379   **     assert( X );
................................................................................
  3355   3405       int n;                                    /* A counter */
  3356   3406       int iCur;                                 /* A cursor number */
  3357   3407       SrcList *pSrcList;                        /* FROM clause */
  3358   3408       struct SrcCount *pSrcCount;               /* Counting column references */
  3359   3409       struct CCurHint *pCCurHint;               /* Used by codeCursorHint() */
  3360   3410       int *aiCol;                               /* array of column indexes */
  3361   3411       struct IdxCover *pIdxCover;               /* Check for index coverage */
  3362         -    struct IdxExprTrans *pIdxTrans;           /* Convert indexed expr to column */
         3412  +    struct IdxExprTrans *pIdxTrans;           /* Convert idxed expr to column */
  3363   3413       ExprList *pGroupBy;                       /* GROUP BY clause */
  3364         -    struct HavingToWhereCtx *pHavingCtx;      /* HAVING to WHERE clause ctx */
         3414  +    Select *pSelect;                          /* HAVING to WHERE clause ctx */
  3365   3415     } u;
  3366   3416   };
  3367   3417   
  3368   3418   /* Forward declarations */
  3369   3419   int sqlite3WalkExpr(Walker*, Expr*);
  3370   3420   int sqlite3WalkExprList(Walker*, ExprList*);
  3371   3421   int sqlite3WalkSelect(Walker*, Select*);
................................................................................
  3736   3786   #ifndef SQLITE_OMIT_AUTOINCREMENT
  3737   3787     void sqlite3AutoincrementBegin(Parse *pParse);
  3738   3788     void sqlite3AutoincrementEnd(Parse *pParse);
  3739   3789   #else
  3740   3790   # define sqlite3AutoincrementBegin(X)
  3741   3791   # define sqlite3AutoincrementEnd(X)
  3742   3792   #endif
  3743         -void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int);
         3793  +void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*);
  3744   3794   void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*);
  3745   3795   IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*);
  3746   3796   int sqlite3IdListIndex(IdList*,const char*);
  3747   3797   SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int);
  3748   3798   SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*);
  3749   3799   SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
  3750   3800                                         Token*, Select*, Expr*, IdList*);
................................................................................
  3766   3816   Table *sqlite3SrcListLookup(Parse*, SrcList*);
  3767   3817   int sqlite3IsReadOnly(Parse*, Table*, int);
  3768   3818   void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
  3769   3819   #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
  3770   3820   Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*);
  3771   3821   #endif
  3772   3822   void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*);
  3773         -void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*);
         3823  +void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*,
         3824  +                   Upsert*);
  3774   3825   WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
  3775   3826   void sqlite3WhereEnd(WhereInfo*);
  3776   3827   LogEst sqlite3WhereOutputRowCount(WhereInfo*);
  3777   3828   int sqlite3WhereIsDistinct(WhereInfo*);
  3778   3829   int sqlite3WhereIsOrdered(WhereInfo*);
  3779   3830   int sqlite3WhereOrderedInnerLoop(WhereInfo*);
  3780   3831   int sqlite3WhereIsSorted(WhereInfo*);
................................................................................
  3821   3872   void sqlite3Vacuum(Parse*,Token*);
  3822   3873   int sqlite3RunVacuum(char**, sqlite3*, int);
  3823   3874   char *sqlite3NameFromToken(sqlite3*, Token*);
  3824   3875   int sqlite3ExprCompare(Parse*,Expr*, Expr*, int);
  3825   3876   int sqlite3ExprCompareSkip(Expr*, Expr*, int);
  3826   3877   int sqlite3ExprListCompare(ExprList*, ExprList*, int);
  3827   3878   int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int);
         3879  +int sqlite3ExprImpliesNonNullRow(Expr*,int);
  3828   3880   void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
  3829   3881   void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*);
  3830   3882   int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx);
  3831   3883   int sqlite3FunctionUsesThisSrc(Expr*, SrcList*);
  3832   3884   Vdbe *sqlite3GetVdbe(Parse*);
  3833   3885   #ifndef SQLITE_UNTESTABLE
  3834   3886   void sqlite3PrngSaveState(void);
................................................................................
  3858   3910   int sqlite3IsRowid(const char*);
  3859   3911   void sqlite3GenerateRowDelete(
  3860   3912       Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
  3861   3913   void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
  3862   3914   int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
  3863   3915   void sqlite3ResolvePartIdxLabel(Parse*,int);
  3864   3916   void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
  3865         -                                     u8,u8,int,int*,int*);
         3917  +                                     u8,u8,int,int*,int*,Upsert*);
  3866   3918   #ifdef SQLITE_ENABLE_NULL_TRIM
  3867   3919     void sqlite3SetMakeRecordP5(Vdbe*,Table*);
  3868   3920   #else
  3869   3921   # define sqlite3SetMakeRecordP5(A,B)
  3870   3922   #endif
  3871   3923   void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int);
  3872   3924   int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*);
................................................................................
  3911   3963                               int, int, int);
  3912   3964     void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, int, int, int);
  3913   3965     void sqliteViewTriggers(Parse*, Table*, Expr*, int, ExprList*);
  3914   3966     void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*);
  3915   3967     TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*,
  3916   3968                                           const char*,const char*);
  3917   3969     TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*,
  3918         -                                        Select*,u8,const char*,const char*);
         3970  +                                        Select*,u8,Upsert*,
         3971  +                                        const char*,const char*);
  3919   3972     TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8,
  3920   3973                                           const char*,const char*);
  3921   3974     TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*,
  3922   3975                                           const char*,const char*);
  3923   3976     void sqlite3DeleteTrigger(sqlite3*, Trigger*);
  3924   3977     void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*);
  3925   3978     u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int);
................................................................................
  4097   4150   void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*);
  4098   4151   void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*);
  4099   4152   int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*);
  4100   4153   void sqlite3ColumnDefault(Vdbe *, Table *, int, int);
  4101   4154   void sqlite3AlterFinishAddColumn(Parse *, Token *);
  4102   4155   void sqlite3AlterBeginAddColumn(Parse *, SrcList *);
  4103   4156   CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*);
  4104         -char sqlite3AffinityType(const char*, u8*);
         4157  +char sqlite3AffinityType(const char*, Column*);
  4105   4158   void sqlite3Analyze(Parse*, Token*, Token*);
  4106         -int sqlite3InvokeBusyHandler(BusyHandler*);
         4159  +int sqlite3InvokeBusyHandler(BusyHandler*, sqlite3_file*);
  4107   4160   int sqlite3FindDb(sqlite3*, Token*);
  4108   4161   int sqlite3FindDbName(sqlite3 *, const char *);
  4109   4162   int sqlite3AnalysisLoad(sqlite3*,int iDB);
  4110   4163   void sqlite3DeleteIndexSamples(sqlite3*,Index*);
  4111   4164   void sqlite3DefaultRowEst(Index*);
  4112   4165   void sqlite3RegisterLikeFunctions(sqlite3*, int);
  4113   4166   int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*);
................................................................................
  4159   4212   char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
  4160   4213   #endif
  4161   4214   
  4162   4215   /*
  4163   4216   ** The interface to the LEMON-generated parser
  4164   4217   */
  4165   4218   #ifndef SQLITE_AMALGAMATION
  4166         -  void *sqlite3ParserAlloc(void*(*)(u64));
         4219  +  void *sqlite3ParserAlloc(void*(*)(u64), Parse*);
  4167   4220     void sqlite3ParserFree(void*, void(*)(void*));
  4168   4221   #endif
  4169         -void sqlite3Parser(void*, int, Token, Parse*);
         4222  +void sqlite3Parser(void*, int, Token);
  4170   4223   #ifdef YYTRACKMAXSTACKDEPTH
  4171   4224     int sqlite3ParserStackPeak(void*);
  4172   4225   #endif
  4173   4226   
  4174   4227   void sqlite3AutoLoadExtensions(sqlite3*);
  4175   4228   #ifndef SQLITE_OMIT_LOAD_EXTENSION
  4176   4229     void sqlite3CloseExtensions(sqlite3*);
................................................................................
  4250   4303     With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*);
  4251   4304     void sqlite3WithDelete(sqlite3*,With*);
  4252   4305     void sqlite3WithPush(Parse*, With*, u8);
  4253   4306   #else
  4254   4307   #define sqlite3WithPush(x,y,z)
  4255   4308   #define sqlite3WithDelete(x,y)
  4256   4309   #endif
         4310  +#ifndef SQLITE_OMIT_UPSERT
         4311  +  Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*);
         4312  +  void sqlite3UpsertDelete(sqlite3*,Upsert*);
         4313  +  Upsert *sqlite3UpsertDup(sqlite3*,Upsert*);
         4314  +  int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*);
         4315  +  void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int);
         4316  +#else
         4317  +#define sqlite3UpsertNew(v,w,x,y,z) ((Upsert*)0)
         4318  +#define sqlite3UpsertDelete(x,y)
         4319  +#define sqlite3UpsertDup(x,y)       ((Upsert*)0)
         4320  +#endif
         4321  +
  4257   4322   
  4258   4323   /* Declarations for functions in fkey.c. All of these are replaced by
  4259   4324   ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign
  4260   4325   ** key functionality is available. If OMIT_TRIGGER is defined but
  4261   4326   ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In
  4262   4327   ** this case foreign keys are parsed, but no other functionality is
  4263   4328   ** provided (enforcement of FK constraints requires the triggers sub-system).

Changes to src/test1.c.

  2251   2251       Tcl_WrongNumArgs(interp, 1, objv, "");
  2252   2252       return TCL_ERROR;
  2253   2253     }
  2254   2254     sqlite3_config(SQLITE_CONFIG_SQLLOG, 0, 0);
  2255   2255     return TCL_OK;
  2256   2256   }
  2257   2257   #endif
         2258  +
         2259  +/*
         2260  +** Usage:  sqlite3_config_sorterref
         2261  +**
         2262  +** Set the SQLITE_CONFIG_SORTERREF_SIZE configuration option
         2263  +*/
         2264  +static int SQLITE_TCLAPI test_config_sorterref(
         2265  +  void * clientData,
         2266  +  Tcl_Interp *interp,
         2267  +  int objc,
         2268  +  Tcl_Obj *CONST objv[]
         2269  +){
         2270  +  int iVal;
         2271  +  if( objc!=2 ){
         2272  +    Tcl_WrongNumArgs(interp, 1, objv, "NBYTE");
         2273  +    return TCL_ERROR;
         2274  +  }
         2275  +  if( Tcl_GetIntFromObj(interp, objv[1], &iVal) ) return TCL_ERROR;
         2276  +  sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, iVal);
         2277  +  return TCL_OK;
         2278  +}
  2258   2279   
  2259   2280   /*
  2260   2281   ** Usage: vfs_current_time_int64
  2261   2282   **
  2262   2283   ** Return the value returned by the default VFS's xCurrentTimeInt64 method.
  2263   2284   */
  2264   2285   static int SQLITE_TCLAPI vfsCurrentTimeInt64(
................................................................................
  7783   7804        { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
  7784   7805        { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
  7785   7806   #endif
  7786   7807        { "sqlite3_delete_database", test_delete_database, 0 },
  7787   7808        { "sqlite3_wal_info", test_wal_info, 0 },
  7788   7809        { "atomic_batch_write",      test_atomic_batch_write,     0   },
  7789   7810        { "sqlite3_mmap_warm",       test_mmap_warm,          0 },
         7811  +     { "sqlite3_config_sorterref", test_config_sorterref,   0 },
  7790   7812     };
  7791   7813     static int bitmask_size = sizeof(Bitmask)*8;
  7792   7814     static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
  7793   7815     int i;
  7794   7816     extern int sqlite3_sync_count, sqlite3_fullsync_count;
  7795   7817     extern int sqlite3_opentemp_count;
  7796   7818     extern int sqlite3_like_count;

Changes to src/tokenize.c.

   492    492     }
   493    493     pParse->rc = SQLITE_OK;
   494    494     pParse->zTail = zSql;
   495    495     assert( pzErrMsg!=0 );
   496    496     /* sqlite3ParserTrace(stdout, "parser: "); */
   497    497   #ifdef sqlite3Parser_ENGINEALWAYSONSTACK
   498    498     pEngine = &sEngine;
   499         -  sqlite3ParserInit(pEngine);
          499  +  sqlite3ParserInit(pEngine, pParse);
   500    500   #else
   501         -  pEngine = sqlite3ParserAlloc(sqlite3Malloc);
          501  +  pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse);
   502    502     if( pEngine==0 ){
   503    503       sqlite3OomFault(db);
   504    504       return SQLITE_NOMEM_BKPT;
   505    505     }
   506    506   #endif
   507    507     assert( pParse->pNewTable==0 );
   508    508     assert( pParse->pNewTrigger==0 );
................................................................................
   538    538           sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
   539    539           break;
   540    540         }
   541    541         zSql += n;
   542    542       }else{
   543    543         pParse->sLastToken.z = zSql;
   544    544         pParse->sLastToken.n = n;
   545         -      sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
          545  +      sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
   546    546         lastTokenParsed = tokenType;
   547    547         zSql += n;
   548    548         if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
   549    549       }
   550    550     }
   551    551     assert( nErr==0 );
   552    552     pParse->zTail = zSql;

Changes to src/treeview.c.

    58     58     sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0);
    59     59     if( p ){
    60     60       for(i=0; i<p->iLevel && i<sizeof(p->bLine)-1; i++){
    61     61         sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|   " : "    ", 4);
    62     62       }
    63     63       sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4);
    64     64     }
    65         -  va_start(ap, zFormat);
    66         -  sqlite3VXPrintf(&acc, zFormat, ap);
    67         -  va_end(ap);
    68         -  assert( acc.nChar>0 );
    69         -  if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1);
           65  +  if( zFormat!=0 ){
           66  +    va_start(ap, zFormat);
           67  +    sqlite3VXPrintf(&acc, zFormat, ap);
           68  +    va_end(ap);
           69  +    assert( acc.nChar>0 );
           70  +    sqlite3StrAccumAppend(&acc, "\n", 1);
           71  +  }
    70     72     sqlite3StrAccumFinish(&acc);
    71     73     fprintf(stdout,"%s", zBuf);
    72     74     fflush(stdout);
    73     75   }
    74     76   
    75     77   /*
    76     78   ** Shorthand for starting a new tree item that consists of a single label
................................................................................
   135    137       sqlite3TreeViewWith(pView, p->pWith, 1);
   136    138       cnt = 1;
   137    139       sqlite3TreeViewPush(pView, 1);
   138    140     }
   139    141     do{
   140    142   #if SELECTTRACE_ENABLED
   141    143       sqlite3TreeViewLine(pView,
   142         -      "SELECT%s%s (%s/%p) selFlags=0x%x nSelectRow=%d",
          144  +      "SELECT%s%s (%s/%d/%p) selFlags=0x%x nSelectRow=%d",
   143    145         ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""),
   144    146         ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""),
   145         -      p->zSelName, p, p->selFlags,
          147  +      p->zSelName, p->iSelectId, p, p->selFlags,
   146    148         (int)p->nSelectRow
   147    149       );
   148    150   #else
   149    151       sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d",
   150    152         ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""),
   151    153         ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags,
   152    154         (int)p->nSelectRow
................................................................................
   532    534       sqlite3TreeViewLine(pView, "%s (empty)", zLabel);
   533    535     }else{
   534    536       int i;
   535    537       sqlite3TreeViewLine(pView, "%s", zLabel);
   536    538       for(i=0; i<pList->nExpr; i++){
   537    539         int j = pList->a[i].u.x.iOrderByCol;
   538    540         char *zName = pList->a[i].zName;
          541  +      int moreToFollow = i<pList->nExpr - 1;
   539    542         if( j || zName ){
   540         -        sqlite3TreeViewPush(pView, 0);
   541         -      }
   542         -      if( zName ){
   543         -        sqlite3TreeViewLine(pView, "AS %s", zName);
          543  +        sqlite3TreeViewPush(pView, moreToFollow);
          544  +        moreToFollow = 0;
          545  +        sqlite3TreeViewLine(pView, 0);
          546  +        if( zName ){
          547  +          fprintf(stdout, "AS %s ", zName);
          548  +        }
          549  +        if( j ){
          550  +          fprintf(stdout, "iOrderByCol=%d", j);
          551  +        }
          552  +        fprintf(stdout, "\n");
          553  +        fflush(stdout);
   544    554         }
   545         -      if( j ){
   546         -        sqlite3TreeViewLine(pView, "iOrderByCol=%d", j);
   547         -      }
   548         -      sqlite3TreeViewExpr(pView, pList->a[i].pExpr, i<pList->nExpr-1);
          555  +      sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow);
   549    556         if( j || zName ){
   550    557           sqlite3TreeViewPop(pView);
   551    558         }
   552    559       }
   553    560     }
   554    561   }
   555    562   void sqlite3TreeViewExprList(

Changes to src/trigger.c.

    21     21       TriggerStep * pTmp = pTriggerStep;
    22     22       pTriggerStep = pTriggerStep->pNext;
    23     23   
    24     24       sqlite3ExprDelete(db, pTmp->pWhere);
    25     25       sqlite3ExprListDelete(db, pTmp->pExprList);
    26     26       sqlite3SelectDelete(db, pTmp->pSelect);
    27     27       sqlite3IdListDelete(db, pTmp->pIdList);
           28  +    sqlite3UpsertDelete(db, pTmp->pUpsert);
    28     29       sqlite3DbFree(db, pTmp->zSpan);
    29     30   
    30     31       sqlite3DbFree(db, pTmp);
    31     32     }
    32     33   }
    33     34   
    34     35   /*
................................................................................
   412    413   */
   413    414   TriggerStep *sqlite3TriggerInsertStep(
   414    415     sqlite3 *db,        /* The database connection */
   415    416     Token *pTableName,  /* Name of the table into which we insert */
   416    417     IdList *pColumn,    /* List of columns in pTableName to insert into */
   417    418     Select *pSelect,    /* A SELECT statement that supplies values */
   418    419     u8 orconf,          /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */
          420  +  Upsert *pUpsert,    /* ON CONFLICT clauses for upsert */
   419    421     const char *zStart, /* Start of SQL text */
   420    422     const char *zEnd    /* End of SQL text */
   421    423   ){
   422    424     TriggerStep *pTriggerStep;
   423    425   
   424    426     assert(pSelect != 0 || db->mallocFailed);
   425    427   
   426    428     pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName, zStart, zEnd);
   427    429     if( pTriggerStep ){
   428    430       pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE);
   429    431       pTriggerStep->pIdList = pColumn;
          432  +    pTriggerStep->pUpsert = pUpsert;
   430    433       pTriggerStep->orconf = orconf;
   431    434     }else{
          435  +    testcase( pColumn );
   432    436       sqlite3IdListDelete(db, pColumn);
          437  +    testcase( pUpsert );
          438  +    sqlite3UpsertDelete(db, pUpsert);
   433    439     }
   434    440     sqlite3SelectDelete(db, pSelect);
   435    441   
   436    442     return pTriggerStep;
   437    443   }
   438    444   
   439    445   /*
................................................................................
   742    748   
   743    749       switch( pStep->op ){
   744    750         case TK_UPDATE: {
   745    751           sqlite3Update(pParse, 
   746    752             targetSrcList(pParse, pStep),
   747    753             sqlite3ExprListDup(db, pStep->pExprList, 0), 
   748    754             sqlite3ExprDup(db, pStep->pWhere, 0), 
   749         -          pParse->eOrconf, 0, 0
          755  +          pParse->eOrconf, 0, 0, 0
   750    756           );
   751    757           break;
   752    758         }
   753    759         case TK_INSERT: {
   754    760           sqlite3Insert(pParse, 
   755    761             targetSrcList(pParse, pStep),
   756    762             sqlite3SelectDup(db, pStep->pSelect, 0), 
   757    763             sqlite3IdListDup(db, pStep->pIdList), 
   758         -          pParse->eOrconf
          764  +          pParse->eOrconf,
          765  +          sqlite3UpsertDup(db, pStep->pUpsert)
   759    766           );
   760    767           break;
   761    768         }
   762    769         case TK_DELETE: {
   763    770           sqlite3DeleteFrom(pParse, 
   764    771             targetSrcList(pParse, pStep),
   765    772             sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0

Changes to src/update.c.

    89     89   void sqlite3Update(
    90     90     Parse *pParse,         /* The parser context */
    91     91     SrcList *pTabList,     /* The table in which we should change things */
    92     92     ExprList *pChanges,    /* Things to be changed */
    93     93     Expr *pWhere,          /* The WHERE clause.  May be null */
    94     94     int onError,           /* How to handle constraint errors */
    95     95     ExprList *pOrderBy,    /* ORDER BY clause. May be null */
    96         -  Expr *pLimit           /* LIMIT clause. May be null */
           96  +  Expr *pLimit,          /* LIMIT clause. May be null */
           97  +  Upsert *pUpsert        /* ON CONFLICT clause, or null */
    97     98   ){
    98     99     int i, j;              /* Loop counters */
    99    100     Table *pTab;           /* The table to be updated */
   100    101     int addrTop = 0;       /* VDBE instruction address of the start of the loop */
   101    102     WhereInfo *pWInfo;     /* Information about the WHERE clause */
   102    103     Vdbe *v;               /* The virtual database engine */
   103    104     Index *pIdx;           /* For looping over indices */
................................................................................
   196    197     }
   197    198   
   198    199     /* Allocate a cursors for the main database table and for all indices.
   199    200     ** The index cursors might not be used, but if they are used they
   200    201     ** need to occur right after the database cursor.  So go ahead and
   201    202     ** allocate enough space, just in case.
   202    203     */
   203         -  pTabList->a[0].iCursor = iBaseCur = iDataCur = pParse->nTab++;
          204  +  iBaseCur = iDataCur = pParse->nTab++;
   204    205     iIdxCur = iDataCur+1;
   205    206     pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab);
          207  +  testcase( pPk!=0 && pPk!=pTab->pIndex );
   206    208     for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){
   207         -    if( IsPrimaryKeyIndex(pIdx) && pPk!=0 ){
          209  +    if( pPk==pIdx ){
   208    210         iDataCur = pParse->nTab;
   209         -      pTabList->a[0].iCursor = iDataCur;
   210    211       }
   211    212       pParse->nTab++;
   212    213     }
          214  +  if( pUpsert ){
          215  +    /* On an UPSERT, reuse the same cursors already opened by INSERT */
          216  +    iDataCur = pUpsert->iDataCur;
          217  +    iIdxCur = pUpsert->iIdxCur;
          218  +    pParse->nTab = iBaseCur;
          219  +  }
          220  +  pTabList->a[0].iCursor = iDataCur;
   213    221   
   214    222     /* Allocate space for aXRef[], aRegIdx[], and aToOpen[].  
   215    223     ** Initialize aXRef[] and aToOpen[] to their default values.
   216    224     */
   217    225     aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 );
   218    226     if( aXRef==0 ) goto update_cleanup;
   219    227     aRegIdx = aXRef+pTab->nCol;
................................................................................
   222    230     aToOpen[nIdx+1] = 0;
   223    231     for(i=0; i<pTab->nCol; i++) aXRef[i] = -1;
   224    232   
   225    233     /* Initialize the name-context */
   226    234     memset(&sNC, 0, sizeof(sNC));
   227    235     sNC.pParse = pParse;
   228    236     sNC.pSrcList = pTabList;
          237  +  sNC.uNC.pUpsert = pUpsert;
          238  +  sNC.ncFlags = NC_UUpsert;
   229    239   
   230    240     /* Resolve the column names in all the expressions of the
   231    241     ** of the UPDATE statement.  Also find the column index
   232    242     ** for each column to be updated in the pChanges array.  For each
   233    243     ** column to be updated, make sure we have authorization to change
   234    244     ** that column.
   235    245     */
................................................................................
   336    346       memset(aToOpen, 1, nIdx+1);
   337    347     }
   338    348   
   339    349     /* Begin generating code. */
   340    350     v = sqlite3GetVdbe(pParse);
   341    351     if( v==0 ) goto update_cleanup;
   342    352     if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
   343         -  sqlite3BeginWriteOperation(pParse, 1, iDb);
          353  +  sqlite3BeginWriteOperation(pParse, pTrigger || hasFK, iDb);
   344    354   
   345    355     /* Allocate required registers. */
   346    356     if( !IsVirtual(pTab) ){
   347    357       regRowSet = ++pParse->nMem;
   348    358       regOldRowid = regNewRowid = ++pParse->nMem;
   349    359       if( chngPk || pTrigger || hasFK ){
   350    360         regOld = pParse->nMem + 1;
................................................................................
   387    397     if( IsVirtual(pTab) ){
   388    398       updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef,
   389    399                          pWhere, onError);
   390    400       goto update_cleanup;
   391    401     }
   392    402   #endif
   393    403   
   394         -  /* Initialize the count of updated rows */
   395         -  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){
          404  +  /* Jump to labelBreak to abandon further processing of this UPDATE */
          405  +  labelContinue = labelBreak = sqlite3VdbeMakeLabel(v);
          406  +
          407  +  /* Not an UPSERT.  Normal processing.  Begin by
          408  +  ** initialize the count of updated rows */
          409  +  if( (db->flags&SQLITE_CountRows)!=0
          410  +   && !pParse->pTriggerTab
          411  +   && !pParse->nested
          412  +   && pUpsert==0
          413  +  ){
   396    414       regRowCount = ++pParse->nMem;
   397    415       sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
   398    416     }
   399    417   
   400    418     if( HasRowid(pTab) ){
   401    419       sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid);
   402    420     }else{
   403    421       assert( pPk!=0 );
   404    422       nPk = pPk->nKeyCol;
   405    423       iPk = pParse->nMem+1;
   406    424       pParse->nMem += nPk;
   407    425       regKey = ++pParse->nMem;
   408         -    iEph = pParse->nTab++;
   409         -
   410         -    sqlite3VdbeAddOp2(v, OP_Null, 0, iPk);
   411         -    addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
   412         -    sqlite3VdbeSetP4KeyInfo(pParse, pPk);
   413         -  }
   414         -
   415         -  /* Begin the database scan. 
   416         -  **
   417         -  ** Do not consider a single-pass strategy for a multi-row update if
   418         -  ** there are any triggers or foreign keys to process, or rows may
   419         -  ** be deleted as a result of REPLACE conflict handling. Any of these
   420         -  ** things might disturb a cursor being used to scan through the table
   421         -  ** or index, causing a single-pass approach to malfunction.  */
   422         -  flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
   423         -  if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
   424         -    flags |= WHERE_ONEPASS_MULTIROW;
   425         -  }
   426         -  pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);
   427         -  if( pWInfo==0 ) goto update_cleanup;
   428         -
   429         -  /* A one-pass strategy that might update more than one row may not
   430         -  ** be used if any column of the index used for the scan is being
   431         -  ** updated. Otherwise, if there is an index on "b", statements like
   432         -  ** the following could create an infinite loop:
   433         -  **
   434         -  **   UPDATE t1 SET b=b+1 WHERE b>?
   435         -  **
   436         -  ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
   437         -  ** strategy that uses an index for which one or more columns are being
   438         -  ** updated.  */
   439         -  eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
   440         -  if( eOnePass==ONEPASS_MULTI ){
   441         -    int iCur = aiCurOnePass[1];
   442         -    if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
   443         -      eOnePass = ONEPASS_OFF;
   444         -    }
   445         -    assert( iCur!=iDataCur || !HasRowid(pTab) );
   446         -  }
          426  +    if( pUpsert==0 ){
          427  +      iEph = pParse->nTab++;
          428  +        sqlite3VdbeAddOp3(v, OP_Null, 0, iPk, iPk+nPk-1);
          429  +      addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
          430  +      sqlite3VdbeSetP4KeyInfo(pParse, pPk);
          431  +    }
          432  +  }
          433  +  
          434  +  if( pUpsert ){
          435  +    /* If this is an UPSERT, then all cursors have already been opened by
          436  +    ** the outer INSERT and the data cursor should be pointing at the row
          437  +    ** that is to be updated.  So bypass the code that searches for the
          438  +    ** row(s) to be updated.
          439  +    */
          440  +    pWInfo = 0;
          441  +    eOnePass = ONEPASS_SINGLE;
          442  +    sqlite3ExprIfFalse(pParse, pWhere, labelBreak, SQLITE_JUMPIFNULL);
          443  +  }else{
          444  +    /* Begin the database scan. 
          445  +    **
          446  +    ** Do not consider a single-pass strategy for a multi-row update if
          447  +    ** there are any triggers or foreign keys to process, or rows may
          448  +    ** be deleted as a result of REPLACE conflict handling. Any of these
          449  +    ** things might disturb a cursor being used to scan through the table
          450  +    ** or index, causing a single-pass approach to malfunction.  */
          451  +    flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
          452  +    if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
          453  +      flags |= WHERE_ONEPASS_MULTIROW;
          454  +    }
          455  +    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);
          456  +    if( pWInfo==0 ) goto update_cleanup;
   447    457     
          458  +    /* A one-pass strategy that might update more than one row may not
          459  +    ** be used if any column of the index used for the scan is being
          460  +    ** updated. Otherwise, if there is an index on "b", statements like
          461  +    ** the following could create an infinite loop:
          462  +    **
          463  +    **   UPDATE t1 SET b=b+1 WHERE b>?
          464  +    **
          465  +    ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
          466  +    ** strategy that uses an index for which one or more columns are being
          467  +    ** updated.  */
          468  +    eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
          469  +    if( eOnePass!=ONEPASS_SINGLE ){
          470  +      sqlite3MultiWrite(pParse);
          471  +      if( eOnePass==ONEPASS_MULTI ){
          472  +        int iCur = aiCurOnePass[1];
          473  +        if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
          474  +          eOnePass = ONEPASS_OFF;
          475  +        }
          476  +        assert( iCur!=iDataCur || !HasRowid(pTab) );
          477  +      }
          478  +    }
          479  +  }
          480  +
   448    481     if( HasRowid(pTab) ){
   449    482       /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF
   450    483       ** mode, write the rowid into the FIFO. In either of the one-pass modes,
   451    484       ** leave it in register regOldRowid.  */
   452    485       sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid);
   453    486       if( eOnePass==ONEPASS_OFF ){
   454    487         sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid);
................................................................................
   460    493       ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table 
   461    494       ** is not required) and leave the PK fields in the array of registers.  */
   462    495       for(i=0; i<nPk; i++){
   463    496         assert( pPk->aiColumn[i]>=0 );
   464    497         sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i);
   465    498       }
   466    499       if( eOnePass ){
   467         -      sqlite3VdbeChangeToNoop(v, addrOpen);
          500  +      if( addrOpen ) sqlite3VdbeChangeToNoop(v, addrOpen);
   468    501         nKey = nPk;
   469    502         regKey = iPk;
   470    503       }else{
   471    504         sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey,
   472    505                           sqlite3IndexAffinityStr(db, pPk), nPk);
   473    506         sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk);
   474    507       }
   475    508     }
   476    509   
   477         -  if( eOnePass!=ONEPASS_MULTI ){
   478         -    sqlite3WhereEnd(pWInfo);
   479         -  }
   480         -
   481         -  labelBreak = sqlite3VdbeMakeLabel(v);
   482         -  if( !isView ){
   483         -    int addrOnce = 0;
   484         -
   485         -    /* Open every index that needs updating. */
          510  +  if( pUpsert==0 ){
          511  +    if( eOnePass!=ONEPASS_MULTI ){
          512  +      sqlite3WhereEnd(pWInfo);
          513  +    }
          514  +  
          515  +    if( !isView ){
          516  +      int addrOnce = 0;
          517  +  
          518  +      /* Open every index that needs updating. */
          519  +      if( eOnePass!=ONEPASS_OFF ){
          520  +        if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
          521  +        if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
          522  +      }
          523  +  
          524  +      if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
          525  +        addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
          526  +      }
          527  +      sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur,
          528  +                                 aToOpen, 0, 0);
          529  +      if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
          530  +    }
          531  +  
          532  +    /* Top of the update loop */
   486    533       if( eOnePass!=ONEPASS_OFF ){
   487         -      if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
   488         -      if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
   489         -    }
   490         -
   491         -    if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
   492         -      addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   493         -    }
   494         -    sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen,
   495         -                               0, 0);
   496         -    if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
   497         -  }
   498         -
   499         -  /* Top of the update loop */
   500         -  if( eOnePass!=ONEPASS_OFF ){
   501         -    if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
   502         -      assert( pPk );
   503         -      sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey);
   504         -      VdbeCoverageNeverTaken(v);
   505         -    }
   506         -    if( eOnePass==ONEPASS_SINGLE ){
   507         -      labelContinue = labelBreak;
   508         -    }else{
          534  +      if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
          535  +        assert( pPk );
          536  +        sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey,nKey);
          537  +        VdbeCoverageNeverTaken(v);
          538  +      }
          539  +      if( eOnePass!=ONEPASS_SINGLE ){
          540  +        labelContinue = sqlite3VdbeMakeLabel(v);
          541  +      }
          542  +      sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
          543  +      VdbeCoverageIf(v, pPk==0);
          544  +      VdbeCoverageIf(v, pPk!=0);
          545  +    }else if( pPk ){
   509    546         labelContinue = sqlite3VdbeMakeLabel(v);
          547  +      sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
          548  +      addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);
          549  +      sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0);
          550  +      VdbeCoverage(v);
          551  +    }else{
          552  +      labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet,labelBreak,
          553  +                               regOldRowid);
          554  +      VdbeCoverage(v);
          555  +      sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
          556  +      VdbeCoverage(v);
   510    557       }
   511         -    sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
   512         -    VdbeCoverageIf(v, pPk==0);
   513         -    VdbeCoverageIf(v, pPk!=0);
   514         -  }else if( pPk ){
   515         -    labelContinue = sqlite3VdbeMakeLabel(v);
   516         -    sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
   517         -    addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);
   518         -    sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0);
   519         -    VdbeCoverage(v);
   520         -  }else{
   521         -    labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, labelBreak,
   522         -                             regOldRowid);
   523         -    VdbeCoverage(v);
   524         -    sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
   525         -    VdbeCoverage(v);
   526    558     }
   527    559   
   528         -  /* If the record number will change, set register regNewRowid to
   529         -  ** contain the new value. If the record number is not being modified,
          560  +  /* If the rowid value will change, set register regNewRowid to
          561  +  ** contain the new value. If the rowid is not being modified,
   530    562     ** then regNewRowid is the same register as regOldRowid, which is
   531    563     ** already populated.  */
   532    564     assert( chngKey || pTrigger || hasFK || regOldRowid==regNewRowid );
   533    565     if( chngRowid ){
   534    566       sqlite3ExprCode(pParse, pRowidExpr, regNewRowid);
   535    567       sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v);
   536    568     }
................................................................................
   633    665     if( !isView ){
   634    666       int addr1 = 0;        /* Address of jump instruction */
   635    667   
   636    668       /* Do constraint checks. */
   637    669       assert( regOldRowid>0 );
   638    670       sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
   639    671           regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace,
   640         -        aXRef);
          672  +        aXRef, 0);
   641    673   
   642    674       /* Do FK constraint checks. */
   643    675       if( hasFK ){
   644    676         sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey);
   645    677       }
   646    678   
   647    679       /* Delete the index entries associated with the current record.  */
................................................................................
   703    735       if( hasFK ){
   704    736         sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey);
   705    737       }
   706    738     }
   707    739   
   708    740     /* Increment the row counter 
   709    741     */
   710         -  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab){
          742  +  if( regRowCount ){
   711    743       sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1);
   712    744     }
   713    745   
   714    746     sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, 
   715    747         TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue);
   716    748   
   717    749     /* Repeat the above with the next record to be updated, until
................................................................................
   730    762     }
   731    763     sqlite3VdbeResolveLabel(v, labelBreak);
   732    764   
   733    765     /* Update the sqlite_sequence table by storing the content of the
   734    766     ** maximum rowid counter values recorded while inserting into
   735    767     ** autoincrement tables.
   736    768     */
   737         -  if( pParse->nested==0 && pParse->pTriggerTab==0 ){
          769  +  if( pParse->nested==0 && pParse->pTriggerTab==0 && pUpsert==0 ){
   738    770       sqlite3AutoincrementEnd(pParse);
   739    771     }
   740    772   
   741    773     /*
   742         -  ** Return the number of rows that were changed. If this routine is 
   743         -  ** generating code because of a call to sqlite3NestedParse(), do not
   744         -  ** invoke the callback function.
          774  +  ** Return the number of rows that were changed, if we are tracking
          775  +  ** that information.
   745    776     */
   746         -  if( (db->flags&SQLITE_CountRows) && !pParse->pTriggerTab && !pParse->nested ){
          777  +  if( regRowCount ){
   747    778       sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1);
   748    779       sqlite3VdbeSetNumCols(v, 1);
   749    780       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC);
   750    781     }
   751    782   
   752    783   update_cleanup:
   753    784     sqlite3AuthContextPop(&sContext);
................................................................................
   860    891       sqlite3VdbeAddOp2(v, OP_SCopy, regArg+2+iPk, regArg+1);
   861    892     }
   862    893   
   863    894     bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy);
   864    895   
   865    896     if( bOnePass ){
   866    897       /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded
   867         -    ** above. Also, if this is a top-level parse (not a trigger), clear the
   868         -    ** multi-write flag so that the VM does not open a statement journal */
          898  +    ** above. */
   869    899       sqlite3VdbeChangeToNoop(v, addr);
   870         -    if( sqlite3IsToplevel(pParse) ){
   871         -      pParse->isMultiWrite = 0;
   872         -    }
   873    900     }else{
   874    901       /* Create a record from the argument register contents and insert it into
   875    902       ** the ephemeral table. */
          903  +    sqlite3MultiWrite(pParse);
   876    904       sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec);
   877    905   #ifdef SQLITE_DEBUG
   878    906       /* Signal an assert() within OP_MakeRecord that it is allowed to
   879    907       ** accept no-change records with serial_type 10 */
   880    908       sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC);
   881    909   #endif
   882    910       sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid);

Added src/upsert.c.

            1  +/*
            2  +** 2018-04-12
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** This file contains code to implement various aspects of UPSERT
           13  +** processing and handling of the Upsert object.
           14  +*/
           15  +#include "sqliteInt.h"
           16  +
           17  +#ifndef SQLITE_OMIT_UPSERT
           18  +/*
           19  +** Free a list of Upsert objects
           20  +*/
           21  +void sqlite3UpsertDelete(sqlite3 *db, Upsert *p){
           22  +  if( p ){
           23  +    sqlite3ExprListDelete(db, p->pUpsertTarget);
           24  +    sqlite3ExprDelete(db, p->pUpsertTargetWhere);
           25  +    sqlite3ExprListDelete(db, p->pUpsertSet);
           26  +    sqlite3ExprDelete(db, p->pUpsertWhere);
           27  +    sqlite3DbFree(db, p);
           28  +  }
           29  +}
           30  +
           31  +/*
           32  +** Duplicate an Upsert object.
           33  +*/
           34  +Upsert *sqlite3UpsertDup(sqlite3 *db, Upsert *p){
           35  +  if( p==0 ) return 0;
           36  +  return sqlite3UpsertNew(db,
           37  +           sqlite3ExprListDup(db, p->pUpsertTarget, 0),
           38  +           sqlite3ExprDup(db, p->pUpsertTargetWhere, 0),
           39  +           sqlite3ExprListDup(db, p->pUpsertSet, 0),
           40  +           sqlite3ExprDup(db, p->pUpsertWhere, 0)
           41  +         );
           42  +}
           43  +
           44  +/*
           45  +** Create a new Upsert object.
           46  +*/
           47  +Upsert *sqlite3UpsertNew(
           48  +  sqlite3 *db,           /* Determines which memory allocator to use */
           49  +  ExprList *pTarget,     /* Target argument to ON CONFLICT, or NULL */
           50  +  Expr *pTargetWhere,    /* Optional WHERE clause on the target */
           51  +  ExprList *pSet,        /* UPDATE columns, or NULL for a DO NOTHING */
           52  +  Expr *pWhere           /* WHERE clause for the ON CONFLICT UPDATE */
           53  +){
           54  +  Upsert *pNew;
           55  +  pNew = sqlite3DbMallocRaw(db, sizeof(Upsert));
           56  +  if( pNew==0 ){
           57  +    sqlite3ExprListDelete(db, pTarget);
           58  +    sqlite3ExprDelete(db, pTargetWhere);
           59  +    sqlite3ExprListDelete(db, pSet);
           60  +    sqlite3ExprDelete(db, pWhere);
           61  +    return 0;
           62  +  }else{
           63  +    pNew->pUpsertTarget = pTarget;
           64  +    pNew->pUpsertTargetWhere = pTargetWhere;
           65  +    pNew->pUpsertSet = pSet;
           66  +    pNew->pUpsertWhere = pWhere;
           67  +    pNew->pUpsertIdx = 0;
           68  +  }
           69  +  return pNew;
           70  +}
           71  +
           72  +/*
           73  +** Analyze the ON CONFLICT clause described by pUpsert.  Resolve all
           74  +** symbols in the conflict-target.
           75  +**
           76  +** Return SQLITE_OK if everything works, or an error code is something
           77  +** is wrong.
           78  +*/
           79  +int sqlite3UpsertAnalyzeTarget(
           80  +  Parse *pParse,     /* The parsing context */
           81  +  SrcList *pTabList, /* Table into which we are inserting */
           82  +  Upsert *pUpsert    /* The ON CONFLICT clauses */
           83  +){
           84  +  Table *pTab;            /* That table into which we are inserting */
           85  +  int rc;                 /* Result code */
           86  +  int iCursor;            /* Cursor used by pTab */
           87  +  Index *pIdx;            /* One of the indexes of pTab */
           88  +  ExprList *pTarget;      /* The conflict-target clause */
           89  +  Expr *pTerm;            /* One term of the conflict-target clause */
           90  +  NameContext sNC;        /* Context for resolving symbolic names */
           91  +  Expr sCol[2];           /* Index column converted into an Expr */
           92  +
           93  +  assert( pTabList->nSrc==1 );
           94  +  assert( pTabList->a[0].pTab!=0 );
           95  +  assert( pUpsert!=0 );
           96  +  assert( pUpsert->pUpsertTarget!=0 );
           97  +
           98  +  /* Resolve all symbolic names in the conflict-target clause, which
           99  +  ** includes both the list of columns and the optional partial-index
          100  +  ** WHERE clause.
          101  +  */
          102  +  memset(&sNC, 0, sizeof(sNC));
          103  +  sNC.pParse = pParse;
          104  +  sNC.pSrcList = pTabList;
          105  +  rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget);
          106  +  if( rc ) return rc;
          107  +  rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere);
          108  +  if( rc ) return rc;
          109  +
          110  +  /* Check to see if the conflict target matches the rowid. */  
          111  +  pTab = pTabList->a[0].pTab;
          112  +  pTarget = pUpsert->pUpsertTarget;
          113  +  iCursor = pTabList->a[0].iCursor;
          114  +  if( HasRowid(pTab) 
          115  +   && pTarget->nExpr==1
          116  +   && (pTerm = pTarget->a[0].pExpr)->op==TK_COLUMN
          117  +   && pTerm->iColumn==XN_ROWID
          118  +  ){
          119  +    /* The conflict-target is the rowid of the primary table */
          120  +    assert( pUpsert->pUpsertIdx==0 );
          121  +    return SQLITE_OK;
          122  +  }
          123  +
          124  +  /* Initialize sCol[0..1] to be an expression parse tree for a
          125  +  ** single column of an index.  The sCol[0] node will be the TK_COLLATE
          126  +  ** operator and sCol[1] will be the TK_COLUMN operator.  Code below
          127  +  ** will populate the specific collation and column number values
          128  +  ** prior to comparing against the conflict-target expression.
          129  +  */
          130  +  memset(sCol, 0, sizeof(sCol));
          131  +  sCol[0].op = TK_COLLATE;
          132  +  sCol[0].pLeft = &sCol[1];
          133  +  sCol[1].op = TK_COLUMN;
          134  +  sCol[1].iTable = pTabList->a[0].iCursor;
          135  +
          136  +  /* Check for matches against other indexes */
          137  +  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
          138  +    int ii, jj, nn;
          139  +    if( !IsUniqueIndex(pIdx) ) continue;
          140  +    if( pTarget->nExpr!=pIdx->nKeyCol ) continue;
          141  +    if( pIdx->pPartIdxWhere ){
          142  +      if( pUpsert->pUpsertTargetWhere==0 ) continue;
          143  +      if( sqlite3ExprCompare(pParse, pUpsert->pUpsertTargetWhere,
          144  +                             pIdx->pPartIdxWhere, iCursor)!=0 ){
          145  +        continue;
          146  +      }
          147  +    }
          148  +    nn = pIdx->nKeyCol;
          149  +    for(ii=0; ii<nn; ii++){
          150  +      Expr *pExpr;
          151  +      sCol[0].u.zToken = (char*)pIdx->azColl[ii];
          152  +      if( pIdx->aiColumn[ii]==XN_EXPR ){
          153  +        assert( pIdx->aColExpr!=0 );
          154  +        assert( pIdx->aColExpr->nExpr>ii );
          155  +        pExpr = pIdx->aColExpr->a[ii].pExpr;
          156  +        if( pExpr->op!=TK_COLLATE ){
          157  +          sCol[0].pLeft = pExpr;
          158  +          pExpr = &sCol[0];
          159  +        }
          160  +      }else{
          161  +        sCol[0].pLeft = &sCol[1];
          162  +        sCol[1].iColumn = pIdx->aiColumn[ii];
          163  +        pExpr = &sCol[0];
          164  +      }
          165  +      for(jj=0; jj<nn; jj++){
          166  +        if( sqlite3ExprCompare(pParse, pTarget->a[jj].pExpr, pExpr,iCursor)<2 ){
          167  +          break;  /* Column ii of the index matches column jj of target */
          168  +        }
          169  +      }
          170  +      if( jj>=nn ){
          171  +        /* The target contains no match for column jj of the index */
          172  +        break;
          173  +      }
          174  +    }
          175  +    if( ii<nn ){
          176  +      /* Column ii of the index did not match any term of the conflict target.
          177  +      ** Continue the search with the next index. */
          178  +      continue;
          179  +    }
          180  +    pUpsert->pUpsertIdx = pIdx;
          181  +    return SQLITE_OK;
          182  +  }
          183  +  sqlite3ErrorMsg(pParse, "ON CONFLICT clause does not match any "
          184  +                          "PRIMARY KEY or UNIQUE constraint");
          185  +  return SQLITE_ERROR;
          186  +}
          187  +
          188  +/*
          189  +** Generate bytecode that does an UPDATE as part of an upsert.
          190  +**
          191  +** If pIdx is NULL, then the UNIQUE constraint that failed was the IPK.
          192  +** In this case parameter iCur is a cursor open on the table b-tree that
          193  +** currently points to the conflicting table row. Otherwise, if pIdx
          194  +** is not NULL, then pIdx is the constraint that failed and iCur is a
          195  +** cursor points to the conflicting row.
          196  +*/
          197  +void sqlite3UpsertDoUpdate(
          198  +  Parse *pParse,        /* The parsing and code-generating context */
          199  +  Upsert *pUpsert,      /* The ON CONFLICT clause for the upsert */
          200  +  Table *pTab,          /* The table being updated */
          201  +  Index *pIdx,          /* The UNIQUE constraint that failed */
          202  +  int iCur              /* Cursor for pIdx (or pTab if pIdx==NULL) */
          203  +){
          204  +  Vdbe *v = pParse->pVdbe;
          205  +  sqlite3 *db = pParse->db;
          206  +  SrcList *pSrc;            /* FROM clause for the UPDATE */
          207  +  int iDataCur = pUpsert->iDataCur;
          208  +
          209  +  assert( v!=0 );
          210  +  VdbeNoopComment((v, "Begin DO UPDATE of UPSERT"));
          211  +  if( pIdx && iCur!=iDataCur ){
          212  +    if( HasRowid(pTab) ){
          213  +      int regRowid = sqlite3GetTempReg(pParse);
          214  +      sqlite3VdbeAddOp2(v, OP_IdxRowid, iCur, regRowid);
          215  +      sqlite3VdbeAddOp3(v, OP_SeekRowid, iDataCur, 0, regRowid);
          216  +      VdbeCoverage(v);
          217  +      sqlite3ReleaseTempReg(pParse, regRowid);
          218  +    }else{
          219  +      Index *pPk = sqlite3PrimaryKeyIndex(pTab);
          220  +      int nPk = pPk->nKeyCol;
          221  +      int iPk = pParse->nMem+1;
          222  +      int i;
          223  +      pParse->nMem += nPk;
          224  +      for(i=0; i<nPk; i++){
          225  +        int k;
          226  +        assert( pPk->aiColumn[i]>=0 );
          227  +        k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]);
          228  +        sqlite3VdbeAddOp3(v, OP_Column, iCur, k, iPk+i);
          229  +        VdbeComment((v, "%s.%s", pIdx->zName,
          230  +                    pTab->aCol[pPk->aiColumn[i]].zName));
          231  +      }
          232  +      i = sqlite3VdbeAddOp4Int(v, OP_Found, iDataCur, 0, iPk, nPk);
          233  +      VdbeCoverage(v);
          234  +      sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CORRUPT, OE_Abort, 0, 
          235  +            "corrupt database", P4_STATIC);
          236  +      sqlite3VdbeJumpHere(v, i);
          237  +    }
          238  +  }
          239  +  /* pUpsert does not own pUpsertSrc - the outer INSERT statement does.  So
          240  +  ** we have to make a copy before passing it down into sqlite3Update() */
          241  +  pSrc = sqlite3SrcListDup(db, pUpsert->pUpsertSrc, 0);
          242  +  sqlite3Update(pParse, pSrc, pUpsert->pUpsertSet,
          243  +      pUpsert->pUpsertWhere, OE_Abort, 0, 0, pUpsert);
          244  +  pUpsert->pUpsertSet = 0;    /* Will have been deleted by sqlite3Update() */
          245  +  pUpsert->pUpsertWhere = 0;  /* Will have been deleted by sqlite3Update() */
          246  +  VdbeNoopComment((v, "End DO UPDATE of UPSERT"));
          247  +}
          248  +
          249  +#endif /* SQLITE_OMIT_UPSERT */

Changes to src/vdbe.h.

   216    216   void sqlite3VdbeRunOnlyOnce(Vdbe*);
   217    217   void sqlite3VdbeReusable(Vdbe*);
   218    218   void sqlite3VdbeDelete(Vdbe*);
   219    219   void sqlite3VdbeClearObject(sqlite3*,Vdbe*);
   220    220   void sqlite3VdbeMakeReady(Vdbe*,Parse*);
   221    221   int sqlite3VdbeFinalize(Vdbe*);
   222    222   void sqlite3VdbeResolveLabel(Vdbe*, int);
          223  +#ifdef SQLITE_COVERAGE_TEST
          224  +  int sqlite3VdbeLabelHasBeenResolved(Vdbe*,int);
          225  +#endif
   223    226   int sqlite3VdbeCurrentAddr(Vdbe*);
   224    227   #ifdef SQLITE_DEBUG
   225    228     int sqlite3VdbeAssertMayAbort(Vdbe *, int);
   226    229   #endif
   227    230   void sqlite3VdbeResetStepResult(Vdbe*);
   228    231   void sqlite3VdbeRewind(Vdbe*);
   229    232   int sqlite3VdbeReset(Vdbe*);

Changes to src/vdbeaux.c.

   388    388   void sqlite3VdbeResolveLabel(Vdbe *v, int x){
   389    389     Parse *p = v->pParse;
   390    390     int j = ADDR(x);
   391    391     assert( v->magic==VDBE_MAGIC_INIT );
   392    392     assert( j<p->nLabel );
   393    393     assert( j>=0 );
   394    394     if( p->aLabel ){
          395  +#ifdef SQLITE_DEBUG
          396  +    if( p->db->flags & SQLITE_VdbeAddopTrace ){
          397  +      printf("RESOLVE LABEL %d to %d\n", x, v->nOp);
          398  +    }
          399  +#endif
          400  +    assert( p->aLabel[j]==(-1) ); /* Labels may only be resolved once */
   395    401       p->aLabel[j] = v->nOp;
   396    402     }
   397    403   }
   398    404   
          405  +#ifdef SQLITE_COVERAGE_TEST
          406  +/*
          407  +** Return TRUE if and only if the label x has already been resolved.
          408  +** Return FALSE (zero) if label x is still unresolved.
          409  +**
          410  +** This routine is only used inside of testcase() macros, and so it
          411  +** only exists when measuring test coverage.
          412  +*/
          413  +int sqlite3VdbeLabelHasBeenResolved(Vdbe *v, int x){
          414  +  return v->pParse->aLabel && v->pParse->aLabel[ADDR(x)]>=0;
          415  +}
          416  +#endif /* SQLITE_COVERAGE_TEST */
          417  +
   399    418   /*
   400    419   ** Mark the VDBE as one that can only be run one time.
   401    420   */
   402    421   void sqlite3VdbeRunOnlyOnce(Vdbe *p){
   403    422     p->runOnlyOnce = 1;
   404    423   }
   405    424   
................................................................................
  1622   1641   ** running the code, it invokes the callback once for each instruction.
  1623   1642   ** This feature is used to implement "EXPLAIN".
  1624   1643   **
  1625   1644   ** When p->explain==1, each instruction is listed.  When
  1626   1645   ** p->explain==2, only OP_Explain instructions are listed and these
  1627   1646   ** are shown in a different format.  p->explain==2 is used to implement
  1628   1647   ** EXPLAIN QUERY PLAN.
         1648  +** 2018-04-24:  In p->explain==2 mode, the OP_Init opcodes of triggers
         1649  +** are also shown, so that the boundaries between the main program and
         1650  +** each trigger are clear.
  1629   1651   **
  1630   1652   ** When p->explain==1, first the main program is listed, then each of
  1631   1653   ** the trigger subprograms are listed one by one.
  1632   1654   */
  1633   1655   int sqlite3VdbeList(
  1634   1656     Vdbe *p                   /* The VDBE */
  1635   1657   ){
................................................................................
  1684   1706         apSub = (SubProgram **)pSub->z;
  1685   1707       }
  1686   1708       for(i=0; i<nSub; i++){
  1687   1709         nRow += apSub[i]->nOp;
  1688   1710       }
  1689   1711     }
  1690   1712   
  1691         -  do{
         1713  +  while(1){  /* Loop exits via break */
  1692   1714       i = p->pc++;
  1693   1715       if( i>=nRow ){
  1694   1716         p->rc = SQLITE_OK;
  1695   1717         rc = SQLITE_DONE;
  1696   1718         break;
  1697   1719       }
  1698   1720       if( i<p->nOp ){
................................................................................
  1730   1752           apSub = (SubProgram **)pSub->z;
  1731   1753           apSub[nSub++] = pOp->p4.pProgram;
  1732   1754           pSub->flags |= MEM_Blob;
  1733   1755           pSub->n = nSub*sizeof(SubProgram*);
  1734   1756           nRow += pOp->p4.pProgram->nOp;
  1735   1757         }
  1736   1758       }
  1737         -  }while( p->explain==2 && pOp->opcode!=OP_Explain );
         1759  +    if( p->explain<2 ) break;
         1760  +    if( pOp->opcode==OP_Explain ) break;
         1761  +    if( pOp->opcode==OP_Init && p->pc>1 ) break;
         1762  +  }
  1738   1763   
  1739   1764     if( rc==SQLITE_OK ){
  1740   1765       if( db->u1.isInterrupted ){
  1741   1766         p->rc = SQLITE_INTERRUPT;
  1742   1767         rc = SQLITE_ERROR;
  1743   1768         sqlite3VdbeError(p, sqlite3ErrStr(p->rc));
  1744   1769       }else{

Changes to src/vdbemem.c.

  1474   1474       zVal = &pExpr->u.zToken[2];
  1475   1475       nVal = sqlite3Strlen30(zVal)-1;
  1476   1476       assert( zVal[nVal]=='\'' );
  1477   1477       sqlite3VdbeMemSetStr(pVal, sqlite3HexToBlob(db, zVal, nVal), nVal/2,
  1478   1478                            0, SQLITE_DYNAMIC);
  1479   1479     }
  1480   1480   #endif
  1481         -
  1482   1481   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1483   1482     else if( op==TK_FUNCTION && pCtx!=0 ){
  1484   1483       rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx);
  1485   1484     }
  1486   1485   #endif
         1486  +  else if( op==TK_TRUEFALSE ){
         1487  +     pVal = valueNew(db, pCtx);
         1488  +     pVal->flags = MEM_Int;
         1489  +     pVal->u.i = pExpr->u.zToken[4]==0;
         1490  +  }
  1487   1491   
  1488   1492     *ppVal = pVal;
  1489   1493     return rc;
  1490   1494   
  1491   1495   no_mem:
  1492   1496   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1493   1497     if( pCtx==0 || pCtx->pParse->nErr==0 )

Changes to src/where.c.

  2369   2369     int rc = SQLITE_OK;             /* Return code */
  2370   2370     LogEst rSize;                   /* Number of rows in the table */
  2371   2371     LogEst rLogSize;                /* Logarithm of table size */
  2372   2372     WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
  2373   2373   
  2374   2374     pNew = pBuilder->pNew;
  2375   2375     if( db->mallocFailed ) return SQLITE_NOMEM_BKPT;
  2376         -  WHERETRACE(0x800, ("BEGIN addBtreeIdx(%s), nEq=%d\n",
  2377         -                     pProbe->zName, pNew->u.btree.nEq));
         2376  +  WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d\n",
         2377  +                     pProbe->pTable->zName,pProbe->zName, pNew->u.btree.nEq));
  2378   2378   
  2379   2379     assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
  2380   2380     assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
  2381   2381     if( pNew->wsFlags & WHERE_BTM_LIMIT ){
  2382   2382       opMask = WO_LT|WO_LE;
  2383   2383     }else{
  2384   2384       assert( pNew->u.btree.nBtm==0 );
................................................................................
  2416   2416       }
  2417   2417       if( pTerm->prereqRight & pNew->maskSelf ) continue;
  2418   2418   
  2419   2419       /* Do not allow the upper bound of a LIKE optimization range constraint
  2420   2420       ** to mix with a lower range bound from some other source */
  2421   2421       if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
  2422   2422   
  2423         -    /* Do not allow IS constraints from the WHERE clause to be used by the
         2423  +    /* Do not allow constraints from the WHERE clause to be used by the
  2424   2424       ** right table of a LEFT JOIN.  Only constraints in the ON clause are
  2425   2425       ** allowed */
  2426   2426       if( (pSrc->fg.jointype & JT_LEFT)!=0
  2427   2427        && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
  2428         -     && (eOp & (WO_IS|WO_ISNULL))!=0
  2429   2428       ){
  2430         -      testcase( eOp & WO_IS );
  2431         -      testcase( eOp & WO_ISNULL );
  2432   2429         continue;
  2433   2430       }
  2434   2431   
  2435   2432       if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
  2436   2433         pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE;
  2437   2434       }else{
  2438   2435         pBuilder->bldFlags |= SQLITE_BLDF_INDEXED;
................................................................................
  2656   2653       whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
  2657   2654       pNew->nOut = saved_nOut;
  2658   2655       pNew->u.btree.nEq = saved_nEq;
  2659   2656       pNew->nSkip = saved_nSkip;
  2660   2657       pNew->wsFlags = saved_wsFlags;
  2661   2658     }
  2662   2659   
  2663         -  WHERETRACE(0x800, ("END addBtreeIdx(%s), nEq=%d, rc=%d\n",
  2664         -                      pProbe->zName, saved_nEq, rc));
         2660  +  WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
         2661  +                      pProbe->pTable->zName, pProbe->zName, saved_nEq, rc));
  2665   2662     return rc;
  2666   2663   }
  2667   2664   
  2668   2665   /*
  2669   2666   ** Return True if it is possible that pIndex might be useful in
  2670   2667   ** implementing the ORDER BY clause in pBuilder.
  2671   2668   **
................................................................................
  3095   3092         int j = pIdxCons->iTermOffset;
  3096   3093         if( iTerm>=nConstraint
  3097   3094          || j<0
  3098   3095          || j>=pWC->nTerm
  3099   3096          || pNew->aLTerm[iTerm]!=0
  3100   3097          || pIdxCons->usable==0
  3101   3098         ){
  3102         -        rc = SQLITE_ERROR;
  3103   3099           sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
  3104         -        return rc;
         3100  +        testcase( pIdxInfo->needToFreeIdxStr );
         3101  +        return SQLITE_ERROR;
  3105   3102         }
  3106   3103         testcase( iTerm==nConstraint-1 );
  3107   3104         testcase( j==0 );
  3108   3105         testcase( j==pWC->nTerm-1 );
  3109   3106         pTerm = &pWC->a[j];
  3110   3107         pNew->prereq |= pTerm->prereqRight;
  3111   3108         assert( iTerm<pNew->nLSlot );
................................................................................
  3125   3122           *pbIn = 1; assert( (mExclude & WO_IN)==0 );
  3126   3123         }
  3127   3124       }
  3128   3125     }
  3129   3126     pNew->u.vtab.omitMask &= ~mNoOmit;
  3130   3127   
  3131   3128     pNew->nLTerm = mxTerm+1;
         3129  +  for(i=0; i<=mxTerm; i++){
         3130  +    if( pNew->aLTerm[i]==0 ){
         3131  +      /* The non-zero argvIdx values must be contiguous.  Raise an
         3132  +      ** error if they are not */
         3133  +      sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
         3134  +      testcase( pIdxInfo->needToFreeIdxStr );
         3135  +      return SQLITE_ERROR;
         3136  +    }
         3137  +  }
  3132   3138     assert( pNew->nLTerm<=pNew->nLSlot );
  3133   3139     pNew->u.vtab.idxNum = pIdxInfo->idxNum;
  3134   3140     pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
  3135   3141     pIdxInfo->needToFreeIdxStr = 0;
  3136   3142     pNew->u.vtab.idxStr = pIdxInfo->idxStr;
  3137   3143     pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
  3138   3144         pIdxInfo->nOrderBy : 0);
................................................................................
  3240   3246     nConstraint = p->nConstraint;
  3241   3247     if( whereLoopResize(pParse->db, pNew, nConstraint) ){
  3242   3248       sqlite3DbFree(pParse->db, p);
  3243   3249       return SQLITE_NOMEM_BKPT;
  3244   3250     }
  3245   3251   
  3246   3252     /* First call xBestIndex() with all constraints usable. */
         3253  +  WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName));
  3247   3254     WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
  3248   3255     rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
  3249   3256   
  3250   3257     /* If the call to xBestIndex() with all terms enabled produced a plan
  3251   3258     ** that does not require any source tables (IOW: a plan with mBest==0),
  3252   3259     ** then there is no point in making any further calls to xBestIndex() 
  3253   3260     ** since they will all return the same result (if the xBestIndex()
................................................................................
  3315   3322         rc = whereLoopAddVirtualOne(
  3316   3323             pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn);
  3317   3324       }
  3318   3325     }
  3319   3326   
  3320   3327     if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
  3321   3328     sqlite3DbFreeNN(pParse->db, p);
         3329  +  WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc));
  3322   3330     return rc;
  3323   3331   }
  3324   3332   #endif /* SQLITE_OMIT_VIRTUALTABLE */
  3325   3333   
  3326   3334   /*
  3327   3335   ** Add WhereLoop entries to handle OR terms.  This works for either
  3328   3336   ** btrees or virtual tables.

Changes to src/whereInt.h.

    15     15   ** a separate source file for easier editing.
    16     16   */
    17     17   
    18     18   /*
    19     19   ** Trace output macros
    20     20   */
    21     21   #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG)
    22         -/***/ int sqlite3WhereTrace;
           22  +/***/ extern int sqlite3WhereTrace;
    23     23   #endif
    24     24   #if defined(SQLITE_DEBUG) \
    25     25       && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE))
    26     26   # define WHERETRACE(K,X)  if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X
    27     27   # define WHERETRACE_ENABLED 1
    28     28   #else
    29     29   # define WHERETRACE(K,X)

Changes to src/wherecode.c.

  1211   1211     addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
  1212   1212     addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v);
  1213   1213   
  1214   1214     /* If this is the right table of a LEFT OUTER JOIN, allocate and
  1215   1215     ** initialize a memory cell that records if this table matches any
  1216   1216     ** row of the left table of the join.
  1217   1217     */
         1218  +  assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
         1219  +       || pLevel->iFrom>0 || (pTabItem[0].fg.jointype & JT_LEFT)==0
         1220  +  );
  1218   1221     if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){
  1219   1222       pLevel->iLeftJoin = ++pParse->nMem;
  1220   1223       sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin);
  1221   1224       VdbeComment((v, "init LEFT JOIN no-match flag"));
  1222   1225     }
  1223   1226   
  1224   1227     /* Compute a safe address to jump to if we discover that the table for
................................................................................
  1744   1747         sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont,
  1745   1748                              iRowidReg, pPk->nKeyCol); VdbeCoverage(v);
  1746   1749       }
  1747   1750   
  1748   1751       /* If pIdx is an index on one or more expressions, then look through
  1749   1752       ** all the expressions in pWInfo and try to transform matching expressions
  1750   1753       ** into reference to index columns.
         1754  +    **
         1755  +    ** Do not do this for the RHS of a LEFT JOIN. This is because the 
         1756  +    ** expression may be evaluated after OP_NullRow has been executed on
         1757  +    ** the cursor. In this case it is important to do the full evaluation,
         1758  +    ** as the result of the expression may not be NULL, even if all table
         1759  +    ** column values are.  https://www.sqlite.org/src/info/7fa8049685b50b5a
  1751   1760       */
  1752         -    whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
  1753         -
         1761  +    if( pLevel->iLeftJoin==0 ){
         1762  +      whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
         1763  +    }
  1754   1764   
  1755   1765       /* Record the instruction used to terminate the loop. */
  1756   1766       if( pLoop->wsFlags & WHERE_ONEROW ){
  1757   1767         pLevel->op = OP_Noop;
  1758   1768       }else if( bRev ){
  1759   1769         pLevel->op = OP_Prev;
  1760   1770       }else{
................................................................................
  1902   1912       ** See ticket http://www.sqlite.org/src/info/f2369304e4
  1903   1913       */
  1904   1914       if( pWC->nTerm>1 ){
  1905   1915         int iTerm;
  1906   1916         for(iTerm=0; iTerm<pWC->nTerm; iTerm++){
  1907   1917           Expr *pExpr = pWC->a[iTerm].pExpr;
  1908   1918           if( &pWC->a[iTerm] == pTerm ) continue;
  1909         -        if( ExprHasProperty(pExpr, EP_FromJoin) ) continue;
  1910   1919           testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL );
  1911   1920           testcase( pWC->a[iTerm].wtFlags & TERM_CODED );
  1912   1921           if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue;
  1913   1922           if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue;
  1914   1923           testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO );
  1915   1924           pExpr = sqlite3ExprDup(db, pExpr, 0);
  1916   1925           pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr);
................................................................................
  1927   1936       wctrlFlags =  WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
  1928   1937       for(ii=0; ii<pOrWc->nTerm; ii++){
  1929   1938         WhereTerm *pOrTerm = &pOrWc->a[ii];
  1930   1939         if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){
  1931   1940           WhereInfo *pSubWInfo;           /* Info for single OR-term scan */
  1932   1941           Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */
  1933   1942           int jmp1 = 0;                   /* Address of jump operation */
  1934         -        if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){
         1943  +        assert( (pTabItem[0].fg.jointype & JT_LEFT)==0 
         1944  +             || ExprHasProperty(pOrExpr, EP_FromJoin) 
         1945  +        );
         1946  +        if( pAndExpr ){
  1935   1947             pAndExpr->pLeft = pOrExpr;
  1936   1948             pOrExpr = pAndExpr;
  1937   1949           }
  1938   1950           /* Loop through table entries that match term pOrTerm. */
  1939   1951           WHERETRACE(0xffff, ("Subplan for OR-clause:\n"));
  1940   1952           pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0,
  1941   1953                                         wctrlFlags, iCovCur);
................................................................................
  2110   2122           testcase( pWInfo->untestedTerms==0
  2111   2123               && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
  2112   2124           pWInfo->untestedTerms = 1;
  2113   2125           continue;
  2114   2126         }
  2115   2127         pE = pTerm->pExpr;
  2116   2128         assert( pE!=0 );
  2117         -      if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){
         2129  +      if( (pTabItem->fg.jointype&JT_LEFT) && !ExprHasProperty(pE,EP_FromJoin) ){
  2118   2130           continue;
  2119   2131         }
  2120   2132         
  2121   2133         if( iLoop==1 && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){
  2122   2134           iNext = 2;
  2123   2135           continue;
  2124   2136         }
  2125   2137         if( iLoop<3 && (pTerm->wtFlags & TERM_VARSELECT) ){
  2126   2138           if( iNext==0 ) iNext = 3;
  2127   2139           continue;
  2128   2140         }
  2129   2141   
  2130         -      if( pTerm->wtFlags & TERM_LIKECOND ){
         2142  +      if( (pTerm->wtFlags & TERM_LIKECOND)!=0 ){
  2131   2143           /* If the TERM_LIKECOND flag is set, that means that the range search
  2132   2144           ** is sufficient to guarantee that the LIKE operator is true, so we
  2133   2145           ** can skip the call to the like(A,B) function.  But this only works
  2134   2146           ** for strings.  So do not skip the call to the function on the pass
  2135   2147           ** that compares BLOBs. */
  2136   2148   #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS
  2137   2149           continue;
  2138   2150   #else
  2139   2151           u32 x = pLevel->iLikeRepCntr;
  2140         -        assert( x>0 );
  2141         -        skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If, (int)(x>>1));
         2152  +        if( x>0 ){
         2153  +          skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If,(int)(x>>1));
         2154  +        }
  2142   2155           VdbeCoverage(v);
  2143   2156   #endif
  2144   2157         }
  2145   2158   #ifdef WHERETRACE_ENABLED /* 0xffff */
  2146   2159         if( sqlite3WhereTrace ){
  2147   2160           VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d",
  2148   2161                            pWC->nTerm-j, pTerm, iLoop));

Changes to src/whereexpr.c.

   814    814           testcase( idxNew==0 );
   815    815           exprAnalyze(pSrc, pWC, idxNew);
   816    816           pTerm = &pWC->a[idxTerm];
   817    817           markTermAsChild(pWC, idxNew, idxTerm);
   818    818         }else{
   819    819           sqlite3ExprListDelete(db, pList);
   820    820         }
   821         -      pTerm->eOperator = WO_NOOP;  /* case 1 trumps case 3 */
   822    821       }
   823    822     }
   824    823   }
   825    824   #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */
   826    825   
   827    826   /*
   828    827   ** We already know that pExpr is a binary operator where both operands are

Changes to test/analyze.test.

   345    345       "
   346    346     } {t4i1 t4i2 t4}
   347    347   }
   348    348   
   349    349   # This test corrupts the database file so it must be the last test
   350    350   # in the series.
   351    351   #
   352         -do_test analyze-99.1 {
          352  +do_test analyze-5.99 {
   353    353     execsql {
   354    354       PRAGMA writable_schema=on;
   355    355       UPDATE sqlite_master SET sql='nonsense' WHERE name='sqlite_stat1';
   356    356     }
   357    357     db close
   358    358     catch { sqlite3 db test.db }
   359    359     catchsql {
   360    360       ANALYZE
   361    361     }
   362    362   } {1 {malformed database schema (sqlite_stat1)}}
          363  +
          364  +# Verify that tables whose names begin with "sqlite" but not
          365  +# "sqlite_" are analyzed.
          366  +#
          367  +db close
          368  +sqlite3 db :memory:
          369  +do_execsql_test analyze-6.1 {
          370  +  CREATE TABLE sqliteDemo(a);
          371  +  INSERT INTO sqliteDemo(a) VALUES(1),(2),(3),(4),(5);
          372  +  CREATE TABLE SQLiteDemo2(a INTEGER PRIMARY KEY AUTOINCREMENT);
          373  +  INSERT INTO SQLiteDemo2 SELECT * FROM sqliteDemo;
          374  +  CREATE TABLE t1(b);
          375  +  INSERT INTO t1(b) SELECT a FROM sqliteDemo;
          376  +  ANALYZE;
          377  +  SELECT tbl FROM sqlite_stat1 WHERE idx IS NULL ORDER BY tbl;
          378  +} {SQLiteDemo2 sqliteDemo t1}
   363    379   
   364    380   finish_test

Changes to test/autoinc.test.

   671    671     CREATE TABLE t10a(a INTEGER PRIMARY KEY AUTOINCREMENT, b UNIQUE);
   672    672     INSERT INTO t10a VALUES(888,9999);
   673    673     CREATE TABLE t10b(x INTEGER PRIMARY KEY AUTOINCREMENT, y UNIQUE);
   674    674     INSERT INTO t10b SELECT * FROM t10a;
   675    675     SELECT * FROM sqlite_sequence;
   676    676   } {t10a 888 t10b 888}
   677    677   
   678         -
          678  +# 2018-04-21 autoincrement does not cause problems for upsert
          679  +#
          680  +do_execsql_test autoinc-11.1 {
          681  +  CREATE TABLE t11(a INTEGER PRIMARY KEY AUTOINCREMENT,b UNIQUE);
          682  +  INSERT INTO t11(a,b) VALUES(2,3),(5,6),(4,3),(1,2)
          683  +    ON CONFLICT(b) DO UPDATE SET a=a+1000;
          684  +  SELECT seq FROM sqlite_sequence WHERE name='t11';
          685  +} {5}
   679    686   
   680    687   finish_test

Changes to test/avtrans.test.

    18     18   set testdir [file dirname $argv0]
    19     19   source $testdir/tester.tcl
    20     20   
    21     21   
    22     22   # Create several tables to work with.
    23     23   #
    24     24   do_test avtrans-1.0 {
    25         -  execsql { PRAGMA auto_vacuum=ON }
           25  +  execsql { PRAGMA auto_vacuum=full }
    26     26     wal_set_journal_mode
    27     27     execsql { 
    28     28       CREATE TABLE one(a int PRIMARY KEY, b text);
    29     29       INSERT INTO one VALUES(1,'one');
    30     30       INSERT INTO one VALUES(2,'two');
    31     31       INSERT INTO one VALUES(3,'three');
    32     32       SELECT b FROM one ORDER BY a;
    33     33     }
    34     34   } {one two three}
           35  +do_test avtrans-1.0.1 { execsql { PRAGMA auto_vacuum } } 1
    35     36   do_test avtrans-1.1 {
    36     37     execsql {
    37     38       CREATE TABLE two(a int PRIMARY KEY, b text);
    38     39       INSERT INTO two VALUES(1,'I');
    39     40       INSERT INTO two VALUES(5,'V');
    40     41       INSERT INTO two VALUES(10,'X');
    41     42       SELECT b FROM two ORDER BY a;

Changes to test/bestindex1.test.

    55     55     0 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
    56     56   }
    57     57   
    58     58   do_eqp_test 1.2 {
    59     59     SELECT * FROM x1 WHERE a IN ('abc', 'def');
    60     60   } {
    61     61     0 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
    62         -  0 0 0 {EXECUTE LIST SUBQUERY 1}
    63     62   }
    64     63   
    65     64   #-------------------------------------------------------------------------
    66     65   #
    67     66   reset_db
    68     67   register_tcl_module db
    69     68   
................................................................................
   142    141   
   143    142     do_execsql_test 2.2.$mode.5 {
   144    143       SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
   145    144     } {1 4} 
   146    145   
   147    146     set plan(use) {
   148    147       0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
   149         -    0 0 0 {EXECUTE LIST SUBQUERY 1}
   150    148       0 0 0 {USE TEMP B-TREE FOR ORDER BY}
   151    149     }
   152    150     set plan(omit) {
   153    151       0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
   154         -    0 0 0 {EXECUTE LIST SUBQUERY 1}
   155    152       0 0 0 {USE TEMP B-TREE FOR ORDER BY}
   156    153     }
   157    154     set plan(use2) {
   158    155       0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x}
   159         -    0 0 0 {EXECUTE LIST SUBQUERY 1}
   160    156       0 0 0 {USE TEMP B-TREE FOR ORDER BY}
   161    157     }
   162    158   
   163    159     do_eqp_test 2.2.$mode.6 { 
   164    160       SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
   165    161     } $plan($mode)
   166    162   }

Changes to test/csv01.test.

   136    136         columns=4,
   137    137         schema=
   138    138         'CREATE TABLE t3(a,b,c,d) WITHOUT ROWID',
   139    139         testflags=1
   140    140         );
   141    141   } {1 {vtable constructor failed: t5}}
   142    142   
          143  +# 2018-04-24
          144  +# Memory leak reported on the sqlite-users mailing list by Ralf Junker.
          145  +#
          146  +do_catchsql_test 4.3 {
          147  +  CREATE VIRTUAL TABLE IF NOT EXISTS temp.t1
          148  +  USING csv(filename='FileDoesNotExist.csv');
          149  +} {1 {cannot open 'FileDoesNotExist.csv' for reading}}
   143    150   
   144    151   finish_test

Changes to test/cursorhint2.test.

   132    132   
   133    133   do_extract_hints_test 2.5 {
   134    134     SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
   135    135   } {
   136    136     x2 {EQ(c0,r[2])}
   137    137   }
   138    138   
   139         -do_extract_hints_test 2.6 {
   140         -  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL)
   141         -} {
   142         -  x2 {EQ(c0,r[2])}
   143         -}
   144         -
   145         -do_extract_hints_test 2.7 {
   146         -  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL)
   147         -} {
   148         -  x2 {EQ(c0,r[2])}
   149         -}
   150         -
   151         -do_extract_hints_test 2.8 {
   152         -  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL
   153         -} {
   154         -  x2 {EQ(c0,r[2])}
   155         -}
   156         -
   157         -do_extract_hints_test 2.9 {
   158         -  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE CASE b WHEN 0 THEN 0 ELSE 1 END;
   159         -} {
   160         -  x2 {EQ(c0,r[2])}
   161         -}
   162         -
   163         -do_extract_hints_test 2.10 {
   164         -  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32
   165         -} {
   166         -  x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))}
   167         -}
   168         -
   169         -ifcapable !icu {
   170         -  # This test only works using the built-in LIKE, not the ICU LIKE extension.
   171         -  do_extract_hints_test 2.11 {
   172         -    SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%'
          139  +if {0} {
          140  +  # These tests no longer work due to the LEFT-JOIN strength reduction
          141  +  # optimization
          142  +  do_extract_hints_test 2.6 {
          143  +    SELECT * FROM x1 CROSS JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL)
          144  +  } {
          145  +    x2 {EQ(c0,r[2])}
          146  +  }
          147  +  
          148  +  do_extract_hints_test 2.7 {
          149  +    SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL)
          150  +  } {
          151  +    x2 {EQ(c0,r[2])}
          152  +  }
          153  +  
          154  +  do_extract_hints_test 2.8 {
          155  +    SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL
          156  +  } {
          157  +    x2 {EQ(c0,r[2])}
          158  +  }
          159  +  
          160  +  do_extract_hints_test 2.9 {
          161  +    SELECT * FROM x1 LEFT JOIN x2 ON (a=x)
          162  +      WHERE CASE b WHEN 0 THEN 0 ELSE 1 END;
          163  +  } {
          164  +    x2 {EQ(c0,r[2])}
          165  +  }
          166  +  
          167  +  do_extract_hints_test 2.10 {
          168  +    SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32
   173    169     } {
   174         -    x2 {AND(expr,EQ(c0,r[2]))}
          170  +    x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))}
          171  +  }
          172  +  
          173  +  ifcapable !icu {
          174  +    # This test only works using the built-in LIKE, not the ICU LIKE extension.
          175  +    do_extract_hints_test 2.11 {
          176  +      SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%'
          177  +    } {
          178  +      x2 {AND(expr,EQ(c0,r[2]))}
          179  +    }
   175    180     }
   176    181   }
   177         -
          182  +  
   178    183   do_extract_hints_test 2.12 {
   179    184     SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE coalesce(x2.b, 1)
   180    185   } {
   181    186     x2 {EQ(c0,r[2])}
   182    187   }
   183    188   
   184    189   finish_test

Changes to test/dbstatus2.test.

   106    106   do_test 3.0 { db_spill db 1 } {0 0 0}
   107    107   do_test 3.1 { db_spill db 0 } {0 0 0}
   108    108   do_execsql_test 3.2 {
   109    109     PRAGMA journal_mode=DELETE;
   110    110     PRAGMA cache_size=3;
   111    111     UPDATE t1 SET b=randomblob(1000);
   112    112   } {delete}
   113         -do_test 3.2 { db_spill db 0 } {0 8 0}
          113  +do_test 3.3 { db_spill db 0 } {0 8 0}
   114    114    
   115    115   finish_test

Changes to test/delete.test.

   398    398   do_execsql_test delete-10.1 {
   399    399     DELETE FROM t1 WHERE a='1' AND b='2';
   400    400   }
   401    401   
   402    402   do_execsql_test delete-10.2 {
   403    403     SELECT * FROM t1 WHERE a='1' AND b='2';
   404    404   }
          405  +
          406  +do_execsql_test delete-11.0 {
          407  +  CREATE TABLE t11(a INTEGER PRIMARY KEY, b INT);
          408  +  WITH RECURSIVE cnt(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM cnt WHERE x<20)
          409  +    INSERT INTO t11(a,b) SELECT x, (x*17)%100 FROM cnt;
          410  +  SELECT * FROM t11;
          411  +} {1 17 2 34 3 51 4 68 5 85 6 2 7 19 8 36 9 53 10 70 11 87 12 4 13 21 14 38 15 55 16 72 17 89 18 6 19 23 20 40}
          412  +do_execsql_test delete-11.1 {
          413  +  DELETE FROM t11 AS xyz
          414  +   WHERE EXISTS(SELECT 1 FROM t11 WHERE t11.a>xyz.a AND t11.b<=xyz.b);
          415  +  SELECT * FROM t11;
          416  +} {6 2 12 4 18 6 19 23 20 40}
   405    417   
   406    418   
   407    419   finish_test

Changes to test/e_select.test.

   744    744   do_execsql_test e_select-3.1.5 { SELECT k FROM x1 WHERE x IS NULL } {4 5}
   745    745   do_execsql_test e_select-3.1.6 { SELECT k FROM x1 WHERE z - 78.43 } {2 4 6}
   746    746   
   747    747   do_execsql_test e_select-3.2.1a {
   748    748     SELECT k FROM x1 LEFT JOIN x2 USING(k)
   749    749   } {1 2 3 4 5 6}
   750    750   do_execsql_test e_select-3.2.1b {
   751         -  SELECT k FROM x1 LEFT JOIN x2 USING(k) WHERE x2.k
          751  +  SELECT k FROM x1 LEFT JOIN x2 USING(k) WHERE x2.k ORDER BY +k
   752    752   } {1 3 5}
   753    753   do_execsql_test e_select-3.2.2 {
   754    754     SELECT k FROM x1 LEFT JOIN x2 USING(k) WHERE x2.k IS NULL
   755    755   } {2 4 6}
   756    756   
   757    757   do_execsql_test e_select-3.2.3 {
   758    758     SELECT k FROM x1 NATURAL JOIN x2 WHERE x2.k

Changes to test/istrue.test.

   118    118   } {1 {CHECK constraint failed: t2}}
   119    119   do_catchsql_test istrue-523 {
   120    120     INSERT INTO t2 VALUES(2,true,false,true,null);
   121    121   } {1 {CHECK constraint failed: t2}}
   122    122   do_catchsql_test istrue-524 {
   123    123     INSERT INTO t2 VALUES(2,true,false,null,false);
   124    124   } {1 {CHECK constraint failed: t2}}
          125  +
          126  +foreach {tn val} [list 1 NaN 2 -NaN 3 NaN0 4 -NaN0 5 Inf 6 -Inf] {
          127  +  do_execsql_test istrue-600.$tn.1 {
          128  +    DROP TABLE IF EXISTS t1;
          129  +    CREATE TABLE t1(x);
          130  +  }
          131  +  do_test istrue-600.$tn.2 {
          132  +    set ::STMT [sqlite3_prepare db "INSERT INTO t1 VALUES(?)" -1 TAIL]
          133  +    sqlite3_bind_double $::STMT 1 $val
          134  +    sqlite3_step $::STMT
          135  +    sqlite3_reset $::STMT
          136  +    sqlite3_finalize $::STMT
          137  +  } {SQLITE_OK}
          138  +  do_execsql_test istrue-600.$tn.3 {
          139  +    SELECT x IS TRUE FROM t1;
          140  +  } [expr {$tn in [list 5 6] ? {1} : {0}}]
          141  +  do_execsql_test istrue-600.$tn.4 {
          142  +    SELECT x IS FALSE FROM t1;
          143  +  } {0}
          144  +}
          145  +
          146  +do_execsql_test istrue-700 {
          147  +  CREATE TABLE t7(
          148  +    a INTEGER PRIMARY KEY,
          149  +    b BOOLEAN DEFAULT false,
          150  +    c BOOLEAN DEFAULT true
          151  +  );
          152  +  INSERT INTO t7(a) VALUES(1);
          153  +  INSERT INTO t7(a,b,c) VALUES(2,true,false);
          154  +  ALTER TABLE t7 ADD COLUMN d BOOLEAN DEFAULT false;
          155  +  ALTER TABLE t7 ADD COLUMN e BOOLEAN DEFAULT true;
          156  +  INSERT INTO t7(a,b,c) VALUES(3,true,false);
          157  +  INSERT INTO t7 VALUES(4,false,true,true,false);
          158  +  SELECT *,'x' FROM t7 ORDER BY a;
          159  +} {1 0 1 0 1 x 2 1 0 0 1 x 3 1 0 0 1 x 4 0 1 1 0 x}
   125    160   
   126    161   finish_test

Changes to test/join.test.

   776    776     INSERT INTO t3(id) VALUES(1),(2);
   777    777     SELECT t1.id, x2.id, x3.id
   778    778     FROM t1
   779    779     LEFT JOIN (SELECT * FROM t2) AS x2 ON t1.id=x2.c2
   780    780     LEFT JOIN t3 AS x3 ON x2.id=x3.c3;
   781    781   } {456 {} {}}
   782    782   
          783  +# 2018-03-24.
          784  +# E.Pasma discovered that the LEFT JOIN strength reduction optimization
          785  +# was misbehaving.  The problem turned out to be that the
          786  +# sqlite3ExprImpliesNotNull() routine was saying that CASE expressions
          787  +# like
          788  +#
          789  +#     CASE WHEN true THEN true ELSE x=0 END
          790  +#
          791  +# could never be true if x is NULL.  The following test cases verify
          792  +# that this error has been resolved.
          793  +#
          794  +db close
          795  +sqlite3 db :memory:
          796  +do_execsql_test join-15.100 {
          797  +  CREATE TABLE t1(a INT, b INT);
          798  +  INSERT INTO t1 VALUES(1,2),(3,4);
          799  +  CREATE TABLE t2(x INT, y INT);
          800  +  SELECT *, 'x'
          801  +    FROM t1 LEFT JOIN t2
          802  +   WHERE CASE WHEN FALSE THEN a=x ELSE 1 END;
          803  +} {1 2 {} {} x 3 4 {} {} x}
          804  +do_execsql_test join-15.105 {
          805  +  SELECT *, 'x'
          806  +    FROM t1 LEFT JOIN t2
          807  +   WHERE a IN (1,3,x,y);
          808  +} {1 2 {} {} x 3 4 {} {} x}
          809  +do_execsql_test join-15.106 {
          810  +  SELECT *, 'x' 
          811  +    FROM t1 LEFT JOIN t2 
          812  +   WHERE NOT ( 'x'='y' AND t2.y=1 );
          813  +} {1 2 {} {} x 3 4 {} {} x}
          814  +do_execsql_test join-15.107 {
          815  +  SELECT *, 'x' 
          816  +    FROM t1 LEFT JOIN t2 
          817  +   WHERE t2.y IS NOT 'abc'
          818  +} {1 2 {} {} x 3 4 {} {} x}
          819  +do_execsql_test join-15.110 {
          820  +  DROP TABLE t1;
          821  +  DROP TABLE t2;
          822  +  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER);
          823  +  INSERT INTO t1(a,b) VALUES(1,0),(11,1),(12,1),(13,1),(121,12);
          824  +  CREATE INDEX t1b ON t1(b);
          825  +  CREATE TABLE t2(x INTEGER PRIMARY KEY);
          826  +  INSERT INTO t2(x) VALUES(0),(1);
          827  +  SELECT  a1, a2, a3, a4, a5
          828  +   FROM (SELECT a AS a1 FROM t1 WHERE b=0)
          829  +        JOIN (SELECT x AS x1 FROM t2)
          830  +        LEFT JOIN (SELECT a AS a2, b AS b2 FROM t1)
          831  +          ON x1 IS TRUE AND b2=a1
          832  +        JOIN (SELECT x AS x2 FROM t2)
          833  +          ON x2<=CASE WHEN x1 THEN CASE WHEN a2 THEN 1 ELSE -1 END ELSE 0 END
          834  +        LEFT JOIN (SELECT a AS a3, b AS b3 FROM t1)
          835  +          ON x2 IS TRUE AND b3=a2
          836  +        JOIN (SELECT x AS x3 FROM t2)
          837  +          ON x3<=CASE WHEN x2 THEN CASE WHEN a3 THEN 1 ELSE -1 END ELSE 0 END
          838  +        LEFT JOIN (SELECT a AS a4, b AS b4 FROM t1)
          839  +          ON x3 IS TRUE AND b4=a3
          840  +        JOIN (SELECT x AS x4 FROM t2)
          841  +          ON x4<=CASE WHEN x3 THEN CASE WHEN a4 THEN 1 ELSE -1 END ELSE 0 END
          842  +        LEFT JOIN (SELECT a AS a5, b AS b5 FROM t1)
          843  +          ON x4 IS TRUE AND b5=a4
          844  +   ORDER BY a1, a2, a3, a4, a5;
          845  +} {1 {} {} {} {} 1 11 {} {} {} 1 12 {} {} {} 1 12 121 {} {} 1 13 {} {} {}}
          846  +
   783    847   finish_test

Changes to test/join2.test.

    82     82     CREATE TABLE cc(c);
    83     83     INSERT INTO aa VALUES('one');
    84     84     INSERT INTO bb VALUES('one');
    85     85     INSERT INTO cc VALUES('one');
    86     86   }
    87     87   
    88     88   do_catchsql_test 2.1 {
    89         -  SELECT * FROM aa LEFT JOIN cc ON (a=b) JOIN bb ON (b=c);
           89  +  SELECT * FROM aa LEFT JOIN cc ON (a=b) JOIN bb ON (b=coalesce(c,1));
    90     90   } {1 {ON clause references tables to its right}}
    91     91   do_catchsql_test 2.2 {
    92     92     SELECT * FROM aa JOIN cc ON (a=b) JOIN bb ON (b=c);
    93     93   } {0 {one one one}}
    94     94   
    95     95   #-------------------------------------------------------------------------
    96     96   # Test that a problem causing where.c to overlook opportunities to
................................................................................
   260    260     CREATE INDEX u1ab ON u1(b, c);
   261    261   }
   262    262   do_eqp_test 6.1 {
   263    263     SELECT u2.* FROM u2 LEFT JOIN u1 ON( u1.a=u2.a AND u1.b=u2.b AND u1.c=u2.c );
   264    264   } {
   265    265     0 0 0 {SCAN TABLE u2}
   266    266   }
          267  +
          268  +db close
          269  +sqlite3 db :memory:
          270  +do_execsql_test 7.0 {
          271  +  CREATE TABLE t1(a,b);  INSERT INTO t1 VALUES(1,2),(3,4),(5,6);
          272  +  CREATE TABLE t2(c,d);  INSERT INTO t2 VALUES(2,4),(3,6);
          273  +  CREATE TABLE t3(x);    INSERT INTO t3 VALUES(9);
          274  +  CREATE VIEW test AS
          275  +    SELECT *, 'x'
          276  +      FROM t1 LEFT JOIN (SELECT * FROM t2, t3) ON (c=b AND x=9)
          277  +      WHERE c IS NULL;
          278  +  SELECT * FROM test;
          279  +} {3 4 {} {} {} x 5 6 {} {} {} x}
          280  +
   267    281   
   268    282   finish_test

Changes to test/join5.test.

   160    160     INSERT INTO x3 VALUES('c', NULL);
   161    161     SELECT * FROM x1 LEFT JOIN x2 JOIN x3 WHERE x3.d = x2.b;
   162    162   } {}
   163    163   
   164    164   # Ticket https://www.sqlite.org/src/tktview/c2a19d81652f40568c770c43 on
   165    165   # 2015-08-20.  LEFT JOIN and the push-down optimization.
   166    166   #
   167         -do_execsql_test join6-4.1 {
          167  +do_execsql_test join5-4.1 {
   168    168     SELECT *
   169    169     FROM (
   170    170         SELECT 'apple' fruit
   171    171         UNION ALL SELECT 'banana'
   172    172     ) a
   173    173     JOIN (
   174    174         SELECT 'apple' fruit
   175    175         UNION ALL SELECT 'banana'
   176    176     ) b ON a.fruit=b.fruit
   177    177     LEFT JOIN (
   178    178         SELECT 1 isyellow
   179    179     ) c ON b.fruit='banana';
   180    180   } {apple apple {} banana banana 1}
   181         -do_execsql_test join6-4.2 {
          181  +do_execsql_test join5-4.2 {
   182    182     SELECT *
   183    183       FROM (SELECT 'apple' fruit UNION ALL SELECT 'banana')
   184    184            LEFT JOIN (SELECT 1) ON fruit='banana';
   185    185   } {apple {} banana 1}
   186    186   
   187    187   #-------------------------------------------------------------------------
   188    188   do_execsql_test 5.0 {
................................................................................
   207    207   do_execsql_test 5.4 {
   208    208     SELECT count(z) FROM ( SELECT * FROM y1 ) LEFT JOIN y2 ON x
   209    209   } 1
   210    210   
   211    211   do_execsql_test 5.5 {
   212    212     SELECT * FROM ( SELECT * FROM y1 ) LEFT JOIN y2 ON x
   213    213   } {0 0 1 {}}
          214  +
          215  +#-------------------------------------------------------------------------
          216  +#
          217  +reset_db
          218  +do_execsql_test 6.1 {
          219  +  CREATE TABLE t1(x); 
          220  +  INSERT INTO t1 VALUES(1);
          221  +
          222  +  CREATE TABLE t2(y INTEGER PRIMARY KEY,a,b);
          223  +  INSERT INTO t2 VALUES(1,2,3);
          224  +  CREATE INDEX t2a ON t2(a); 
          225  +  CREATE INDEX t2b ON t2(b); 
          226  +}
          227  +
          228  +do_execsql_test 6.2 {
          229  +  SELECT * FROM t1 LEFT JOIN t2 ON a=2 OR b=3 WHERE y IS NULL;
          230  +} {}
          231  +
          232  +do_execsql_test 6.3.1 {
          233  +  CREATE TABLE t3(x);
          234  +  INSERT INTO t3 VALUES(1);
          235  +  CREATE TABLE t4(y, z);
          236  +  SELECT ifnull(z, '!!!') FROM t3 LEFT JOIN t4 ON (x=y);
          237  +} {!!!}
          238  +
          239  +do_execsql_test 6.3.2 {
          240  +  CREATE INDEX t4i ON t4(y, ifnull(z, '!!!'));
          241  +  SELECT ifnull(z, '!!!') FROM t3 LEFT JOIN t4 ON (x=y);
          242  +} {!!!}
          243  +
          244  +#-------------------------------------------------------------------------
          245  +#
          246  +reset_db
          247  +do_execsql_test 7.0 {
          248  +  CREATE TABLE t1(x);
          249  +  INSERT INTO t1 VALUES(1);
          250  +}
          251  +
          252  +do_execsql_test 7.1 {
          253  +  CREATE TABLE t2(x, y, z);
          254  +  CREATE INDEX t2xy ON t2(x, y);
          255  +  WITH s(i) AS (
          256  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000
          257  +  )
          258  +  INSERT INTO t2 SELECT i/10, i, NULL FROM s;
          259  +  ANALYZE;
          260  +}
          261  +
          262  +do_eqp_test 7.2 {
          263  +  SELECT * FROM t1 LEFT JOIN t2 ON (
          264  +    t2.x = t1.x AND (t2.y=? OR (t2.y=? AND t2.z IS NOT NULL))
          265  +  );
          266  +} {
          267  +  0 0 0 {SCAN TABLE t1} 
          268  +  0 1 1 {SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)} 
          269  +  0 1 1 {SEARCH TABLE t2 USING INDEX t2xy (x=? AND y=?)}
          270  +}
          271  +
          272  +do_execsql_test 7.3 {
          273  +  CREATE TABLE t3(x);
          274  +
          275  +  CREATE TABLE t4(x, y, z);
          276  +  CREATE INDEX t4xy ON t4(x, y);
          277  +  CREATE INDEX t4xz ON t4(x, z);
          278  +
          279  +  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000)
          280  +  INSERT INTO t4 SELECT i/10, i, i FROM s;
          281  +
          282  +  ANALYZE;
          283  +}
          284  +
          285  +do_eqp_test 7.4 {
          286  +  SELECT * FROM t3 LEFT JOIN t4 ON (t4.x = t3.x) WHERE (t4.y = ? OR t4.z = ?);
          287  +} {
          288  +  0 0 0 {SCAN TABLE t3} 
          289  +  0 1 1 {SEARCH TABLE t4 USING INDEX t4xz (x=?)}
          290  +} 
   214    291   
   215    292   finish_test
          293  +

Changes to test/kvtest.c.

   558    558         sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",
   559    559                          iKey/10000, (iKey/100)%100, iKey%100);
   560    560       }
   561    561       out = fopen(zFN, "wb");      
   562    562       nWrote = fwrite(pData, 1, (size_t)nData, out);
   563    563       fclose(out);
   564    564       printf("\r%s   ", zTail); fflush(stdout);
   565         -    if( nWrote!=nData ){
          565  +    if( nWrote!=(size_t)nData ){
   566    566         fatalError("Wrote only %d of %d bytes to %s\n",
   567    567                     (int)nWrote, nData, zFN);
   568    568       }
   569    569     }
   570    570     sqlite3_finalize(pStmt);
   571    571     sqlite3_close(db);
   572    572     sqlite3_free(zFN);

Changes to test/memdb1.test.

    53     53   do_execsql_test 110 {
    54     54     SELECT * FROM t1;
    55     55   } {1 2}
    56     56   
    57     57   # What happens when we try to VACUUM a MEMDB database?
    58     58   #
    59     59   do_execsql_test 120 {
           60  +  PRAGMA auto_vacuum = off;
    60     61     VACUUM;
    61     62   } {}
    62     63   do_execsql_test 130 {
    63     64     CREATE TABLE t2(x, y);
    64     65     WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
    65     66      INSERT INTO t2(x, y) SELECT x, randomblob(1000) FROM c;
    66     67     DROP TABLE t2;

Changes to test/misc8.test.

    53     53     BEGIN;
    54     54     CREATE TABLE t2(x);
    55     55     SELECT a, coalesce(b, eval('ROLLBACK; SELECT ''bam''')), c
    56     56       FROM t1
    57     57      ORDER BY rowid;
    58     58   } {1 {abort due to ROLLBACK}}
    59     59   
           60  +do_catchsql_test misc8-1.8 {
           61  +  PRAGMA empty_result_callbacks = 1;
           62  +  SELECT eval('SELECT * FROM t1 WHERE 1 = 0;');
           63  +} {0 {{}}}
    60     64   
    61     65   reset_db
    62     66   
    63     67   proc dbeval {sql} { db eval $sql }
    64     68   db func eval dbeval
    65     69   
    66     70   do_execsql_test misc8-2.1 {

Added test/optfuzz-db01.c.

            1  +/* content of file testdb01.db */
            2  +unsigned char data001[] = {
            3  +  83, 81, 76,105,116,101, 32,102,111,114,109, 97,116, 32, 51,  0,  2,  0,  1,
            4  +   1,  0, 64, 32, 32,  0,  0,  0,  2,  0,  0,  0, 35,  0,  0,  0,  0,  0,  0,
            5  +   0,  0,  0,  0,  0, 31,  0,  0,  0,  4,  0,  0,  0,  0,  0,  0,  0,  0,  0,
            6  +   0,  0,  1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
            7  +   0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
            8  +   2,  0, 46, 32,152,  5,  0,  0,  0,  7,  1,221,  0,  0,  0,  0, 35,  1,251,
            9  +   1,246,  1,241,  1,236,  1,231,  1,226,  1,221, 84,  4,  7, 23, 17, 17,  1,
           10  + 129, 19,116, 97, 98,108,101,116, 52,116, 52,  5, 67, 82, 69, 65, 84, 69, 32,
           11  +  84, 65, 66, 76, 69, 32,116, 52, 40, 97, 32, 73, 78, 84, 32, 85, 78, 73, 81,
           12  +  85, 69, 32, 78, 79, 84, 32, 78, 85, 76, 76, 44, 32, 98, 32, 73, 78, 84, 32,
           13  +  85, 78, 73, 81, 85, 69, 32, 78, 79, 84, 32, 78, 85, 76, 76, 44, 99, 44,100,
           14  +  44,101, 41, 35,  6,  6, 23, 55, 17,  1,  0,105,110,100,101,120,115,113,108,
           15  + 105,116,101, 95, 97,117,116,111,105,110,100,101,120, 95,116, 52, 95, 50,116,
           16  +  52,  7, 35,  5,  6, 23, 55, 17,  1,  0,105,110,100,101,120,115,113,108,105,
           17  + 116,101, 95, 97,117,116,111,105,110,100,101,120, 95,116, 52, 95, 49,116, 52,
           18  +   6, 42,  3,  6, 23, 17, 17,  1, 65,116, 97, 98,108,101,116, 51,116, 51,  4,
           19  +  67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32,116, 51, 40, 97, 44, 98,
           20  +  44, 99, 44,100, 44,101, 41, 95,  2,  7, 23, 17, 17,  1,129, 41,116, 97, 98,
           21  + 108,101,116, 50,116, 50,  3, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69,
           22  +  32,116, 50, 40, 97, 32, 73, 78, 84, 44, 32, 98, 32, 73, 78, 84, 44, 32, 99,
           23  +  32, 73, 78, 84, 44,100, 32, 73, 78, 84, 44,101, 32, 73, 78, 84, 44, 80, 82,
           24  +  73, 77, 65, 82, 89, 32, 75, 69, 89, 40, 98, 44, 97, 41, 41, 87, 73, 84, 72,
           25  </