/ Check-in [815cc2bb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge changes from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1:815cc2bb48db713a0166b82a55ae931ac43783d1
User & Date: drh 2016-05-23 02:57:33
Context
2016-06-15
10:21
Merge changes from trunk. check-in: 0f707d15 user: drh tags: apple-osx
2016-05-23
02:57
Merge changes from trunk. check-in: 815cc2bb user: drh tags: apple-osx
00:10
Update the configure script with additional hints on the location of tclConfig.sh, hints needed by the latest versions of Xcode. check-in: 90411a28 user: drh tags: trunk
2016-05-18
20:53
Merge the 3.13.0 release changes. check-in: e2e9b985 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   597    597   sqlite3$(TEXE):	$(TOP)/src/shell.c sqlite3.c
   598    598   	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
   599    599   		$(TOP)/src/shell.c sqlite3.c \
   600    600   		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
   601    601   
   602    602   sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
   603    603   	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
          604  +
          605  +scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
          606  +	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
          607  +		$(TOP)/ext/misc/scrub.c sqlite3.o $(TLIBS)
   604    608   
   605    609   srcck1$(BEXE):	$(TOP)/tool/srcck1.c
   606    610   	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
   607    611   
   608    612   sourcetest:	srcck1$(BEXE) sqlite3.c
   609    613   	./srcck1 sqlite3.c
   610    614   

Changes to Makefile.msc.

  1451   1451   	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \
  1452   1452   		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
  1453   1453   
  1454   1454   # <<mark>>
  1455   1455   sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
  1456   1456   	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1457   1457   
         1458  +scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
         1459  +	$(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1460  +
  1458   1461   srcck1.exe:	$(TOP)\tool\srcck1.c
  1459   1462   	$(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c
  1460   1463   
  1461   1464   sourcetest:	srcck1.exe sqlite3.c
  1462   1465   	srcck1.exe sqlite3.c
  1463   1466   
  1464   1467   fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)

Changes to VERSION.

     1         -3.13.0
            1  +3.14.0

Changes to autoconf/configure.ac.

    37     37   AC_ARG_ENABLE(editline, [AS_HELP_STRING(
    38     38     [--enable-editline], 
    39     39     [use BSD libedit])], 
    40     40     [], [enable_editline=yes])
    41     41   AC_ARG_ENABLE(readline, [AS_HELP_STRING(
    42     42     [--enable-readline], 
    43     43     [use readline])], 
    44         -  [], [enable_readline=no])
           44  +  [], [enable_readline=yes])
    45     45   if test x"$enable_editline" != xno ; then
    46     46     sLIBS=$LIBS
    47     47     LIBS=""
    48     48     AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
    49     49     READLINE_LIBS=$LIBS
    50     50     if test x"$LIBS" != "x"; then
    51         -     AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
           51  +    AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
           52  +    enable_readline=no
    52     53     else
    53     54       unset ac_cv_search_readline
    54     55     fi
    55     56     LIBS=$sLIBS
    56     57   fi
    57     58   if test x"$enable_readline" != xno ; then
    58     59     sLIBS=$LIBS

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.13.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.14.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.13.0'
   730         -PACKAGE_STRING='sqlite 3.13.0'
          729  +PACKAGE_VERSION='3.14.0'
          730  +PACKAGE_STRING='sqlite 3.14.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1457   1457   #
  1458   1458   # Report the --help message.
  1459   1459   #
  1460   1460   if test "$ac_init_help" = "long"; then
  1461   1461     # Omit some internal or obsolete options to make the list less imposing.
  1462   1462     # This message is too long to be a string in the A/UX 3.1 sh.
  1463   1463     cat <<_ACEOF
  1464         -\`configure' configures sqlite 3.13.0 to adapt to many kinds of systems.
         1464  +\`configure' configures sqlite 3.14.0 to adapt to many kinds of systems.
  1465   1465   
  1466   1466   Usage: $0 [OPTION]... [VAR=VALUE]...
  1467   1467   
  1468   1468   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1469   1469   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1470   1470   
  1471   1471   Defaults for the options are specified in brackets.
................................................................................
  1522   1522     --build=BUILD     configure for building on BUILD [guessed]
  1523   1523     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1524   1524   _ACEOF
  1525   1525   fi
  1526   1526   
  1527   1527   if test -n "$ac_init_help"; then
  1528   1528     case $ac_init_help in
  1529         -     short | recursive ) echo "Configuration of sqlite 3.13.0:";;
         1529  +     short | recursive ) echo "Configuration of sqlite 3.14.0:";;
  1530   1530      esac
  1531   1531     cat <<\_ACEOF
  1532   1532   
  1533   1533   Optional Features:
  1534   1534     --disable-option-checking  ignore unrecognized --enable/--with options
  1535   1535     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1536   1536     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1644   1644       cd "$ac_pwd" || { ac_status=$?; break; }
  1645   1645     done
  1646   1646   fi
  1647   1647   
  1648   1648   test -n "$ac_init_help" && exit $ac_status
  1649   1649   if $ac_init_version; then
  1650   1650     cat <<\_ACEOF
  1651         -sqlite configure 3.13.0
         1651  +sqlite configure 3.14.0
  1652   1652   generated by GNU Autoconf 2.69
  1653   1653   
  1654   1654   Copyright (C) 2012 Free Software Foundation, Inc.
  1655   1655   This configure script is free software; the Free Software Foundation
  1656   1656   gives unlimited permission to copy, distribute and modify it.
  1657   1657   _ACEOF
  1658   1658     exit
................................................................................
  2063   2063     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2064   2064   
  2065   2065   } # ac_fn_c_check_header_mongrel
  2066   2066   cat >config.log <<_ACEOF
  2067   2067   This file contains any messages produced by compilers while
  2068   2068   running configure, to aid debugging if configure makes a mistake.
  2069   2069   
  2070         -It was created by sqlite $as_me 3.13.0, which was
         2070  +It was created by sqlite $as_me 3.14.0, which was
  2071   2071   generated by GNU Autoconf 2.69.  Invocation command line was
  2072   2072   
  2073   2073     $ $0 $@
  2074   2074   
  2075   2075   _ACEOF
  2076   2076   exec 5>>config.log
  2077   2077   {
................................................................................
 10752  10752             if test -f "$i/tclConfig.sh" ; then
 10753  10753               ac_cv_c_tclconfig="$i"
 10754  10754               break
 10755  10755             fi
 10756  10756           done
 10757  10757         fi
 10758  10758       fi
        10759  +
        10760  +    # Recent versions of Xcode on Macs hid the tclConfig.sh file
        10761  +    # in a strange place.
        10762  +    if test x"${ac_cv_c_tclconfig}" = x ; then
        10763  +      if test x"$cross_compiling" = xno; then
        10764  +        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
        10765  +        do
        10766  +          if test -f "$i/tclConfig.sh" ; then
        10767  +            ac_cv_c_tclconfig="$i"
        10768  +            break
        10769  +          fi
        10770  +        done
        10771  +      fi
        10772  +    fi
 10759  10773   
 10760  10774       # then check for a private Tcl installation
 10761  10775       if test x"${ac_cv_c_tclconfig}" = x ; then
 10762  10776         for i in \
 10763  10777               ../tcl \
 10764  10778               `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
 10765  10779               `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \
................................................................................
 12091  12105   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12092  12106   
 12093  12107   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12094  12108   # Save the log message, to keep $0 and so on meaningful, and to
 12095  12109   # report actual input values of CONFIG_FILES etc. instead of their
 12096  12110   # values after options handling.
 12097  12111   ac_log="
 12098         -This file was extended by sqlite $as_me 3.13.0, which was
        12112  +This file was extended by sqlite $as_me 3.14.0, which was
 12099  12113   generated by GNU Autoconf 2.69.  Invocation command line was
 12100  12114   
 12101  12115     CONFIG_FILES    = $CONFIG_FILES
 12102  12116     CONFIG_HEADERS  = $CONFIG_HEADERS
 12103  12117     CONFIG_LINKS    = $CONFIG_LINKS
 12104  12118     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12105  12119     $ $0 $@
................................................................................
 12157  12171   
 12158  12172   Report bugs to the package provider."
 12159  12173   
 12160  12174   _ACEOF
 12161  12175   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12162  12176   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12163  12177   ac_cs_version="\\
 12164         -sqlite config.status 3.13.0
        12178  +sqlite config.status 3.14.0
 12165  12179   configured by $0, generated by GNU Autoconf 2.69,
 12166  12180     with options \\"\$ac_cs_config\\"
 12167  12181   
 12168  12182   Copyright (C) 2012 Free Software Foundation, Inc.
 12169  12183   This config.status script is free software; the Free Software Foundation
 12170  12184   gives unlimited permission to copy, distribute and modify it."
 12171  12185   

Changes to configure.ac.

   322    322   
   323    323       # On ubuntu 14.10, $auto_path on tclsh is not quite correct.
   324    324       # So try again after applying corrections.
   325    325       if test x"${ac_cv_c_tclconfig}" = x ; then
   326    326         if test x"$cross_compiling" = xno; then
   327    327           for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
   328    328           do
          329  +          if test -f "$i/tclConfig.sh" ; then
          330  +            ac_cv_c_tclconfig="$i"
          331  +            break
          332  +          fi
          333  +        done
          334  +      fi
          335  +    fi
          336  +
          337  +    # Recent versions of Xcode on Macs hid the tclConfig.sh file
          338  +    # in a strange place.
          339  +    if test x"${ac_cv_c_tclconfig}" = x ; then
          340  +      if test x"$cross_compiling" = xno; then
          341  +        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
          342  +        do
   329    343             if test -f "$i/tclConfig.sh" ; then
   330    344               ac_cv_c_tclconfig="$i"
   331    345               break
   332    346             fi
   333    347           done
   334    348         fi
   335    349       fi

Added ext/misc/scrub.c.

            1  +/*
            2  +** 2016-05-05
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file implements a utility function (and a utility program) that
           14  +** makes a copy of an SQLite database while simultaneously zeroing out all
           15  +** deleted content.
           16  +**
           17  +** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite
           18  +** deletes content, it does not overwrite the deleted content but rather marks
           19  +** the region of the file that held that content as being reusable.  This can
           20  +** cause deleted content to recoverable from the database file.  This stale
           21  +** content is removed by the VACUUM command, but VACUUM can be expensive for
           22  +** large databases.  When in PRAGMA secure_delete=ON mode, the deleted content
           23  +** is zeroed, but secure_delete=ON has overhead as well.
           24  +**
           25  +** This utility attempts to make a copy of a complete SQLite database where
           26  +** all of the deleted content is zeroed out in the copy, and it attempts to
           27  +** do so while being faster than running VACUUM.
           28  +**
           29  +** Usage:
           30  +**
           31  +**   int sqlite3_scrub_backup(
           32  +**       const char *zSourceFile,   // Source database filename
           33  +**       const char *zDestFile,     // Destination database filename
           34  +**       char **pzErrMsg            // Write error message here
           35  +**   );
           36  +**
           37  +** Simply call the API above specifying the filename of the source database
           38  +** and the name of the backup copy.  The source database must already exist
           39  +** and can be in active use. (A read lock is held during the backup.)  The
           40  +** destination file should not previously exist.  If the pzErrMsg parameter
           41  +** is non-NULL and if an error occurs, then an error message might be written
           42  +** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to
           43  +** that error message.  But if the error is an OOM, the error might not be
           44  +** reported.  The routine always returns non-zero if there is an error.
           45  +**
           46  +** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and
           47  +** this file becomes a standalone program that can be run as follows:
           48  +**
           49  +**      ./sqlite3scrub SOURCE DEST
           50  +*/
           51  +#include "sqlite3.h"
           52  +#include <assert.h>
           53  +#include <stdio.h>
           54  +#include <stdlib.h>
           55  +#include <stdarg.h>
           56  +#include <string.h>
           57  +
           58  +typedef struct ScrubState ScrubState;
           59  +typedef unsigned char u8;
           60  +typedef unsigned short u16;
           61  +typedef unsigned int u32;
           62  +
           63  +
           64  +/* State information for a scrub-and-backup operation */
           65  +struct ScrubState {
           66  +  const char *zSrcFile;    /* Name of the source file */
           67  +  const char *zDestFile;   /* Name of the destination file */
           68  +  int rcErr;               /* Error code */
           69  +  char *zErr;              /* Error message text */
           70  +  sqlite3 *dbSrc;          /* Source database connection */
           71  +  sqlite3_file *pSrc;      /* Source file handle */
           72  +  sqlite3 *dbDest;         /* Destination database connection */
           73  +  sqlite3_file *pDest;     /* Destination file handle */
           74  +  u32 szPage;              /* Page size */
           75  +  u32 szUsable;            /* Usable bytes on each page */
           76  +  u32 nPage;               /* Number of pages */
           77  +  u8 *page1;               /* Content of page 1 */
           78  +};
           79  +
           80  +/* Store an error message */
           81  +static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){
           82  +  va_list ap;
           83  +  sqlite3_free(p->zErr);
           84  +  va_start(ap, zFormat);
           85  +  p->zErr = sqlite3_vmprintf(zFormat, ap);
           86  +  va_end(ap);
           87  +  if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR;
           88  +}
           89  +
           90  +/* Allocate memory to hold a single page of content */
           91  +static u8 *scrubBackupAllocPage(ScrubState *p){
           92  +  u8 *pPage;
           93  +  if( p->rcErr ) return 0;
           94  +  pPage = sqlite3_malloc( p->szPage );
           95  +  if( pPage==0 ) p->rcErr = SQLITE_NOMEM;
           96  +  return pPage;
           97  +}
           98  +
           99  +/* Read a page from the source database into memory.  Use the memory
          100  +** provided by pBuf if not NULL or allocate a new page if pBuf==NULL.
          101  +*/
          102  +static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){
          103  +  int rc;
          104  +  sqlite3_int64 iOff;
          105  +  u8 *pOut = pBuf;
          106  +  if( p->rcErr ) return 0;
          107  +  if( pOut==0 ){
          108  +    pOut = scrubBackupAllocPage(p);
          109  +    if( pOut==0 ) return 0;
          110  +  }
          111  +  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
          112  +  rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff);
          113  +  if( rc!=SQLITE_OK ){
          114  +    if( pBuf==0 ) sqlite3_free(pOut);
          115  +    pOut = 0;
          116  +    scrubBackupErr(p, "read failed for page %d", pgno);
          117  +    p->rcErr = SQLITE_IOERR;
          118  +  }
          119  +  return pOut;  
          120  +}
          121  +
          122  +/* Write a page to the destination database */
          123  +static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){
          124  +  int rc;
          125  +  sqlite3_int64 iOff;
          126  +  if( p->rcErr ) return;
          127  +  iOff = (pgno-1)*(sqlite3_int64)p->szPage;
          128  +  rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff);
          129  +  if( rc!=SQLITE_OK ){
          130  +    scrubBackupErr(p, "write failed for page %d", pgno);
          131  +    p->rcErr = SQLITE_IOERR;
          132  +  }
          133  +}
          134  +
          135  +/* Prepare a statement against the "db" database. */
          136  +static sqlite3_stmt *scrubBackupPrepare(
          137  +  ScrubState *p,      /* Backup context */
          138  +  sqlite3 *db,        /* Database to prepare against */
          139  +  const char *zSql    /* SQL statement */
          140  +){
          141  +  sqlite3_stmt *pStmt;
          142  +  if( p->rcErr ) return 0;
          143  +  p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
          144  +  if( p->rcErr ){
          145  +    scrubBackupErr(p, "SQL error \"%s\" on \"%s\"",
          146  +                   sqlite3_errmsg(db), zSql);
          147  +    sqlite3_finalize(pStmt);
          148  +    return 0;
          149  +  }
          150  +  return pStmt;
          151  +}
          152  +
          153  +
          154  +/* Open the source database file */
          155  +static void scrubBackupOpenSrc(ScrubState *p){
          156  +  sqlite3_stmt *pStmt;
          157  +  int rc;
          158  +  /* Open the source database file */
          159  +  p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc,
          160  +                 SQLITE_OPEN_READWRITE |
          161  +                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
          162  +  if( p->rcErr ){
          163  +    scrubBackupErr(p, "cannot open source database: %s",
          164  +                      sqlite3_errmsg(p->dbSrc));
          165  +    return;
          166  +  }
          167  +  p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;",
          168  +                          0, 0, 0);
          169  +  if( p->rcErr ){
          170  +    scrubBackupErr(p,
          171  +       "cannot start a read transaction on the source database: %s",
          172  +       sqlite3_errmsg(p->dbSrc));
          173  +    return;
          174  +  }
          175  +  rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL,
          176  +                                 0, 0);
          177  +  if( rc ){
          178  +    scrubBackupErr(p, "cannot checkpoint the source database");
          179  +    return;
          180  +  }
          181  +  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size");
          182  +  if( pStmt==0 ) return;
          183  +  rc = sqlite3_step(pStmt);
          184  +  if( rc==SQLITE_ROW ){
          185  +    p->szPage = sqlite3_column_int(pStmt, 0);
          186  +  }else{
          187  +    scrubBackupErr(p, "unable to determine the page size");
          188  +  }
          189  +  sqlite3_finalize(pStmt);
          190  +  if( p->rcErr ) return;
          191  +  pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count");
          192  +  if( pStmt==0 ) return;
          193  +  rc = sqlite3_step(pStmt);
          194  +  if( rc==SQLITE_ROW ){
          195  +    p->nPage = sqlite3_column_int(pStmt, 0);
          196  +  }else{
          197  +    scrubBackupErr(p, "unable to determine the size of the source database");
          198  +  }
          199  +  sqlite3_finalize(pStmt);
          200  +  sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc);
          201  +  if( p->pSrc==0 || p->pSrc->pMethods==0 ){
          202  +    scrubBackupErr(p, "cannot get the source file handle");
          203  +    p->rcErr = SQLITE_ERROR;
          204  +  }
          205  +}
          206  +
          207  +/* Create and open the destination file */
          208  +static void scrubBackupOpenDest(ScrubState *p){
          209  +  sqlite3_stmt *pStmt;
          210  +  int rc;
          211  +  char *zSql;
          212  +  if( p->rcErr ) return;
          213  +  p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest,
          214  +                 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
          215  +                 SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0);
          216  +  if( p->rcErr ){
          217  +    scrubBackupErr(p, "cannot open destination database: %s",
          218  +                      sqlite3_errmsg(p->dbDest));
          219  +    return;
          220  +  }
          221  +  zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage);
          222  +  if( zSql==0 ){
          223  +    p->rcErr = SQLITE_NOMEM;
          224  +    return;
          225  +  }
          226  +  p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0);
          227  +  sqlite3_free(zSql);
          228  +  if( p->rcErr ){
          229  +    scrubBackupErr(p,
          230  +       "cannot set the page size on the destination database: %s",
          231  +       sqlite3_errmsg(p->dbDest));
          232  +    return;
          233  +  }
          234  +  sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0);
          235  +  p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0);
          236  +  if( p->rcErr ){
          237  +    scrubBackupErr(p,
          238  +       "cannot start a write transaction on the destination database: %s",
          239  +       sqlite3_errmsg(p->dbDest));
          240  +    return;
          241  +  }
          242  +  pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;");
          243  +  if( pStmt==0 ) return;
          244  +  rc = sqlite3_step(pStmt);
          245  +  if( rc!=SQLITE_ROW ){
          246  +    scrubBackupErr(p, "cannot measure the size of the destination");
          247  +  }else if( sqlite3_column_int(pStmt, 0)>1 ){
          248  +    scrubBackupErr(p, "destination database is not empty - holds %d pages",
          249  +                   sqlite3_column_int(pStmt, 0));
          250  +  }
          251  +  sqlite3_finalize(pStmt);
          252  +  sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest);
          253  +  if( p->pDest==0 || p->pDest->pMethods==0 ){
          254  +    scrubBackupErr(p, "cannot get the destination file handle");
          255  +    p->rcErr = SQLITE_ERROR;
          256  +  }
          257  +}
          258  +
          259  +/* Read a 32-bit big-endian integer */
          260  +static u32 scrubBackupInt32(const u8 *a){
          261  +  u32 v = a[3];
          262  +  v += ((u32)a[2])<<8;
          263  +  v += ((u32)a[1])<<16;
          264  +  v += ((u32)a[0])<<24;
          265  +  return v;
          266  +}
          267  +
          268  +/* Read a 16-bit big-endian integer */
          269  +static u32 scrubBackupInt16(const u8 *a){
          270  +  return (a[0]<<8) + a[1];
          271  +}
          272  +
          273  +/*
          274  +** Read a varint.  Put the value in *pVal and return the number of bytes.
          275  +*/
          276  +static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){
          277  +  sqlite3_int64 v = 0;
          278  +  int i;
          279  +  for(i=0; i<8; i++){
          280  +    v = (v<<7) + (z[i]&0x7f);
          281  +    if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
          282  +  }
          283  +  v = (v<<8) + (z[i]&0xff);
          284  +  *pVal = v;
          285  +  return 9;
          286  +}
          287  +
          288  +/*
          289  +** Return the number of bytes in a varint.
          290  +*/
          291  +static int scrubBackupVarintSize(const u8 *z){
          292  +  int i;
          293  +  for(i=0; i<8; i++){
          294  +    if( (z[i]&0x80)==0 ){ return i+1; }
          295  +  }
          296  +  return 9;
          297  +}
          298  +
          299  +/*
          300  +** Copy the freelist trunk page given, and all its descendents,
          301  +** zeroing out as much as possible in the process.
          302  +*/
          303  +static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){
          304  +  u8 *a, *aBuf;
          305  +  u32 n, mx;
          306  +
          307  +  if( p->rcErr ) return;
          308  +  aBuf = scrubBackupAllocPage(p);
          309  +  if( aBuf==0 ) return;
          310  + 
          311  +  while( pgno && nFree){
          312  +    a = scrubBackupRead(p, pgno, aBuf);
          313  +    if( a==0 ) break;
          314  +    n = scrubBackupInt32(&a[4]);
          315  +    mx = p->szUsable/4 - 2;
          316  +    if( n<mx ){
          317  +      memset(&a[n*4+8], 0, 4*(mx-n));
          318  +    }
          319  +    scrubBackupWrite(p, pgno, a);
          320  +    pgno = scrubBackupInt32(a);
          321  +#if 0
          322  +    /* There is really no point in copying the freelist leaf pages.
          323  +    ** Simply leave them uninitialized in the destination database.  The
          324  +    ** OS filesystem should zero those pages for us automatically.
          325  +    */
          326  +    for(i=0; i<n && nFree; i++){
          327  +      u32 iLeaf = scrubBackupInt32(&a[i*4+8]);
          328  +      if( aZero==0 ){
          329  +        aZero = scrubBackupAllocPage(p);
          330  +        if( aZero==0 ){ pgno = 0; break; }
          331  +        memset(aZero, 0, p->szPage);
          332  +      }
          333  +      scrubBackupWrite(p, iLeaf, aZero);
          334  +      nFree--;
          335  +    }
          336  +#endif
          337  +  }
          338  +  sqlite3_free(aBuf);
          339  +}
          340  +
          341  +/*
          342  +** Copy an overflow chain from source to destination.  Zero out any
          343  +** unused tail at the end of the overflow chain.
          344  +*/
          345  +static void scrubBackupOverflow(ScrubState *p, int pgno, u32 nByte){
          346  +  u8 *a, *aBuf;
          347  +
          348  +  aBuf = scrubBackupAllocPage(p);
          349  +  if( aBuf==0 ) return;
          350  +  while( nByte>0 && pgno!=0 ){
          351  +    a = scrubBackupRead(p, pgno, aBuf);
          352  +    if( a==0 ) break;
          353  +    if( nByte >= (p->szUsable)-4 ){
          354  +      nByte -= (p->szUsable) - 4;
          355  +    }else{
          356  +      u32 x = (p->szUsable - 4) - nByte;
          357  +      u32 i = p->szUsable - x;
          358  +      memset(&a[i], 0, x);
          359  +      nByte = 0;
          360  +    }
          361  +    scrubBackupWrite(p, pgno, a);
          362  +    pgno = scrubBackupInt32(a);
          363  +  }
          364  +  sqlite3_free(aBuf);      
          365  +}
          366  +   
          367  +
          368  +/*
          369  +** Copy B-Tree page pgno, and all of its children, from source to destination.
          370  +** Zero out deleted content during the copy.
          371  +*/
          372  +static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){
          373  +  u8 *a;
          374  +  u32 i, n, pc;
          375  +  u32 nCell;
          376  +  u32 nPrefix;
          377  +  u32 szHdr;
          378  +  u32 iChild;
          379  +  u8 *aTop;
          380  +  u8 *aCell;
          381  +  u32 x, y;
          382  +  int ln = 0;
          383  +
          384  +  
          385  +  if( p->rcErr ) return;
          386  +  if( iDepth>50 ){
          387  +    scrubBackupErr(p, "corrupt: b-tree too deep at page %d", pgno);
          388  +    return;
          389  +  }
          390  +  if( pgno==1 ){
          391  +    a = p->page1;
          392  +  }else{
          393  +    a = scrubBackupRead(p, pgno, 0);
          394  +    if( a==0 ) return;
          395  +  }
          396  +  nPrefix = pgno==1 ? 100 : 0;
          397  +  aTop = &a[nPrefix];
          398  +  szHdr = 8 + 4*(aTop[0]==0x02 || aTop[0]==0x05);
          399  +  aCell = aTop + szHdr;
          400  +  nCell = scrubBackupInt16(&aTop[3]);
          401  +
          402  +  /* Zero out the gap between the cell index and the start of the
          403  +  ** cell content area */
          404  +  x = scrubBackupInt16(&aTop[5]);  /* First byte of cell content area */
          405  +  if( x>p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
          406  +  y = szHdr + nPrefix + nCell*2;
          407  +  if( y>x ){ ln=__LINE__; goto btree_corrupt; }
          408  +  if( y<x ) memset(a+y, 0, x-y);  /* Zero the gap */
          409  +
          410  +  /* Zero out all the free blocks */  
          411  +  pc = scrubBackupInt16(&aTop[1]);
          412  +  if( pc>0 && pc<x ){ ln=__LINE__; goto btree_corrupt; }
          413  +  while( pc ){
          414  +    if( pc>(p->szUsable)-4 ){ ln=__LINE__; goto btree_corrupt; }
          415  +    n = scrubBackupInt16(&a[pc+2]);
          416  +    if( pc+n>(p->szUsable) ){ ln=__LINE__; goto btree_corrupt; }
          417  +    if( n>4 ) memset(&a[pc+4], 0, n-4);
          418  +    x = scrubBackupInt16(&a[pc]);
          419  +    if( x<pc+4 && x>0 ){ ln=__LINE__; goto btree_corrupt; }
          420  +    pc = x;
          421  +  }
          422  +
          423  +  /* Write this one page */
          424  +  scrubBackupWrite(p, pgno, a);
          425  +
          426  +  /* Walk the tree and process child pages */
          427  +  for(i=0; i<nCell; i++){
          428  +    u32 X, M, K, nLocal;
          429  +    sqlite3_int64 P;
          430  +    pc = scrubBackupInt16(&aCell[i*2]);
          431  +    if( pc <= szHdr ){ ln=__LINE__; goto btree_corrupt; }
          432  +    if( pc > p->szUsable-3 ){ ln=__LINE__; goto btree_corrupt; }
          433  +    if( aTop[0]==0x05 || aTop[0]==0x02 ){
          434  +      if( pc+4 > p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
          435  +      iChild = scrubBackupInt32(&a[pc]);
          436  +      pc += 4;
          437  +      scrubBackupBtree(p, iChild, iDepth+1);
          438  +      if( aTop[0]==0x05 ) continue;
          439  +    }
          440  +    pc += scrubBackupVarint(&a[pc], &P);
          441  +    if( pc >= p->szUsable ){ ln=__LINE__; goto btree_corrupt; }
          442  +    if( aTop[0]==0x0d ){
          443  +      X = p->szUsable - 35;
          444  +    }else{
          445  +      X = ((p->szUsable - 12)*64/255) - 23;
          446  +    }
          447  +    if( P<=X ){
          448  +      /* All content is local.  No overflow */
          449  +      continue;
          450  +    }
          451  +    M = ((p->szUsable - 12)*32/255)-23;
          452  +    K = M + ((P-M)%(p->szUsable-4));
          453  +    if( aTop[0]==0x0d ){
          454  +      pc += scrubBackupVarintSize(&a[pc]);
          455  +      if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; }
          456  +    }
          457  +    nLocal = K<=X ? K : M;
          458  +    if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; }
          459  +    iChild = scrubBackupInt32(&a[pc+nLocal]);
          460  +    scrubBackupOverflow(p, iChild, P-nLocal);
          461  +  }
          462  +
          463  +  /* Walk the right-most tree */
          464  +  if( aTop[0]==0x05 || aTop[0]==0x02 ){
          465  +    iChild = scrubBackupInt32(&aTop[8]);
          466  +    scrubBackupBtree(p, iChild, iDepth+1);
          467  +  }
          468  +
          469  +  /* All done */
          470  +  if( pgno>1 ) sqlite3_free(a);
          471  +  return;
          472  +
          473  +btree_corrupt:
          474  +  scrubBackupErr(p, "corruption on page %d of source database (errid=%d)",
          475  +                 pgno, ln);
          476  +  if( pgno>1 ) sqlite3_free(a);  
          477  +}
          478  +
          479  +/*
          480  +** Copy all ptrmap pages from source to destination.
          481  +** This routine is only called if the source database is in autovacuum
          482  +** or incremental vacuum mode.
          483  +*/
          484  +static void scrubBackupPtrmap(ScrubState *p){
          485  +  u32 pgno = 2;
          486  +  u32 J = p->szUsable/5;
          487  +  u32 iLock = (1073742335/p->szPage)+1;
          488  +  u8 *a, *pBuf;
          489  +  if( p->rcErr ) return;
          490  +  pBuf = scrubBackupAllocPage(p);
          491  +  if( pBuf==0 ) return;
          492  +  while( pgno<=p->nPage ){
          493  +    a = scrubBackupRead(p, pgno, pBuf);
          494  +    if( a==0 ) break;
          495  +    scrubBackupWrite(p, pgno, a);
          496  +    pgno += J+1;
          497  +    if( pgno==iLock ) pgno++;
          498  +  }
          499  +  sqlite3_free(pBuf);
          500  +}
          501  +
          502  +int sqlite3_scrub_backup(
          503  +  const char *zSrcFile,    /* Source file */
          504  +  const char *zDestFile,   /* Destination file */
          505  +  char **pzErr             /* Write error here if non-NULL */
          506  +){
          507  +  ScrubState s;
          508  +  u32 n, i;
          509  +  sqlite3_stmt *pStmt;
          510  +
          511  +  memset(&s, 0, sizeof(s));
          512  +  s.zSrcFile = zSrcFile;
          513  +  s.zDestFile = zDestFile;
          514  +
          515  +  /* Open both source and destination databases */
          516  +  scrubBackupOpenSrc(&s);
          517  +  scrubBackupOpenDest(&s);
          518  +
          519  +  /* Read in page 1 */
          520  +  s.page1 = scrubBackupRead(&s, 1, 0);
          521  +  if( s.page1==0 ) goto scrub_abort;
          522  +  s.szUsable = s.szPage - s.page1[20];
          523  +
          524  +  /* Copy the freelist */    
          525  +  n = scrubBackupInt32(&s.page1[36]);
          526  +  i = scrubBackupInt32(&s.page1[32]);
          527  +  if( n ) scrubBackupFreelist(&s, i, n);
          528  +
          529  +  /* Copy ptrmap pages */
          530  +  n = scrubBackupInt32(&s.page1[52]);
          531  +  if( n ) scrubBackupPtrmap(&s);
          532  +
          533  +  /* Copy all of the btrees */
          534  +  scrubBackupBtree(&s, 1, 0);
          535  +  pStmt = scrubBackupPrepare(&s, s.dbSrc,
          536  +       "SELECT rootpage FROM sqlite_master WHERE coalesce(rootpage,0)>0");
          537  +  if( pStmt==0 ) goto scrub_abort;
          538  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          539  +    i = (u32)sqlite3_column_int(pStmt, 0);
          540  +    scrubBackupBtree(&s, i, 0);
          541  +  }
          542  +  sqlite3_finalize(pStmt);
          543  +
          544  +scrub_abort:    
          545  +  /* Close the destination database without closing the transaction. If we
          546  +  ** commit, page zero will be overwritten. */
          547  +  sqlite3_close(s.dbDest);
          548  +
          549  +  /* But do close out the read-transaction on the source database */
          550  +  sqlite3_exec(s.dbSrc, "COMMIT;", 0, 0, 0);
          551  +  sqlite3_close(s.dbSrc);
          552  +  sqlite3_free(s.page1);
          553  +  if( pzErr ){
          554  +    *pzErr = s.zErr;
          555  +  }else{
          556  +    sqlite3_free(s.zErr);
          557  +  }
          558  +  return s.rcErr;
          559  +}   
          560  +
          561  +#ifdef SCRUB_STANDALONE
          562  +/* Error and warning log */
          563  +static void errorLogCallback(void *pNotUsed, int iErr, const char *zMsg){
          564  +  const char *zType;
          565  +  switch( iErr&0xff ){
          566  +    case SQLITE_WARNING: zType = "WARNING";  break;
          567  +    case SQLITE_NOTICE:  zType = "NOTICE";   break;
          568  +    default:             zType = "ERROR";    break;
          569  +  }
          570  +  fprintf(stderr, "%s: %s\n", zType, zMsg);
          571  +}
          572  +
          573  +/* The main() routine when this utility is run as a stand-alone program */
          574  +int main(int argc, char **argv){
          575  +  char *zErr = 0;
          576  +  int rc;
          577  +  if( argc!=3 ){
          578  +    fprintf(stderr,"Usage: %s SOURCE DESTINATION\n", argv[0]);
          579  +    exit(1);
          580  +  }
          581  +  sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, 0);
          582  +  rc = sqlite3_scrub_backup(argv[1], argv[2], &zErr);
          583  +  if( rc==SQLITE_NOMEM ){
          584  +    fprintf(stderr, "%s: out of memory\n", argv[0]);
          585  +    exit(1);
          586  +  }
          587  +  if( zErr ){
          588  +    fprintf(stderr, "%s: %s\n", argv[0], zErr);
          589  +    sqlite3_free(zErr);
          590  +    exit(1);
          591  +  }
          592  +  return 0;
          593  +}
          594  +#endif

Changes to main.mk.

   471    471   	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \
   472    472   		$(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
   473    473   
   474    474   sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
   475    475   	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
   476    476   		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)
   477    477   
          478  +scrub$(EXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
          479  +	$(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB)
          480  +
   478    481   srcck1$(EXE):	$(TOP)/tool/srcck1.c
   479    482   	$(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c
   480    483   
   481    484   sourcetest:	srcck1$(EXE) sqlite3.c
   482    485   	./srcck1 sqlite3.c
   483    486   
   484    487   fuzzershell$(EXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h

Changes to src/backup.c.

   780    780     ** or an error code.
   781    781     */
   782    782     sqlite3_backup_step(&b, 0x7FFFFFFF);
   783    783     assert( b.rc!=SQLITE_OK );
   784    784     rc = sqlite3_backup_finish(&b);
   785    785     if( rc==SQLITE_OK ){
   786    786       pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED;
   787         -  }else{
   788         -    sqlite3PagerClearCache(sqlite3BtreePager(b.pDest));
   789    787     }
   790    788   
   791    789     assert( sqlite3BtreeIsInTrans(pTo)==0 );
   792    790   copy_finished:
   793    791     sqlite3BtreeLeave(pFrom);
   794    792     sqlite3BtreeLeave(pTo);
   795    793     return rc;
   796    794   }
   797    795   #endif /* SQLITE_OMIT_VACUUM */

Changes to src/btree.c.

  6074   6074   ** area.  pCell might point to some temporary storage.  The cell will
  6075   6075   ** be constructed in this temporary area then copied into pPage->aData
  6076   6076   ** later.
  6077   6077   */
  6078   6078   static int fillInCell(
  6079   6079     MemPage *pPage,                /* The page that contains the cell */
  6080   6080     unsigned char *pCell,          /* Complete text of the cell */
  6081         -  const void *pKey, i64 nKey,    /* The key */
  6082         -  const void *pData,int nData,   /* The data */
  6083         -  int nZero,                     /* Extra zero bytes to append to pData */
         6081  +  const BtreePayload *pX,        /* Payload with which to construct the cell */
  6084   6082     int *pnSize                    /* Write cell size here */
  6085   6083   ){
  6086   6084     int nPayload;
  6087   6085     const u8 *pSrc;
  6088   6086     int nSrc, n, rc;
  6089   6087     int spaceLeft;
  6090   6088     MemPage *pOvfl = 0;
................................................................................
  6100   6098     /* pPage is not necessarily writeable since pCell might be auxiliary
  6101   6099     ** buffer space that is separate from the pPage buffer area */
  6102   6100     assert( pCell<pPage->aData || pCell>=&pPage->aData[pBt->pageSize]
  6103   6101               || sqlite3PagerIswriteable(pPage->pDbPage) );
  6104   6102   
  6105   6103     /* Fill in the header. */
  6106   6104     nHeader = pPage->childPtrSize;
  6107         -  nPayload = nData + nZero;
  6108         -  if( pPage->intKeyLeaf ){
         6105  +  if( pPage->intKey ){
         6106  +    nPayload = pX->nData + pX->nZero;
         6107  +    pSrc = pX->pData;
         6108  +    nSrc = pX->nData;
         6109  +    assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */
  6109   6110       nHeader += putVarint32(&pCell[nHeader], nPayload);
         6111  +    nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey);
  6110   6112     }else{
  6111         -    assert( nData==0 );
  6112         -    assert( nZero==0 );
         6113  +    assert( pX->nData==0 );
         6114  +    assert( pX->nZero==0 );
         6115  +    assert( pX->nKey<=0x7fffffff && pX->pKey!=0 );
         6116  +    nSrc = nPayload = (int)pX->nKey;
         6117  +    pSrc = pX->pKey;
         6118  +    nHeader += putVarint32(&pCell[nHeader], nPayload);
  6113   6119     }
  6114         -  nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey);
  6115   6120     
  6116         -  /* Fill in the payload size */
  6117         -  if( pPage->intKey ){
  6118         -    pSrc = pData;
  6119         -    nSrc = nData;
  6120         -    nData = 0;
  6121         -  }else{ 
  6122         -    assert( nKey<=0x7fffffff && pKey!=0 );
  6123         -    nPayload = (int)nKey;
  6124         -    pSrc = pKey;
  6125         -    nSrc = (int)nKey;
  6126         -  }
         6121  +  /* Fill in the payload */
  6127   6122     if( nPayload<=pPage->maxLocal ){
  6128   6123       n = nHeader + nPayload;
  6129   6124       testcase( n==3 );
  6130   6125       testcase( n==4 );
  6131   6126       if( n<4 ) n = 4;
  6132   6127       *pnSize = n;
  6133   6128       spaceLeft = nPayload;
................................................................................
  6157   6152     ** were computed correctly.
  6158   6153     */
  6159   6154   #if SQLITE_DEBUG
  6160   6155     {
  6161   6156       CellInfo info;
  6162   6157       pPage->xParseCell(pPage, pCell, &info);
  6163   6158       assert( nHeader==(int)(info.pPayload - pCell) );
  6164         -    assert( info.nKey==nKey );
         6159  +    assert( info.nKey==pX->nKey );
  6165   6160       assert( *pnSize == info.nSize );
  6166   6161       assert( spaceLeft == info.nLocal );
  6167   6162     }
  6168   6163   #endif
  6169   6164   
  6170   6165     /* Write the payload into the local Cell and any extra into overflow pages */
  6171   6166     while( nPayload>0 ){
................................................................................
  6242   6237         memset(pPayload, 0, n);
  6243   6238       }
  6244   6239       nPayload -= n;
  6245   6240       pPayload += n;
  6246   6241       pSrc += n;
  6247   6242       nSrc -= n;
  6248   6243       spaceLeft -= n;
  6249         -    if( nSrc==0 ){
  6250         -      nSrc = nData;
  6251         -      pSrc = pData;
  6252         -    }
  6253   6244     }
  6254   6245     releasePage(pToRelease);
  6255   6246     return SQLITE_OK;
  6256   6247   }
  6257   6248   
  6258   6249   /*
  6259   6250   ** Remove the i-th cell from pPage.  This routine effects pPage only.
................................................................................
  6312   6303   ** If the cell content will fit on the page, then put it there.  If it
  6313   6304   ** will not fit, then make a copy of the cell content into pTemp if
  6314   6305   ** pTemp is not null.  Regardless of pTemp, allocate a new entry
  6315   6306   ** in pPage->apOvfl[] and make it point to the cell content (either
  6316   6307   ** in pTemp or the original pCell) and also record its index. 
  6317   6308   ** Allocating a new entry in pPage->aCell[] implies that 
  6318   6309   ** pPage->nOverflow is incremented.
         6310  +**
         6311  +** *pRC must be SQLITE_OK when this routine is called.
  6319   6312   */
  6320   6313   static void insertCell(
  6321   6314     MemPage *pPage,   /* Page into which we are copying */
  6322   6315     int i,            /* New cell becomes the i-th cell of the page */
  6323   6316     u8 *pCell,        /* Content of the new cell */
  6324   6317     int sz,           /* Bytes of content in pCell */
  6325   6318     u8 *pTemp,        /* Temp storage space for pCell, if needed */
................................................................................
  6327   6320     int *pRC          /* Read and write return code from here */
  6328   6321   ){
  6329   6322     int idx = 0;      /* Where to write new cell content in data[] */
  6330   6323     int j;            /* Loop counter */
  6331   6324     u8 *data;         /* The content of the whole page */
  6332   6325     u8 *pIns;         /* The point in pPage->aCellIdx[] where no cell inserted */
  6333   6326   
  6334         -  if( *pRC ) return;
  6335         -
         6327  +  assert( *pRC==SQLITE_OK );
  6336   6328     assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
  6337   6329     assert( MX_CELL(pPage->pBt)<=10921 );
  6338   6330     assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB );
  6339   6331     assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) );
  6340   6332     assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) );
  6341   6333     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  6342   6334     /* The cell should normally be sized correctly.  However, when moving a
................................................................................
  6834   6826       pCell = findCell(pPage, pPage->nCell-1);
  6835   6827       pStop = &pCell[9];
  6836   6828       while( (*(pCell++)&0x80) && pCell<pStop );
  6837   6829       pStop = &pCell[9];
  6838   6830       while( ((*(pOut++) = *(pCell++))&0x80) && pCell<pStop );
  6839   6831   
  6840   6832       /* Insert the new divider cell into pParent. */
  6841         -    insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace),
  6842         -               0, pPage->pgno, &rc);
         6833  +    if( rc==SQLITE_OK ){
         6834  +      insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace),
         6835  +                   0, pPage->pgno, &rc);
         6836  +    }
  6843   6837   
  6844   6838       /* Set the right-child pointer of pParent to point to the new page. */
  6845   6839       put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew);
  6846   6840     
  6847   6841       /* Release the reference to the new page. */
  6848   6842       releasePage(pNew);
  6849   6843     }
................................................................................
  7927   7921       sqlite3PageFree(pFree);
  7928   7922     }
  7929   7923     return rc;
  7930   7924   }
  7931   7925   
  7932   7926   
  7933   7927   /*
  7934         -** Insert a new record into the BTree.  The key is given by (pKey,nKey)
  7935         -** and the data is given by (pData,nData).  The cursor is used only to
  7936         -** define what table the record should be inserted into.  The cursor
  7937         -** is left pointing at a random location.
         7928  +** Insert a new record into the BTree.  The content of the new record
         7929  +** is described by the pX object.  The pCur cursor is used only to
         7930  +** define what table the record should be inserted into, and is left
         7931  +** pointing at a random location.
  7938   7932   **
  7939         -** For an INTKEY table, only the nKey value of the key is used.  pKey is
  7940         -** ignored.  For a ZERODATA table, the pData and nData are both ignored.
         7933  +** For a table btree (used for rowid tables), only the pX.nKey value of
         7934  +** the key is used. The pX.pKey value must be NULL.  The pX.nKey is the
         7935  +** rowid or INTEGER PRIMARY KEY of the row.  The pX.nData,pData,nZero fields
         7936  +** hold the content of the row.
         7937  +**
         7938  +** For an index btree (used for indexes and WITHOUT ROWID tables), the
         7939  +** key is an arbitrary byte sequence stored in pX.pKey,nKey.  The 
         7940  +** pX.pData,nData,nZero fields must be zero.
  7941   7941   **
  7942   7942   ** If the seekResult parameter is non-zero, then a successful call to
  7943   7943   ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already
  7944   7944   ** been performed. seekResult is the search result returned (a negative
  7945   7945   ** number if pCur points at an entry that is smaller than (pKey, nKey), or
  7946   7946   ** a positive value if pCur points at an entry that is larger than 
  7947   7947   ** (pKey, nKey)). 
................................................................................
  7950   7950   ** cursor pCur is pointing at the existing copy of a row that is to be
  7951   7951   ** overwritten.  If the seekResult parameter is 0, then cursor pCur may
  7952   7952   ** point to any entry or to no entry at all and so this function has to seek
  7953   7953   ** the cursor before the new key can be inserted.
  7954   7954   */
  7955   7955   int sqlite3BtreeInsert(
  7956   7956     BtCursor *pCur,                /* Insert data into the table of this cursor */
  7957         -  const void *pKey, i64 nKey,    /* The key of the new record */
  7958         -  const void *pData, int nData,  /* The data of the new record */
  7959         -  int nZero,                     /* Number of extra 0 bytes to append to data */
         7957  +  const BtreePayload *pX,        /* Content of the row to be inserted */
  7960   7958     int appendBias,                /* True if this is likely an append */
  7961   7959     int seekResult                 /* Result of prior MovetoUnpacked() call */
  7962   7960   ){
  7963   7961     int rc;
  7964   7962     int loc = seekResult;          /* -1: before desired location  +1: after */
  7965   7963     int szNew = 0;
  7966   7964     int idx;
................................................................................
  7982   7980     assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) );
  7983   7981   
  7984   7982     /* Assert that the caller has been consistent. If this cursor was opened
  7985   7983     ** expecting an index b-tree, then the caller should be inserting blob
  7986   7984     ** keys with no associated data. If the cursor was opened expecting an
  7987   7985     ** intkey table, the caller should be inserting integer keys with a
  7988   7986     ** blob of associated data.  */
  7989         -  assert( (pKey==0)==(pCur->pKeyInfo==0) );
         7987  +  assert( (pX->pKey==0)==(pCur->pKeyInfo==0) );
  7990   7988   
  7991   7989     /* Save the positions of any other cursors open on this table.
  7992   7990     **
  7993   7991     ** In some cases, the call to btreeMoveto() below is a no-op. For
  7994   7992     ** example, when inserting data into a table with auto-generated integer
  7995   7993     ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the 
  7996   7994     ** integer key to use. It then calls this function to actually insert the 
................................................................................
  8001   7999     */
  8002   8000     if( pCur->curFlags & BTCF_Multiple ){
  8003   8001       rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur);
  8004   8002       if( rc ) return rc;
  8005   8003     }
  8006   8004   
  8007   8005     if( pCur->pKeyInfo==0 ){
  8008         -    assert( pKey==0 );
         8006  +    assert( pX->pKey==0 );
  8009   8007       /* If this is an insert into a table b-tree, invalidate any incrblob 
  8010   8008       ** cursors open on the row being replaced */
  8011         -    invalidateIncrblobCursors(p, nKey, 0);
         8009  +    invalidateIncrblobCursors(p, pX->nKey, 0);
  8012   8010   
  8013   8011       /* If the cursor is currently on the last row and we are appending a
  8014   8012       ** new row onto the end, set the "loc" to avoid an unnecessary
  8015   8013       ** btreeMoveto() call */
  8016         -    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0
  8017         -      && pCur->info.nKey==nKey-1 ){
         8014  +    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0
         8015  +      && pCur->info.nKey==pX->nKey-1 ){
  8018   8016          loc = -1;
  8019   8017       }else if( loc==0 ){
  8020         -      rc = sqlite3BtreeMovetoUnpacked(pCur, 0, nKey, appendBias, &loc);
         8018  +      rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, appendBias, &loc);
  8021   8019         if( rc ) return rc;
  8022   8020       }
  8023   8021     }else if( loc==0 ){
  8024         -    rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc);
         8022  +    rc = btreeMoveto(pCur, pX->pKey, pX->nKey, appendBias, &loc);
  8025   8023       if( rc ) return rc;
  8026   8024     }
  8027   8025     assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) );
  8028   8026   
  8029   8027     pPage = pCur->apPage[pCur->iPage];
  8030         -  assert( pPage->intKey || nKey>=0 );
         8028  +  assert( pPage->intKey || pX->nKey>=0 );
  8031   8029     assert( pPage->leaf || !pPage->intKey );
  8032   8030   
  8033   8031     TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n",
  8034         -          pCur->pgnoRoot, nKey, nData, pPage->pgno,
         8032  +          pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno,
  8035   8033             loc==0 ? "overwrite" : "new entry"));
  8036   8034     assert( pPage->isInit );
  8037   8035     newCell = pBt->pTmpSpace;
  8038   8036     assert( newCell!=0 );
  8039         -  rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew);
         8037  +  rc = fillInCell(pPage, newCell, pX, &szNew);
  8040   8038     if( rc ) goto end_insert;
  8041   8039     assert( szNew==pPage->xCellSize(pPage, newCell) );
  8042   8040     assert( szNew <= MX_CELL_SIZE(pBt) );
  8043   8041     idx = pCur->aiIdx[pCur->iPage];
  8044   8042     if( loc==0 ){
  8045   8043       u16 szOld;
  8046   8044       assert( idx<pPage->nCell );
................................................................................
  8058   8056     }else if( loc<0 && pPage->nCell>0 ){
  8059   8057       assert( pPage->leaf );
  8060   8058       idx = ++pCur->aiIdx[pCur->iPage];
  8061   8059     }else{
  8062   8060       assert( pPage->leaf );
  8063   8061     }
  8064   8062     insertCell(pPage, idx, newCell, szNew, 0, 0, &rc);
         8063  +  assert( pPage->nOverflow==0 || rc==SQLITE_OK );
  8065   8064     assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 );
  8066   8065   
  8067   8066     /* If no error has occurred and pPage has an overflow cell, call balance() 
  8068   8067     ** to redistribute the cells within the tree. Since balance() may move
  8069   8068     ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey
  8070   8069     ** variables.
  8071   8070     **
................................................................................
  8081   8080     ** is advantageous to leave the cursor pointing to the last entry in
  8082   8081     ** the b-tree if possible. If the cursor is left pointing to the last
  8083   8082     ** entry in the table, and the next row inserted has an integer key
  8084   8083     ** larger than the largest existing key, it is possible to insert the
  8085   8084     ** row without seeking the cursor. This can be a big performance boost.
  8086   8085     */
  8087   8086     pCur->info.nSize = 0;
  8088         -  if( rc==SQLITE_OK && pPage->nOverflow ){
         8087  +  if( pPage->nOverflow ){
         8088  +    assert( rc==SQLITE_OK );
  8089   8089       pCur->curFlags &= ~(BTCF_ValidNKey);
  8090   8090       rc = balance(pCur);
  8091   8091   
  8092   8092       /* Must make sure nOverflow is reset to zero even if the balance()
  8093   8093       ** fails. Internal data structure corruption will result otherwise. 
  8094   8094       ** Also, set the cursor state to invalid. This stops saveCursorPosition()
  8095   8095       ** from trying to save the current position of the cursor.  */
................................................................................
  8217   8217       pCell = findCell(pLeaf, pLeaf->nCell-1);
  8218   8218       if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT;
  8219   8219       nCell = pLeaf->xCellSize(pLeaf, pCell);
  8220   8220       assert( MX_CELL_SIZE(pBt) >= nCell );
  8221   8221       pTmp = pBt->pTmpSpace;
  8222   8222       assert( pTmp!=0 );
  8223   8223       rc = sqlite3PagerWrite(pLeaf->pDbPage);
  8224         -    insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc);
         8224  +    if( rc==SQLITE_OK ){
         8225  +      insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc);
         8226  +    }
  8225   8227       dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc);
  8226   8228       if( rc ) return rc;
  8227   8229     }
  8228   8230   
  8229   8231     /* Balance the tree. If the entry deleted was located on a leaf page,
  8230   8232     ** then the cursor still points to that page. In this case the first
  8231   8233     ** call to balance() repairs the tree, and the if(...) condition is

Changes to src/btree.h.

    35     35   
    36     36   /*
    37     37   ** Forward declarations of structure
    38     38   */
    39     39   typedef struct Btree Btree;
    40     40   typedef struct BtCursor BtCursor;
    41     41   typedef struct BtShared BtShared;
           42  +typedef struct BtreePayload BtreePayload;
    42     43   
    43     44   
    44     45   int sqlite3BtreeOpen(
    45     46     sqlite3_vfs *pVfs,       /* VFS to use with this b-tree */
    46     47     const char *zFilename,   /* Name of database file to open */
    47     48     sqlite3 *db,             /* Associated database connection */
    48     49     Btree **ppBtree,         /* Return open Btree* here */
................................................................................
   246    247   int sqlite3BtreeCursorRestore(BtCursor*, int*);
   247    248   int sqlite3BtreeDelete(BtCursor*, u8 flags);
   248    249   
   249    250   /* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */
   250    251   #define BTREE_SAVEPOSITION 0x02  /* Leave cursor pointing at NEXT or PREV */
   251    252   #define BTREE_AUXDELETE    0x04  /* not the primary delete operation */
   252    253   
   253         -int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey,
   254         -                                  const void *pData, int nData,
   255         -                                  int nZero, int bias, int seekResult);
          254  +/* An instance of the BtreePayload object describes the content of a single
          255  +** entry in either an index or table btree.
          256  +**
          257  +** Index btrees (used for indexes and also WITHOUT ROWID tables) contain
          258  +** an arbitrary key and no data.  These btrees have pKey,nKey set to their
          259  +** key and pData,nData,nZero set to zero.
          260  +**
          261  +** Table btrees (used for rowid tables) contain an integer rowid used as
          262  +** the key and passed in the nKey field.  The pKey field is zero.  
          263  +** pData,nData hold the content of the new entry.  nZero extra zero bytes
          264  +** are appended to the end of the content when constructing the entry.
          265  +**
          266  +** This object is used to pass information into sqlite3BtreeInsert().  The
          267  +** same information used to be passed as five separate parameters.  But placing
          268  +** the information into this object helps to keep the interface more 
          269  +** organized and understandable, and it also helps the resulting code to
          270  +** run a little faster by using fewer registers for parameter passing.
          271  +*/
          272  +struct BtreePayload {
          273  +  const void *pKey;       /* Key content for indexes.  NULL for tables */
          274  +  sqlite3_int64 nKey;     /* Size of pKey for indexes.  PRIMARY KEY for tabs */
          275  +  const void *pData;      /* Data for tables.  NULL for indexes */
          276  +  int nData;              /* Size of pData.  0 if none. */
          277  +  int nZero;              /* Extra zero data appended after pData,nData */
          278  +};
          279  +
          280  +int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload,
          281  +                       int bias, int seekResult);
   256    282   int sqlite3BtreeFirst(BtCursor*, int *pRes);
   257    283   int sqlite3BtreeLast(BtCursor*, int *pRes);
   258    284   int sqlite3BtreeNext(BtCursor*, int *pRes);
   259    285   int sqlite3BtreeEof(BtCursor*);
   260    286   int sqlite3BtreePrevious(BtCursor*, int *pRes);
   261    287   int sqlite3BtreeKeySize(BtCursor*, i64 *pSize);
   262    288   int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*);

Changes to src/build.c.

  3361   3361       ** to invalidate all pre-compiled statements.
  3362   3362       */
  3363   3363       if( pTblName ){
  3364   3364         sqlite3RefillIndex(pParse, pIndex, iMem);
  3365   3365         sqlite3ChangeCookie(pParse, iDb);
  3366   3366         sqlite3VdbeAddParseSchemaOp(v, iDb,
  3367   3367            sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName));
  3368         -      sqlite3VdbeAddOp1(v, OP_Expire, 0);
         3368  +      sqlite3VdbeAddOp0(v, OP_Expire);
  3369   3369       }
  3370   3370   
  3371   3371       sqlite3VdbeJumpHere(v, pIndex->tnum);
  3372   3372     }
  3373   3373   
  3374   3374     /* When adding an index to the list of indices for a table, make
  3375   3375     ** sure all indices labeled OE_Replace come after all those labeled

Changes to src/pager.c.

  7191   7191   ** uses it opaquely as an argument to sqlite3BackupRestart() and
  7192   7192   ** sqlite3BackupUpdate() only.
  7193   7193   */
  7194   7194   sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){
  7195   7195     return &pPager->pBackup;
  7196   7196   }
  7197   7197   
  7198         -#ifndef SQLITE_OMIT_VACUUM
  7199         -/*
  7200         -** Unless this is an in-memory or temporary database, clear the pager cache.
  7201         -*/
  7202         -void sqlite3PagerClearCache(Pager *pPager){
  7203         -  assert( MEMDB==0 || pPager->tempFile );
  7204         -  if( pPager->tempFile==0 ) pager_reset(pPager);
  7205         -}
  7206         -#endif
  7207         -
  7208   7198   #ifndef SQLITE_OMIT_WAL
  7209   7199   /*
  7210   7200   ** This function is called when the user invokes "PRAGMA wal_checkpoint",
  7211   7201   ** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint()
  7212   7202   ** or wal_blocking_checkpoint() API functions.
  7213   7203   **
  7214   7204   ** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART.

Changes to src/pager.h.

   200    200   sqlite3_file *sqlite3PagerFile(Pager*);
   201    201   sqlite3_file *sqlite3PagerWalFile(Pager *pPager);
   202    202   sqlite3_file *sqlite3PagerJrnlFile(Pager*);
   203    203   const char *sqlite3PagerJournalname(Pager*);
   204    204   void *sqlite3PagerTempSpace(Pager*);
   205    205   int sqlite3PagerIsMemdb(Pager*);
   206    206   void sqlite3PagerCacheStat(Pager *, int, int, int *);
   207         -void sqlite3PagerClearCache(Pager *);
   208    207   int sqlite3SectorSize(sqlite3_file *);
   209    208   
   210    209   /* Functions used to truncate the database file. */
   211    210   void sqlite3PagerTruncateImage(Pager*,Pgno);
   212    211   
   213    212   void sqlite3PagerRekey(DbPage*, Pgno, u16);
   214    213   

Changes to src/pcache.c.

   683    683   */
   684    684   void sqlite3PcacheClear(PCache *pCache){
   685    685     sqlite3PcacheTruncate(pCache, 0);
   686    686   }
   687    687   
   688    688   /*
   689    689   ** Merge two lists of pages connected by pDirty and in pgno order.
   690         -** Do not both fixing the pDirtyPrev pointers.
          690  +** Do not bother fixing the pDirtyPrev pointers.
   691    691   */
   692    692   static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){
   693    693     PgHdr result, *pTail;
   694    694     pTail = &result;
   695         -  while( pA && pB ){
          695  +  assert( pA!=0 && pB!=0 );
          696  +  for(;;){
   696    697       if( pA->pgno<pB->pgno ){
   697    698         pTail->pDirty = pA;
   698    699         pTail = pA;
   699    700         pA = pA->pDirty;
          701  +      if( pA==0 ){
          702  +        pTail->pDirty = pB;
          703  +        break;
          704  +      }
   700    705       }else{
   701    706         pTail->pDirty = pB;
   702    707         pTail = pB;
   703    708         pB = pB->pDirty;
          709  +      if( pB==0 ){
          710  +        pTail->pDirty = pA;
          711  +        break;
          712  +      }
   704    713       }
   705    714     }
   706         -  if( pA ){
   707         -    pTail->pDirty = pA;
   708         -  }else if( pB ){
   709         -    pTail->pDirty = pB;
   710         -  }else{
   711         -    pTail->pDirty = 0;
   712         -  }
   713    715     return result.pDirty;
   714    716   }
   715    717   
   716    718   /*
   717    719   ** Sort the list of pages in accending order by pgno.  Pages are
   718    720   ** connected by pDirty pointers.  The pDirtyPrev pointers are
   719    721   ** corrupted by this sort.
................................................................................
   746    748         ** the input list.  But that is impossible.
   747    749         */
   748    750         a[i] = pcacheMergeDirtyList(a[i], p);
   749    751       }
   750    752     }
   751    753     p = a[0];
   752    754     for(i=1; i<N_SORT_BUCKET; i++){
   753         -    p = pcacheMergeDirtyList(p, a[i]);
          755  +    if( a[i]==0 ) continue;
          756  +    p = p ? pcacheMergeDirtyList(p, a[i]) : a[i];
   754    757     }
   755    758     return p;
   756    759   }
   757    760   
   758    761   /*
   759    762   ** Return a list of all dirty pages in the cache, sorted by page number.
   760    763   */

Changes to src/pragma.c.

  1026   1026           if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0;
  1027   1027         }
  1028   1028   
  1029   1029         /* Many of the flag-pragmas modify the code generated by the SQL 
  1030   1030         ** compiler (eg. count_changes). So add an opcode to expire all
  1031   1031         ** compiled SQL statements after modifying a pragma value.
  1032   1032         */
  1033         -      sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
         1033  +      sqlite3VdbeAddOp0(v, OP_Expire);
  1034   1034         setAllPagerFlags(db);
  1035   1035       }
  1036   1036       break;
  1037   1037     }
  1038   1038   #endif /* SQLITE_OMIT_FLAG_PRAGMAS */
  1039   1039   
  1040   1040   #ifndef SQLITE_OMIT_SCHEMA_PRAGMAS

Changes to src/rowset.c.

   238    238     struct RowSetEntry *pA,    /* First sorted list to be merged */
   239    239     struct RowSetEntry *pB     /* Second sorted list to be merged */
   240    240   ){
   241    241     struct RowSetEntry head;
   242    242     struct RowSetEntry *pTail;
   243    243   
   244    244     pTail = &head;
   245         -  while( pA && pB ){
          245  +  assert( pA!=0 && pB!=0 );
          246  +  for(;;){
   246    247       assert( pA->pRight==0 || pA->v<=pA->pRight->v );
   247    248       assert( pB->pRight==0 || pB->v<=pB->pRight->v );
   248         -    if( pA->v<pB->v ){
   249         -      pTail->pRight = pA;
          249  +    if( pA->v<=pB->v ){
          250  +      if( pA->v<pB->v ) pTail = pTail->pRight = pA;
   250    251         pA = pA->pRight;
   251         -      pTail = pTail->pRight;
   252         -    }else if( pB->v<pA->v ){
   253         -      pTail->pRight = pB;
   254         -      pB = pB->pRight;
   255         -      pTail = pTail->pRight;
          252  +      if( pA==0 ){
          253  +        pTail->pRight = pB;
          254  +        break;
          255  +      }
   256    256       }else{
   257         -      pA = pA->pRight;
          257  +      pTail = pTail->pRight = pB;
          258  +      pB = pB->pRight;
          259  +      if( pB==0 ){
          260  +        pTail->pRight = pA;
          261  +        break;
          262  +      }
   258    263       }
   259    264     }
   260         -  if( pA ){
   261         -    assert( pA->pRight==0 || pA->v<=pA->pRight->v );
   262         -    pTail->pRight = pA;
   263         -  }else{
   264         -    assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v );
   265         -    pTail->pRight = pB;
   266         -  }
   267    265     return head.pRight;
   268    266   }
   269    267   
   270    268   /*
   271    269   ** Sort all elements on the list of RowSetEntry objects into order of
   272    270   ** increasing v.
   273    271   */ 
................................................................................
   282    280       for(i=0; aBucket[i]; i++){
   283    281         pIn = rowSetEntryMerge(aBucket[i], pIn);
   284    282         aBucket[i] = 0;
   285    283       }
   286    284       aBucket[i] = pIn;
   287    285       pIn = pNext;
   288    286     }
   289         -  pIn = 0;
   290         -  for(i=0; i<sizeof(aBucket)/sizeof(aBucket[0]); i++){
   291         -    pIn = rowSetEntryMerge(pIn, aBucket[i]);
          287  +  pIn = aBucket[0];
          288  +  for(i=1; i<sizeof(aBucket)/sizeof(aBucket[0]); i++){
          289  +    if( aBucket[i]==0 ) continue;
          290  +    pIn = pIn ? rowSetEntryMerge(pIn, aBucket[i]) : aBucket[i];
   292    291     }
   293    292     return pIn;
   294    293   }
   295    294   
   296    295   
   297    296   /*
   298    297   ** The input, pIn, is a binary tree (or subtree) of RowSetEntry objects.

Changes to src/select.c.

    52     52     int nOBSat;           /* Number of ORDER BY terms satisfied by indices */
    53     53     int iECursor;         /* Cursor number for the sorter */
    54     54     int regReturn;        /* Register holding block-output return address */
    55     55     int labelBkOut;       /* Start label for the block-output subroutine */
    56     56     int addrSortIndex;    /* Address of the OP_SorterOpen or OP_OpenEphemeral */
    57     57     int labelDone;        /* Jump here when done, ex: LIMIT reached */
    58     58     u8 sortFlags;         /* Zero or more SORTFLAG_* bits */
           59  +  u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */
    59     60   };
    60     61   #define SORTFLAG_UseSorter  0x01   /* Use SorterOpen instead of OpenEphemeral */
    61     62   
    62     63   /*
    63     64   ** Delete all the content of a Select structure.  Deallocate the structure
    64     65   ** itself only if bFree is true.
    65     66   */
................................................................................
   585    586       op = OP_SorterInsert;
   586    587     }else{
   587    588       op = OP_IdxInsert;
   588    589     }
   589    590     sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord);
   590    591     if( iLimit ){
   591    592       int addr;
          593  +    int r1 = 0;
          594  +    /* Fill the sorter until it contains LIMIT+OFFSET entries.  (The iLimit
          595  +    ** register is initialized with value of LIMIT+OFFSET.)  After the sorter
          596  +    ** fills up, delete the least entry in the sorter after each insert.
          597  +    ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */
   592    598       addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v);
   593    599       sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
          600  +    if( pSort->bOrderedInnerLoop ){
          601  +      r1 = ++pParse->nMem;
          602  +      sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1);
          603  +      VdbeComment((v, "seq"));
          604  +    }
   594    605       sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
          606  +    if( pSort->bOrderedInnerLoop ){
          607  +      /* If the inner loop is driven by an index such that values from
          608  +      ** the same iteration of the inner loop are in sorted order, then
          609  +      ** immediately jump to the next iteration of an inner loop if the
          610  +      ** entry from the current iteration does not fit into the top
          611  +      ** LIMIT+OFFSET entries of the sorter. */
          612  +      int iBrk = sqlite3VdbeCurrentAddr(v) + 2;
          613  +      sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1);
          614  +      sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
          615  +      VdbeCoverage(v);
          616  +    }
   595    617       sqlite3VdbeJumpHere(v, addr);
   596    618     }
   597    619   }
   598    620   
   599    621   /*
   600    622   ** Add code to implement the OFFSET
   601    623   */
................................................................................
  5172   5194         p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo);
  5173   5195       }
  5174   5196       if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){
  5175   5197         sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo);
  5176   5198       }
  5177   5199       if( sSort.pOrderBy ){
  5178   5200         sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo);
         5201  +      sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo);
  5179   5202         if( sSort.nOBSat==sSort.pOrderBy->nExpr ){
  5180   5203           sSort.pOrderBy = 0;
  5181   5204         }
  5182   5205       }
  5183   5206   
  5184   5207       /* If sorting index that was created by a prior OP_OpenEphemeral 
  5185   5208       ** instruction ended up not being needed, then change the OP_OpenEphemeral

Changes to src/sqliteInt.h.

  2527   2527   ** Value constraints (enforced via assert()):
  2528   2528   **     WHERE_USE_LIMIT  == SF_FixedLimit
  2529   2529   */
  2530   2530   #define WHERE_ORDERBY_NORMAL   0x0000 /* No-op */
  2531   2531   #define WHERE_ORDERBY_MIN      0x0001 /* ORDER BY processing for min() func */
  2532   2532   #define WHERE_ORDERBY_MAX      0x0002 /* ORDER BY processing for max() func */
  2533   2533   #define WHERE_ONEPASS_DESIRED  0x0004 /* Want to do one-pass UPDATE/DELETE */
  2534         -#define WHERE_DUPLICATES_OK    0x0008 /* Ok to return a row more than once */
  2535         -#define WHERE_OMIT_OPEN_CLOSE  0x0010 /* Table cursors are already open */
  2536         -#define WHERE_FORCE_TABLE      0x0020 /* Do not use an index-only search */
  2537         -#define WHERE_ONETABLE_ONLY    0x0040 /* Only code the 1st table in pTabList */
  2538         -#define WHERE_NO_AUTOINDEX     0x0080 /* Disallow automatic indexes */
  2539         -#define WHERE_GROUPBY          0x0100 /* pOrderBy is really a GROUP BY */
  2540         -#define WHERE_DISTINCTBY       0x0200 /* pOrderby is really a DISTINCT clause */
  2541         -#define WHERE_WANT_DISTINCT    0x0400 /* All output needs to be distinct */
  2542         -#define WHERE_SORTBYGROUP      0x0800 /* Support sqlite3WhereIsSorted() */
  2543         -#define WHERE_REOPEN_IDX       0x1000 /* Try to use OP_ReopenIdx */
  2544         -#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */
  2545         -#define WHERE_USE_LIMIT        0x4000 /* There is a constant LIMIT clause */
  2546         -#define WHERE_SEEK_TABLE       0x8000 /* Do not defer seeks on main table */
         2534  +#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */
         2535  +#define WHERE_DUPLICATES_OK    0x0010 /* Ok to return a row more than once */
         2536  +#define WHERE_OR_SUBCLAUSE     0x0020 /* Processing a sub-WHERE as part of
         2537  +                                      ** the OR optimization  */
         2538  +#define WHERE_GROUPBY          0x0040 /* pOrderBy is really a GROUP BY */
         2539  +#define WHERE_DISTINCTBY       0x0080 /* pOrderby is really a DISTINCT clause */
         2540  +#define WHERE_WANT_DISTINCT    0x0100 /* All output needs to be distinct */
         2541  +#define WHERE_SORTBYGROUP      0x0200 /* Support sqlite3WhereIsSorted() */
         2542  +#define WHERE_SEEK_TABLE       0x0400 /* Do not defer seeks on main table */
         2543  +#define WHERE_ORDERBY_LIMIT    0x0800 /* ORDERBY+LIMIT on the inner loop */
         2544  +                        /*     0x1000    not currently used */
         2545  +                        /*     0x2000    not currently used */
         2546  +#define WHERE_USE_LIMIT        0x4000 /* Use the LIMIT in cost estimates */
         2547  +                        /*     0x8000    not currently used */
  2547   2548   
  2548   2549   /* Allowed return values from sqlite3WhereIsDistinct()
  2549   2550   */
  2550   2551   #define WHERE_DISTINCT_NOOP      0  /* DISTINCT keyword not used */
  2551   2552   #define WHERE_DISTINCT_UNIQUE    1  /* No duplicates */
  2552   2553   #define WHERE_DISTINCT_ORDERED   2  /* All duplicates are adjacent */
  2553   2554   #define WHERE_DISTINCT_UNORDERED 3  /* Duplicates are scattered */
................................................................................
  3632   3633   void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
  3633   3634   void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
  3634   3635   WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
  3635   3636   void sqlite3WhereEnd(WhereInfo*);
  3636   3637   LogEst sqlite3WhereOutputRowCount(WhereInfo*);
  3637   3638   int sqlite3WhereIsDistinct(WhereInfo*);
  3638   3639   int sqlite3WhereIsOrdered(WhereInfo*);
         3640  +int sqlite3WhereOrderedInnerLoop(WhereInfo*);
  3639   3641   int sqlite3WhereIsSorted(WhereInfo*);
  3640   3642   int sqlite3WhereContinueLabel(WhereInfo*);
  3641   3643   int sqlite3WhereBreakLabel(WhereInfo*);
  3642   3644   int sqlite3WhereOkOnePass(WhereInfo*, int*);
  3643   3645   #define ONEPASS_OFF      0        /* Use of ONEPASS not allowed */
  3644   3646   #define ONEPASS_SINGLE   1        /* ONEPASS valid for a single row update */
  3645   3647   #define ONEPASS_MULTI    2        /* ONEPASS is valid for multiple rows */

Changes to src/test3.c.

   614    614     ClientData clientData,
   615    615     Tcl_Interp *interp,
   616    616     int objc,
   617    617     Tcl_Obj *const objv[]
   618    618   ){
   619    619     BtCursor *pCur;
   620    620     int rc;
   621         -  void *pKey = 0;
   622         -  int nKey = 0;
   623         -  void *pData = 0;
   624         -  int nData = 0;
          621  +  BtreePayload x;
   625    622   
   626    623     if( objc!=4 && objc!=3 ){
   627    624       Tcl_WrongNumArgs(interp, 1, objv, "?-intkey? CSR KEY VALUE");
   628    625       return TCL_ERROR;
   629    626     }
   630    627   
          628  +  memset(&x, 0, sizeof(x));
   631    629     if( objc==4 ){
   632         -    if( Tcl_GetIntFromObj(interp, objv[2], &nKey) ) return TCL_ERROR;
   633         -    pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &nData);
          630  +    if( Tcl_GetIntFromObj(interp, objv[2], &rc) ) return TCL_ERROR;
          631  +    x.nKey = rc;
          632  +    x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &x.nData);
   634    633     }else{
   635         -    pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &nKey);
          634  +    x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &rc);
          635  +    x.nKey = rc;
   636    636     }
   637    637     pCur = (BtCursor*)sqlite3TestTextToPtr(Tcl_GetString(objv[1]));
   638    638   
   639    639     sqlite3_mutex_enter(pCur->pBtree->db->mutex);
   640    640     sqlite3BtreeEnter(pCur->pBtree);
   641         -  rc = sqlite3BtreeInsert(pCur, pKey, nKey, pData, nData, 0, 0, 0);
          641  +  rc = sqlite3BtreeInsert(pCur, &x, 0, 0);
   642    642     sqlite3BtreeLeave(pCur->pBtree);
   643    643     sqlite3_mutex_leave(pCur->pBtree->db->mutex);
   644    644   
   645    645     Tcl_ResetResult(interp);
   646    646     if( rc ){
   647    647       Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
   648    648       return TCL_ERROR;

Changes to src/vdbe.c.

  2400   2400     Mem *pReg;         /* PseudoTable input register */
  2401   2401   
  2402   2402     pC = p->apCsr[pOp->p1];
  2403   2403     p2 = pOp->p2;
  2404   2404   
  2405   2405     /* If the cursor cache is stale, bring it up-to-date */
  2406   2406     rc = sqlite3VdbeCursorMoveto(&pC, &p2);
         2407  +  if( rc ) goto abort_due_to_error;
  2407   2408   
  2408   2409     assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
  2409   2410     pDest = &aMem[pOp->p3];
  2410   2411     memAboutToChange(p, pDest);
  2411   2412     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  2412   2413     assert( pC!=0 );
  2413   2414     assert( p2<pC->nField );
  2414   2415     aOffset = pC->aOffset;
  2415   2416     assert( pC->eCurType!=CURTYPE_VTAB );
  2416   2417     assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow );
  2417   2418     assert( pC->eCurType!=CURTYPE_SORTER );
  2418   2419     pCrsr = pC->uc.pCursor;
  2419   2420   
  2420         -  if( rc ) goto abort_due_to_error;
  2421         -  if( pC->cacheStatus!=p->cacheCtr ){
         2421  +  if( pC->cacheStatus!=p->cacheCtr ){                /*OPTIMIZATION-IF-FALSE*/
  2422   2422       if( pC->nullRow ){
  2423   2423         if( pC->eCurType==CURTYPE_PSEUDO ){
  2424   2424           assert( pC->uc.pseudoTableReg>0 );
  2425   2425           pReg = &aMem[pC->uc.pseudoTableReg];
  2426   2426           assert( pReg->flags & MEM_Blob );
  2427   2427           assert( memIsValid(pReg) );
  2428   2428           pC->payloadSize = pC->szRow = avail = pReg->n;
................................................................................
  2461   2461       }
  2462   2462       pC->cacheStatus = p->cacheCtr;
  2463   2463       pC->iHdrOffset = getVarint32(pC->aRow, offset);
  2464   2464       pC->nHdrParsed = 0;
  2465   2465       aOffset[0] = offset;
  2466   2466   
  2467   2467   
  2468         -    if( avail<offset ){
         2468  +    if( avail<offset ){      /*OPTIMIZATION-IF-FALSE*/
  2469   2469         /* pC->aRow does not have to hold the entire row, but it does at least
  2470   2470         ** need to cover the header of the record.  If pC->aRow does not contain
  2471   2471         ** the complete header, then set it to zero, forcing the header to be
  2472   2472         ** dynamically allocated. */
  2473   2473         pC->aRow = 0;
  2474   2474         pC->szRow = 0;
  2475   2475   
................................................................................
  2482   2482         ** 3-byte type for each of the maximum of 32768 columns plus three
  2483   2483         ** extra bytes for the header length itself.  32768*3 + 3 = 98307.
  2484   2484         */
  2485   2485         if( offset > 98307 || offset > pC->payloadSize ){
  2486   2486           rc = SQLITE_CORRUPT_BKPT;
  2487   2487           goto abort_due_to_error;
  2488   2488         }
         2489  +    }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/
         2490  +      /* The following goto is an optimization.  It can be omitted and
         2491  +      ** everything will still work.  But OP_Column is measurably faster
         2492  +      ** by skipping the subsequent conditional, which is always true.
         2493  +      */
         2494  +      zData = pC->aRow;
         2495  +      assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
         2496  +      goto op_column_read_header;
  2489   2497       }
  2490         -
  2491         -    /* The following goto is an optimization.  It can be omitted and
  2492         -    ** everything will still work.  But OP_Column is measurably faster
  2493         -    ** by skipping the subsequent conditional, which is always true.
  2494         -    */
  2495         -    assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
  2496         -    goto op_column_read_header;
  2497   2498     }
  2498   2499   
  2499   2500     /* Make sure at least the first p2+1 entries of the header have been
  2500   2501     ** parsed and valid information is in aOffset[] and pC->aType[].
  2501   2502     */
  2502   2503     if( pC->nHdrParsed<=p2 ){
  2503   2504       /* If there is more header available for parsing in the record, try
  2504   2505       ** to extract additional fields up through the p2+1-th field 
  2505   2506       */
  2506         -    op_column_read_header:
  2507   2507       if( pC->iHdrOffset<aOffset[0] ){
  2508   2508         /* Make sure zData points to enough of the record to cover the header. */
  2509   2509         if( pC->aRow==0 ){
  2510   2510           memset(&sMem, 0, sizeof(sMem));
  2511   2511           rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], !pC->isTable, &sMem);
  2512   2512           if( rc!=SQLITE_OK ) goto abort_due_to_error;
  2513   2513           zData = (u8*)sMem.z;
  2514   2514         }else{
  2515   2515           zData = pC->aRow;
  2516   2516         }
  2517   2517     
  2518   2518         /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */
         2519  +    op_column_read_header:
  2519   2520         i = pC->nHdrParsed;
  2520   2521         offset64 = aOffset[i];
  2521   2522         zHdr = zData + pC->iHdrOffset;
  2522   2523         zEndHdr = zData + aOffset[0];
  2523         -      assert( i<=p2 && zHdr<zEndHdr );
  2524   2524         do{
  2525   2525           if( (t = zHdr[0])<0x80 ){
  2526   2526             zHdr++;
  2527   2527             offset64 += sqlite3VdbeOneByteSerialTypeLen(t);
  2528   2528           }else{
  2529   2529             zHdr += sqlite3GetVarint32(zHdr, &t);
  2530   2530             offset64 += sqlite3VdbeSerialTypeLen(t);
  2531   2531           }
  2532   2532           pC->aType[i++] = t;
  2533   2533           aOffset[i] = (u32)(offset64 & 0xffffffff);
  2534   2534         }while( i<=p2 && zHdr<zEndHdr );
  2535         -      pC->nHdrParsed = i;
  2536         -      pC->iHdrOffset = (u32)(zHdr - zData);
  2537         -  
         2535  +
  2538   2536         /* The record is corrupt if any of the following are true:
  2539   2537         ** (1) the bytes of the header extend past the declared header size
  2540   2538         ** (2) the entire header was used but not all data was used
  2541   2539         ** (3) the end of the data extends beyond the end of the record.
  2542   2540         */
  2543   2541         if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize))
  2544   2542          || (offset64 > pC->payloadSize)
  2545   2543         ){
  2546   2544           if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
  2547   2545           rc = SQLITE_CORRUPT_BKPT;
  2548   2546           goto abort_due_to_error;
  2549   2547         }
         2548  +
         2549  +      pC->nHdrParsed = i;
         2550  +      pC->iHdrOffset = (u32)(zHdr - zData);
  2550   2551         if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
  2551         -
  2552   2552       }else{
  2553   2553         t = 0;
  2554   2554       }
  2555   2555   
  2556   2556       /* If after trying to extract new entries from the header, nHdrParsed is
  2557   2557       ** still not up to p2, that means that the record has fewer than p2
  2558   2558       ** columns.  So the result will be either the default value or a NULL.
................................................................................
  2572   2572     /* Extract the content for the p2+1-th column.  Control can only
  2573   2573     ** reach this point if aOffset[p2], aOffset[p2+1], and pC->aType[p2] are
  2574   2574     ** all valid.
  2575   2575     */
  2576   2576     assert( p2<pC->nHdrParsed );
  2577   2577     assert( rc==SQLITE_OK );
  2578   2578     assert( sqlite3VdbeCheckMemInvariants(pDest) );
  2579         -  if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest);
         2579  +  if( VdbeMemDynamic(pDest) ){
         2580  +    sqlite3VdbeMemSetNull(pDest);
         2581  +  }
  2580   2582     assert( t==pC->aType[p2] );
  2581         -  pDest->enc = encoding;
  2582   2583     if( pC->szRow>=aOffset[p2+1] ){
  2583   2584       /* This is the common case where the desired content fits on the original
  2584   2585       ** page - where the content is not on an overflow page */
  2585   2586       zData = pC->aRow + aOffset[p2];
  2586   2587       if( t<12 ){
  2587   2588         sqlite3VdbeSerialGet(zData, t, pDest);
  2588   2589       }else{
  2589   2590         /* If the column value is a string, we need a persistent value, not
  2590   2591         ** a MEM_Ephem value.  This branch is a fast short-cut that is equivalent
  2591   2592         ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize().
  2592   2593         */
  2593   2594         static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term };
  2594   2595         pDest->n = len = (t-12)/2;
         2596  +      pDest->enc = encoding;
  2595   2597         if( pDest->szMalloc < len+2 ){
  2596   2598           pDest->flags = MEM_Null;
  2597   2599           if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem;
  2598   2600         }else{
  2599   2601           pDest->z = pDest->zMalloc;
  2600   2602         }
  2601   2603         memcpy(pDest->z, zData, len);
  2602   2604         pDest->z[len] = 0;
  2603   2605         pDest->z[len+1] = 0;
  2604   2606         pDest->flags = aFlag[t&1];
  2605   2607       }
  2606   2608     }else{
         2609  +    pDest->enc = encoding;
  2607   2610       /* This branch happens only when content is on overflow pages */
  2608   2611       if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
  2609   2612             && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
  2610   2613        || (len = sqlite3VdbeSerialTypeLen(t))==0
  2611   2614       ){
  2612   2615         /* Content is irrelevant for
  2613   2616         **    1. the typeof() function,
................................................................................
  4279   4282   ** This works exactly like OP_Insert except that the key is the
  4280   4283   ** integer value P3, not the value of the integer stored in register P3.
  4281   4284   */
  4282   4285   case OP_Insert: 
  4283   4286   case OP_InsertInt: {
  4284   4287     Mem *pData;       /* MEM cell holding data for the record to be inserted */
  4285   4288     Mem *pKey;        /* MEM cell holding key  for the record */
  4286         -  i64 iKey;         /* The integer ROWID or key for the record to be inserted */
  4287   4289     VdbeCursor *pC;   /* Cursor to table into which insert is written */
  4288         -  int nZero;        /* Number of zero-bytes to append */
  4289   4290     int seekResult;   /* Result of prior seek or 0 if no USESEEKRESULT flag */
  4290   4291     const char *zDb;  /* database name - used by the update hook */
  4291   4292     Table *pTab;      /* Table structure - used by update and pre-update hooks */
  4292   4293     int op;           /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
         4294  +  BtreePayload x;   /* Payload to be inserted */
  4293   4295   
  4294   4296     op = 0;
  4295   4297     pData = &aMem[pOp->p2];
  4296   4298     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  4297   4299     assert( memIsValid(pData) );
  4298   4300     pC = p->apCsr[pOp->p1];
  4299   4301     assert( pC!=0 );
................................................................................
  4304   4306     REGISTER_TRACE(pOp->p2, pData);
  4305   4307   
  4306   4308     if( pOp->opcode==OP_Insert ){
  4307   4309       pKey = &aMem[pOp->p3];
  4308   4310       assert( pKey->flags & MEM_Int );
  4309   4311       assert( memIsValid(pKey) );
  4310   4312       REGISTER_TRACE(pOp->p3, pKey);
  4311         -    iKey = pKey->u.i;
         4313  +    x.nKey = pKey->u.i;
  4312   4314     }else{
  4313   4315       assert( pOp->opcode==OP_InsertInt );
  4314         -    iKey = pOp->p3;
         4316  +    x.nKey = pOp->p3;
  4315   4317     }
  4316   4318   
  4317   4319     if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){
  4318   4320       assert( pC->isTable );
  4319   4321       assert( pC->iDb>=0 );
  4320   4322       zDb = db->aDb[pC->iDb].zName;
  4321   4323       pTab = pOp->p4.pTab;
................................................................................
  4328   4330   
  4329   4331   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
  4330   4332     /* Invoke the pre-update hook, if any */
  4331   4333     if( db->xPreUpdateCallback 
  4332   4334      && pOp->p4type==P4_TABLE
  4333   4335      && !(pOp->p5 & OPFLAG_ISUPDATE)
  4334   4336     ){
  4335         -    sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, iKey, pOp->p2);
         4337  +    sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2);
  4336   4338     }
  4337   4339   #endif
  4338   4340   
  4339   4341     if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++;
  4340         -  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey;
         4342  +  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = x.nKey;
  4341   4343     if( pData->flags & MEM_Null ){
  4342         -    pData->z = 0;
  4343         -    pData->n = 0;
         4344  +    x.pData = 0;
         4345  +    x.nData = 0;
  4344   4346     }else{
  4345   4347       assert( pData->flags & (MEM_Blob|MEM_Str) );
         4348  +    x.pData = pData->z;
         4349  +    x.nData = pData->n;
  4346   4350     }
  4347   4351     seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0);
  4348   4352     if( pData->flags & MEM_Zero ){
  4349         -    nZero = pData->u.nZero;
         4353  +    x.nZero = pData->u.nZero;
  4350   4354     }else{
  4351         -    nZero = 0;
         4355  +    x.nZero = 0;
  4352   4356     }
  4353         -  rc = sqlite3BtreeInsert(pC->uc.pCursor, 0, iKey,
  4354         -                          pData->z, pData->n, nZero,
         4357  +  x.pKey = 0;
         4358  +  rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
  4355   4359                             (pOp->p5 & OPFLAG_APPEND)!=0, seekResult
  4356   4360     );
  4357   4361     pC->deferredMoveto = 0;
  4358   4362     pC->cacheStatus = CACHE_STALE;
  4359   4363   
  4360   4364     /* Invoke the update-hook if required. */
  4361   4365     if( rc ) goto abort_due_to_error;
  4362   4366     if( db->xUpdateCallback && op ){
  4363         -    db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, iKey);
         4367  +    db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey);
  4364   4368     }
  4365   4369     break;
  4366   4370   }
  4367   4371   
  4368   4372   /* Opcode: Delete P1 P2 P3 P4 P5
  4369   4373   **
  4370   4374   ** Delete the record at which the P1 cursor is currently pointing.
................................................................................
  4971   4975   **
  4972   4976   ** This instruction only works for indices.  The equivalent instruction
  4973   4977   ** for tables is OP_Insert.
  4974   4978   */
  4975   4979   case OP_SorterInsert:       /* in2 */
  4976   4980   case OP_IdxInsert: {        /* in2 */
  4977   4981     VdbeCursor *pC;
  4978         -  int nKey;
  4979         -  const char *zKey;
         4982  +  BtreePayload x;
  4980   4983   
  4981   4984     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  4982   4985     pC = p->apCsr[pOp->p1];
  4983   4986     assert( pC!=0 );
  4984   4987     assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) );
  4985   4988     pIn2 = &aMem[pOp->p2];
  4986   4989     assert( pIn2->flags & MEM_Blob );
................................................................................
  4988   4991     assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert );
  4989   4992     assert( pC->isTable==0 );
  4990   4993     rc = ExpandBlob(pIn2);
  4991   4994     if( rc ) goto abort_due_to_error;
  4992   4995     if( pOp->opcode==OP_SorterInsert ){
  4993   4996       rc = sqlite3VdbeSorterWrite(pC, pIn2);
  4994   4997     }else{
  4995         -    nKey = pIn2->n;
  4996         -    zKey = pIn2->z;
  4997         -    rc = sqlite3BtreeInsert(pC->uc.pCursor, zKey, nKey, "", 0, 0, pOp->p3, 
         4998  +    x.nKey = pIn2->n;
         4999  +    x.pKey = pIn2->z;
         5000  +    x.nData = 0;
         5001  +    x.nZero = 0;
         5002  +    x.pData = 0;
         5003  +    rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, pOp->p3, 
  4998   5004           ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
  4999   5005           );
  5000   5006       assert( pC->deferredMoveto==0 );
  5001   5007       pC->cacheStatus = CACHE_STALE;
  5002   5008     }
  5003   5009     if( rc) goto abort_due_to_error;
  5004   5010     break;

Changes to src/vdbeaux.c.

   787    787   
   788    788   
   789    789   /*
   790    790   ** If the input FuncDef structure is ephemeral, then free it.  If
   791    791   ** the FuncDef is not ephermal, then do nothing.
   792    792   */
   793    793   static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){
   794         -  if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
          794  +  if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
   795    795       sqlite3DbFree(db, pDef);
   796    796     }
   797    797   }
   798    798   
   799    799   static void vdbeFreeOpArray(sqlite3 *, Op *, int);
   800    800   
   801    801   /*
   802    802   ** Delete a P4 value if necessary.
   803    803   */
          804  +static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){
          805  +  if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
          806  +  sqlite3DbFree(db, p);
          807  +}
          808  +static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){
          809  +  freeEphemeralFunction(db, p->pFunc);
          810  +  sqlite3DbFree(db, p);
          811  +}
   804    812   static void freeP4(sqlite3 *db, int p4type, void *p4){
   805    813     assert( db );
   806    814     switch( p4type ){
   807    815       case P4_FUNCCTX: {
   808         -      freeEphemeralFunction(db, ((sqlite3_context*)p4)->pFunc);
   809         -      /* Fall through into the next case */
          816  +      freeP4FuncCtx(db, (sqlite3_context*)p4);
          817  +      break;
   810    818       }
   811    819       case P4_REAL:
   812    820       case P4_INT64:
   813    821       case P4_DYNAMIC:
   814    822       case P4_INTARRAY: {
   815    823         sqlite3DbFree(db, p4);
   816    824         break;
................................................................................
   833    841         freeEphemeralFunction(db, (FuncDef*)p4);
   834    842         break;
   835    843       }
   836    844       case P4_MEM: {
   837    845         if( db->pnBytesFreed==0 ){
   838    846           sqlite3ValueFree((sqlite3_value*)p4);
   839    847         }else{
   840         -        Mem *p = (Mem*)p4;
   841         -        if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
   842         -        sqlite3DbFree(db, p);
          848  +        freeP4Mem(db, (Mem*)p4);
   843    849         }
   844    850         break;
   845    851       }
   846    852       case P4_VTAB : {
   847    853         if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
   848    854         break;
   849    855       }

Changes to src/vdbesort.c.

  1338   1338     }
  1339   1339     return SQLITE_OK;
  1340   1340   }
  1341   1341   
  1342   1342   
  1343   1343   /*
  1344   1344   ** Merge the two sorted lists p1 and p2 into a single list.
  1345         -** Set *ppOut to the head of the new list.
  1346   1345   */
  1347         -static void vdbeSorterMerge(
         1346  +static SorterRecord *vdbeSorterMerge(
  1348   1347     SortSubtask *pTask,             /* Calling thread context */
  1349   1348     SorterRecord *p1,               /* First list to merge */
  1350         -  SorterRecord *p2,               /* Second list to merge */
  1351         -  SorterRecord **ppOut            /* OUT: Head of merged list */
         1349  +  SorterRecord *p2                /* Second list to merge */
  1352   1350   ){
  1353   1351     SorterRecord *pFinal = 0;
  1354   1352     SorterRecord **pp = &pFinal;
  1355   1353     int bCached = 0;
  1356   1354   
  1357         -  while( p1 && p2 ){
         1355  +  assert( p1!=0 && p2!=0 );
         1356  +  for(;;){
  1358   1357       int res;
  1359   1358       res = pTask->xCompare(
  1360   1359           pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal
  1361   1360       );
  1362   1361   
  1363   1362       if( res<=0 ){
  1364   1363         *pp = p1;
  1365   1364         pp = &p1->u.pNext;
  1366   1365         p1 = p1->u.pNext;
         1366  +      if( p1==0 ){
         1367  +        *pp = p2;
         1368  +        break;
         1369  +      }
  1367   1370       }else{
  1368   1371         *pp = p2;
  1369   1372         pp = &p2->u.pNext;
  1370   1373         p2 = p2->u.pNext;
  1371   1374         bCached = 0;
         1375  +      if( p2==0 ){
         1376  +        *pp = p1;
         1377  +        break;
         1378  +      }
  1372   1379       }
  1373   1380     }
  1374         -  *pp = p1 ? p1 : p2;
  1375         -  *ppOut = pFinal;
         1381  +  return pFinal;
  1376   1382   }
  1377   1383   
  1378   1384   /*
  1379   1385   ** Return the SorterCompare function to compare values collected by the
  1380   1386   ** sorter object passed as the only argument.
  1381   1387   */
  1382   1388   static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){
................................................................................
  1421   1427         }
  1422   1428       }else{
  1423   1429         pNext = p->u.pNext;
  1424   1430       }
  1425   1431   
  1426   1432       p->u.pNext = 0;
  1427   1433       for(i=0; aSlot[i]; i++){
  1428         -      vdbeSorterMerge(pTask, p, aSlot[i], &p);
         1434  +      p = vdbeSorterMerge(pTask, p, aSlot[i]);
  1429   1435         aSlot[i] = 0;
  1430   1436       }
  1431   1437       aSlot[i] = p;
  1432   1438       p = pNext;
  1433   1439     }
  1434   1440   
  1435   1441     p = 0;
  1436   1442     for(i=0; i<64; i++){
  1437         -    vdbeSorterMerge(pTask, p, aSlot[i], &p);
         1443  +    if( aSlot[i]==0 ) continue;
         1444  +    p = p ? vdbeSorterMerge(pTask, p, aSlot[i]) : aSlot[i];
  1438   1445     }
  1439   1446     pList->pList = p;
  1440   1447   
  1441   1448     sqlite3_free(aSlot);
  1442   1449     assert( pTask->pUnpacked->errCode==SQLITE_OK 
  1443   1450          || pTask->pUnpacked->errCode==SQLITE_NOMEM 
  1444   1451     );

Changes to src/vtab.c.

   414    414         zStmt,
   415    415         pParse->regRowid
   416    416       );
   417    417       sqlite3DbFree(db, zStmt);
   418    418       v = sqlite3GetVdbe(pParse);
   419    419       sqlite3ChangeCookie(pParse, iDb);
   420    420   
   421         -    sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
          421  +    sqlite3VdbeAddOp0(v, OP_Expire);
   422    422       zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
   423    423       sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
   424    424   
   425    425       iReg = ++pParse->nMem;
   426    426       sqlite3VdbeLoadString(v, iReg, pTab->zName);
   427    427       sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg);
   428    428     }

Changes to src/where.c.

    46     46   /*
    47     47   ** Return TRUE if the WHERE clause returns rows in ORDER BY order.
    48     48   ** Return FALSE if the output needs to be sorted.
    49     49   */
    50     50   int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
    51     51     return pWInfo->nOBSat;
    52     52   }
           53  +
           54  +/*
           55  +** Return TRUE if the innermost loop of the WHERE clause implementation
           56  +** returns rows in ORDER BY order for complete run of the inner loop.
           57  +**
           58  +** Across multiple iterations of outer loops, the output rows need not be
           59  +** sorted.  As long as rows are sorted for just the innermost loop, this
           60  +** routine can return TRUE.
           61  +*/
           62  +int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){
           63  +  return pWInfo->bOrderedInnerLoop;
           64  +}
    53     65   
    54     66   /*
    55     67   ** Return the VDBE address or label to jump to in order to continue
    56     68   ** immediately with the next row of a WHERE clause.
    57     69   */
    58     70   int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
    59     71     assert( pWInfo->iContinue!=0 );
................................................................................
  2631   2643     }
  2632   2644     rSize = pTab->nRowLogEst;
  2633   2645     rLogSize = estLog(rSize);
  2634   2646   
  2635   2647   #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
  2636   2648     /* Automatic indexes */
  2637   2649     if( !pBuilder->pOrSet      /* Not part of an OR optimization */
  2638         -   && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0
         2650  +   && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
  2639   2651      && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
  2640   2652      && pSrc->pIBIndex==0      /* Has no INDEXED BY clause */
  2641   2653      && !pSrc->fg.notIndexed   /* Has no NOT INDEXED clause */
  2642   2654      && HasRowid(pTab)         /* Not WITHOUT ROWID table. (FIXME: Why not?) */
  2643   2655      && !pSrc->fg.isCorrelated /* Not a correlated subquery */
  2644   2656      && !pSrc->fg.isRecursive  /* Not a recursive common table expression. */
  2645   2657     ){
................................................................................
  3246   3258   ** the pOrderBy terms can be matched in any order.  With ORDER BY, the 
  3247   3259   ** pOrderBy terms must be matched in strict left-to-right order.
  3248   3260   */
  3249   3261   static i8 wherePathSatisfiesOrderBy(
  3250   3262     WhereInfo *pWInfo,    /* The WHERE clause */
  3251   3263     ExprList *pOrderBy,   /* ORDER BY or GROUP BY or DISTINCT clause to check */
  3252   3264     WherePath *pPath,     /* The WherePath to check */
  3253         -  u16 wctrlFlags,       /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */
         3265  +  u16 wctrlFlags,       /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
  3254   3266     u16 nLoop,            /* Number of entries in pPath->aLoop[] */
  3255   3267     WhereLoop *pLast,     /* Add this WhereLoop to the end of pPath->aLoop[] */
  3256   3268     Bitmask *pRevMask     /* OUT: Mask of WhereLoops to run in reverse order */
  3257   3269   ){
  3258   3270     u8 revSet;            /* True if rev is known */
  3259   3271     u8 rev;               /* Composite sort order */
  3260   3272     u8 revIdx;            /* Index sort order */
  3261   3273     u8 isOrderDistinct;   /* All prior WhereLoops are order-distinct */
  3262   3274     u8 distinctColumns;   /* True if the loop has UNIQUE NOT NULL columns */
  3263   3275     u8 isMatch;           /* iColumn matches a term of the ORDER BY clause */
         3276  +  u16 eqOpMask;         /* Allowed equality operators */
  3264   3277     u16 nKeyCol;          /* Number of key columns in pIndex */
  3265   3278     u16 nColumn;          /* Total number of ordered columns in the index */
  3266   3279     u16 nOrderBy;         /* Number terms in the ORDER BY clause */
  3267   3280     int iLoop;            /* Index of WhereLoop in pPath being processed */
  3268   3281     int i, j;             /* Loop counters */
  3269   3282     int iCur;             /* Cursor number for current WhereLoop */
  3270   3283     int iColumn;          /* A column number within table iCur */
................................................................................
  3307   3320     nOrderBy = pOrderBy->nExpr;
  3308   3321     testcase( nOrderBy==BMS-1 );
  3309   3322     if( nOrderBy>BMS-1 ) return 0;  /* Cannot optimize overly large ORDER BYs */
  3310   3323     isOrderDistinct = 1;
  3311   3324     obDone = MASKBIT(nOrderBy)-1;
  3312   3325     orderDistinctMask = 0;
  3313   3326     ready = 0;
         3327  +  eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
         3328  +  if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN;
  3314   3329     for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
  3315   3330       if( iLoop>0 ) ready |= pLoop->maskSelf;
  3316         -    pLoop = iLoop<nLoop ? pPath->aLoop[iLoop] : pLast;
         3331  +    if( iLoop<nLoop ){
         3332  +      pLoop = pPath->aLoop[iLoop];
         3333  +      if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
         3334  +    }else{
         3335  +      pLoop = pLast;
         3336  +    }
  3317   3337       if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
  3318   3338         if( pLoop->u.vtab.isOrdered ) obSat = obDone;
  3319   3339         break;
  3320   3340       }
  3321   3341       iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
  3322   3342   
  3323   3343       /* Mark off any ORDER BY term X that is a column in the table of
................................................................................
  3327   3347       */
  3328   3348       for(i=0; i<nOrderBy; i++){
  3329   3349         if( MASKBIT(i) & obSat ) continue;
  3330   3350         pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr);
  3331   3351         if( pOBExpr->op!=TK_COLUMN ) continue;
  3332   3352         if( pOBExpr->iTable!=iCur ) continue;
  3333   3353         pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
  3334         -                       ~ready, WO_EQ|WO_ISNULL|WO_IS, 0);
         3354  +                       ~ready, eqOpMask, 0);
  3335   3355         if( pTerm==0 ) continue;
  3336   3356         if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
  3337   3357           const char *z1, *z2;
  3338   3358           pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
  3339   3359           if( !pColl ) pColl = db->pDfltColl;
  3340   3360           z1 = pColl->zName;
  3341   3361           pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr);
................................................................................
  3367   3387         ** that are not constrained by == or IN.
  3368   3388         */
  3369   3389         rev = revSet = 0;
  3370   3390         distinctColumns = 0;
  3371   3391         for(j=0; j<nColumn; j++){
  3372   3392           u8 bOnce;   /* True to run the ORDER BY search loop */
  3373   3393   
  3374         -        /* Skip over == and IS NULL terms */
         3394  +        /* Skip over == and IS and ISNULL terms.
         3395  +        ** (Also skip IN terms when doing WHERE_ORDERBY_LIMIT processing)
         3396  +        */
  3375   3397           if( j<pLoop->u.btree.nEq
  3376   3398            && pLoop->nSkip==0
  3377         -         && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL|WO_IS))!=0
         3399  +         && ((i = pLoop->aLTerm[j]->eOperator) & eqOpMask)!=0
  3378   3400           ){
  3379   3401             if( i & WO_ISNULL ){
  3380   3402               testcase( isOrderDistinct );
  3381   3403               isOrderDistinct = 0;
  3382   3404             }
  3383   3405             continue;  
  3384   3406           }
................................................................................
  3894   3916     if( pWInfo->pOrderBy ){
  3895   3917       if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
  3896   3918         if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
  3897   3919           pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
  3898   3920         }
  3899   3921       }else{
  3900   3922         pWInfo->nOBSat = pFrom->isOrdered;
  3901         -      if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0;
  3902   3923         pWInfo->revMask = pFrom->revLoop;
         3924  +      if( pWInfo->nOBSat<=0 ){
         3925  +        pWInfo->nOBSat = 0;
         3926  +        if( nLoop>0 ){
         3927  +          Bitmask m;
         3928  +          int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
         3929  +                      WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
         3930  +          if( rc==pWInfo->pOrderBy->nExpr ){
         3931  +            pWInfo->bOrderedInnerLoop = 1;
         3932  +            pWInfo->revMask = m;
         3933  +          }
         3934  +        }
         3935  +      }
  3903   3936       }
  3904   3937       if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
  3905   3938           && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
  3906   3939       ){
  3907   3940         Bitmask revMask = 0;
  3908   3941         int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, 
  3909   3942             pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
................................................................................
  3943   3976     WhereLoop *pLoop;
  3944   3977     int iCur;
  3945   3978     int j;
  3946   3979     Table *pTab;
  3947   3980     Index *pIdx;
  3948   3981   
  3949   3982     pWInfo = pBuilder->pWInfo;
  3950         -  if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0;
         3983  +  if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
  3951   3984     assert( pWInfo->pTabList->nSrc>=1 );
  3952   3985     pItem = pWInfo->pTabList->a;
  3953   3986     pTab = pItem->pTab;
  3954   3987     if( IsVirtual(pTab) ) return 0;
  3955   3988     if( pItem->fg.isIndexedBy ) return 0;
  3956   3989     iCur = pItem->iCursor;
  3957   3990     pWC = &pWInfo->sWC;
................................................................................
  4090   4123   **
  4091   4124   ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
  4092   4125   ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
  4093   4126   ** if there is one.  If there is no ORDER BY clause or if this routine
  4094   4127   ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
  4095   4128   **
  4096   4129   ** The iIdxCur parameter is the cursor number of an index.  If 
  4097         -** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index
         4130  +** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
  4098   4131   ** to use for OR clause processing.  The WHERE clause should use this
  4099   4132   ** specific cursor.  If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
  4100   4133   ** the first cursor in an array of cursors for all indices.  iIdxCur should
  4101   4134   ** be used to compute the appropriate cursor depending on which index is
  4102   4135   ** used.
  4103   4136   */
  4104   4137   WhereInfo *sqlite3WhereBegin(
  4105   4138     Parse *pParse,          /* The parser context */
  4106   4139     SrcList *pTabList,      /* FROM clause: A list of all tables to be scanned */
  4107   4140     Expr *pWhere,           /* The WHERE clause */
  4108   4141     ExprList *pOrderBy,     /* An ORDER BY (or GROUP BY) clause, or NULL */
  4109   4142     ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */
  4110   4143     u16 wctrlFlags,         /* The WHERE_* flags defined in sqliteInt.h */
  4111         -  int iAuxArg             /* If WHERE_ONETABLE_ONLY is set, index cursor number
         4144  +  int iAuxArg             /* If WHERE_OR_SUBCLAUSE is set, index cursor number
  4112   4145                             ** If WHERE_USE_LIMIT, then the limit amount */
  4113   4146   ){
  4114   4147     int nByteWInfo;            /* Num. bytes allocated for WhereInfo struct */
  4115   4148     int nTabList;              /* Number of elements in pTabList */
  4116   4149     WhereInfo *pWInfo;         /* Will become the return value of this function */
  4117   4150     Vdbe *v = pParse->pVdbe;   /* The virtual database engine */
  4118   4151     Bitmask notReady;          /* Cursors that are not yet positioned */
................................................................................
  4123   4156     int ii;                    /* Loop counter */
  4124   4157     sqlite3 *db;               /* Database connection */
  4125   4158     int rc;                    /* Return code */
  4126   4159     u8 bFordelete = 0;         /* OPFLAG_FORDELETE or zero, as appropriate */
  4127   4160   
  4128   4161     assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
  4129   4162           (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 
  4130         -     && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 
         4163  +     && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 
  4131   4164     ));
  4132   4165   
  4133         -  /* Only one of WHERE_ONETABLE_ONLY or WHERE_USE_LIMIT */
  4134         -  assert( (wctrlFlags & WHERE_ONETABLE_ONLY)==0
         4166  +  /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
         4167  +  assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
  4135   4168               || (wctrlFlags & WHERE_USE_LIMIT)==0 );
  4136   4169   
  4137   4170     /* Variable initialization */
  4138   4171     db = pParse->db;
  4139   4172     memset(&sWLB, 0, sizeof(sWLB));
  4140   4173   
  4141   4174     /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
................................................................................
  4155   4188     testcase( pTabList->nSrc==BMS );
  4156   4189     if( pTabList->nSrc>BMS ){
  4157   4190       sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
  4158   4191       return 0;
  4159   4192     }
  4160   4193   
  4161   4194     /* This function normally generates a nested loop for all tables in 
  4162         -  ** pTabList.  But if the WHERE_ONETABLE_ONLY flag is set, then we should
         4195  +  ** pTabList.  But if the WHERE_OR_SUBCLAUSE flag is set, then we should
  4163   4196     ** only generate code for the first table in pTabList and assume that
  4164   4197     ** any cursors associated with subsequent tables are uninitialized.
  4165   4198     */
  4166         -  nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc;
         4199  +  nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
  4167   4200   
  4168   4201     /* Allocate and initialize the WhereInfo structure that will become the
  4169   4202     ** return value. A single allocation is used to store the WhereInfo
  4170   4203     ** struct, the contents of WhereInfo.a[], the WhereClause structure
  4171   4204     ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
  4172   4205     ** field (type Bitmask) it must be aligned on an 8-byte boundary on
  4173   4206     ** some architectures. Hence the ROUND8() below.
................................................................................
  4235   4268     ** a table T, then X-1 is the bitmask for all other tables to the left of T.
  4236   4269     ** Knowing the bitmask for all tables to the left of a left join is
  4237   4270     ** important.  Ticket #3015.
  4238   4271     **
  4239   4272     ** Note that bitmasks are created for all pTabList->nSrc tables in
  4240   4273     ** pTabList, not just the first nTabList tables.  nTabList is normally
  4241   4274     ** equal to pTabList->nSrc but might be shortened to 1 if the
  4242         -  ** WHERE_ONETABLE_ONLY flag is set.
         4275  +  ** WHERE_OR_SUBCLAUSE flag is set.
  4243   4276     */
  4244   4277     for(ii=0; ii<pTabList->nSrc; ii++){
  4245   4278       createMask(pMaskSet, pTabList->a[ii].iCursor);
  4246   4279       sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
  4247   4280     }
  4248   4281   #ifdef SQLITE_DEBUG
  4249   4282     for(ii=0; ii<pTabList->nSrc; ii++){
................................................................................
  4418   4451         int iCur = pTabItem->iCursor;
  4419   4452         sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
  4420   4453       }else if( IsVirtual(pTab) ){
  4421   4454         /* noop */
  4422   4455       }else
  4423   4456   #endif
  4424   4457       if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
  4425         -         && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){
         4458  +         && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){
  4426   4459         int op = OP_OpenRead;
  4427   4460         if( pWInfo->eOnePass!=ONEPASS_OFF ){
  4428   4461           op = OP_OpenWrite;
  4429   4462           pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
  4430   4463         };
  4431   4464         sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
  4432   4465         assert( pTabItem->iCursor==pLevel->iTabCur );
................................................................................
  4457   4490       if( pLoop->wsFlags & WHERE_INDEXED ){
  4458   4491         Index *pIx = pLoop->u.btree.pIndex;
  4459   4492         int iIndexCur;
  4460   4493         int op = OP_OpenRead;
  4461   4494         /* iAuxArg is always set if to a positive value if ONEPASS is possible */
  4462   4495         assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
  4463   4496         if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
  4464         -       && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0
         4497  +       && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
  4465   4498         ){
  4466   4499           /* This is one term of an OR-optimization using the PRIMARY KEY of a
  4467   4500           ** WITHOUT ROWID table.  No need for a separate index */
  4468   4501           iIndexCur = pLevel->iTabCur;
  4469   4502           op = 0;
  4470   4503         }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
  4471   4504           Index *pJ = pTabItem->pTab->pIndex;
................................................................................
  4473   4506           assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
  4474   4507           while( ALWAYS(pJ) && pJ!=pIx ){
  4475   4508             iIndexCur++;
  4476   4509             pJ = pJ->pNext;
  4477   4510           }
  4478   4511           op = OP_OpenWrite;
  4479   4512           pWInfo->aiCurOnePass[1] = iIndexCur;
  4480         -      }else if( iAuxArg && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){
         4513  +      }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
  4481   4514           iIndexCur = iAuxArg;
  4482         -        if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx;
         4515  +        op = OP_ReopenIdx;
  4483   4516         }else{
  4484   4517           iIndexCur = pParse->nTab++;
  4485   4518         }
  4486   4519         pLevel->iIdxCur = iIndexCur;
  4487   4520         assert( pIx->pSchema==pTab->pSchema );
  4488   4521         assert( iIndexCur>=0 );
  4489   4522         if( op ){
................................................................................
  4537   4570   #endif
  4538   4571       addrExplain = sqlite3WhereExplainOneScan(
  4539   4572           pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags
  4540   4573       );
  4541   4574       pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
  4542   4575       notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady);
  4543   4576       pWInfo->iContinue = pLevel->addrCont;
  4544         -    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){
         4577  +    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
  4545   4578         sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
  4546   4579       }
  4547   4580     }
  4548   4581   
  4549   4582     /* Done. */
  4550   4583     VdbeModuleComment((v, "Begin WHERE-core"));
  4551   4584     return pWInfo;
................................................................................
  4660   4693         translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur,
  4661   4694                               pTabItem->regResult, 0);
  4662   4695         continue;
  4663   4696       }
  4664   4697   
  4665   4698       /* Close all of the cursors that were opened by sqlite3WhereBegin.
  4666   4699       ** Except, do not close cursors that will be reused by the OR optimization
  4667         -    ** (WHERE_OMIT_OPEN_CLOSE).  And do not close the OP_OpenWrite cursors
         4700  +    ** (WHERE_OR_SUBCLAUSE).  And do not close the OP_OpenWrite cursors
  4668   4701       ** created for the ONEPASS optimization.
  4669   4702       */
  4670   4703       if( (pTab->tabFlags & TF_Ephemeral)==0
  4671   4704        && pTab->pSelect==0
  4672         -     && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0
         4705  +     && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
  4673   4706       ){
  4674   4707         int ws = pLoop->wsFlags;
  4675   4708         if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){
  4676   4709           sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor);
  4677   4710         }
  4678   4711         if( (ws & WHERE_INDEXED)!=0
  4679   4712          && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 

Changes to src/whereInt.h.

   414    414     LogEst nRowOut;           /* Estimated number of output rows */
   415    415     LogEst iLimit;            /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */
   416    416     u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
   417    417     i8 nOBSat;                /* Number of ORDER BY terms satisfied by indices */
   418    418     u8 sorted;                /* True if really sorted (not just grouped) */
   419    419     u8 eOnePass;              /* ONEPASS_OFF, or _SINGLE, or _MULTI */
   420    420     u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
   421         -  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
          421  +  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values */
   422    422     u8 nLevel;                /* Number of nested loop */
          423  +  u8 bOrderedInnerLoop;     /* True if only the inner-most loop is ordered */
   423    424     int iTop;                 /* The very beginning of the WHERE loop */
   424    425     int iContinue;            /* Jump here to continue with next record */
   425    426     int iBreak;               /* Jump here to break out of the loop */
   426    427     int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
   427    428     int aiCurOnePass[2];      /* OP_OpenWrite cursors for the ONEPASS opt */
   428    429     WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
   429    430     WhereClause sWC;          /* Decomposition of the WHERE clause */

Changes to src/wherecode.c.

   122    122       u32 flags;                    /* Flags that describe this loop */
   123    123       char *zMsg;                   /* Text to add to EQP output */
   124    124       StrAccum str;                 /* EQP output string */
   125    125       char zBuf[100];               /* Initial space for EQP output string */
   126    126   
   127    127       pLoop = pLevel->pWLoop;
   128    128       flags = pLoop->wsFlags;
   129         -    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0;
          129  +    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0;
   130    130   
   131    131       isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
   132    132               || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
   133    133               || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
   134    134   
   135    135       sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
   136    136       sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN");
................................................................................
   772    772     Parse *pParse = pWInfo->pParse; /* Parse context */
   773    773     Vdbe *v = pParse->pVdbe;        /* Vdbe to generate code within */
   774    774   
   775    775     assert( iIdxCur>0 );
   776    776     assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 );
   777    777     
   778    778     sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur);
   779         -  if( (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)
          779  +  if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
   780    780      && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask)
   781    781     ){
   782    782       int i;
   783    783       Table *pTab = pIdx->pTable;
   784    784       int *ai = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*(pTab->nCol+1));
   785    785       if( ai ){
   786    786         ai[0] = pTab->nCol;
................................................................................
   827    827     pLevel = &pWInfo->a[iLevel];
   828    828     pLoop = pLevel->pWLoop;
   829    829     pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
   830    830     iCur = pTabItem->iCursor;
   831    831     pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur);
   832    832     bRev = (pWInfo->revMask>>iLevel)&1;
   833    833     omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 
   834         -           && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0;
          834  +           && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0;
   835    835     VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName));
   836    836   
   837    837     /* Create labels for the "break" and "continue" instructions
   838    838     ** for the current loop.  Jump to addrBrk to break out of a loop.
   839    839     ** Jump to cont to go immediately to the next iteration of the
   840    840     ** loop.
   841    841     **
................................................................................
  1513   1513         }
  1514   1514       }
  1515   1515   
  1516   1516       /* Run a separate WHERE clause for each term of the OR clause.  After
  1517   1517       ** eliminating duplicates from other WHERE clauses, the action for each
  1518   1518       ** sub-WHERE clause is to to invoke the main loop body as a subroutine.
  1519   1519       */
  1520         -    wctrlFlags =  WHERE_OMIT_OPEN_CLOSE
  1521         -                | WHERE_FORCE_TABLE
  1522         -                | WHERE_ONETABLE_ONLY
  1523         -                | WHERE_NO_AUTOINDEX
  1524         -                | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
         1520  +    wctrlFlags =  WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
  1525   1521       for(ii=0; ii<pOrWc->nTerm; ii++){
  1526   1522         WhereTerm *pOrTerm = &pOrWc->a[ii];
  1527   1523         if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){
  1528   1524           WhereInfo *pSubWInfo;           /* Info for single OR-term scan */
  1529   1525           Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */
  1530   1526           int jmp1 = 0;                   /* Address of jump operation */
  1531   1527           if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){
................................................................................
  1625   1621             assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
  1626   1622             if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0
  1627   1623              && (ii==0 || pSubLoop->u.btree.pIndex==pCov)
  1628   1624              && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex))
  1629   1625             ){
  1630   1626               assert( pSubWInfo->a[0].iIdxCur==iCovCur );
  1631   1627               pCov = pSubLoop->u.btree.pIndex;
  1632         -            wctrlFlags |= WHERE_REOPEN_IDX;
  1633   1628             }else{
  1634   1629               pCov = 0;
  1635   1630             }
  1636   1631   
  1637   1632             /* Finish the loop through table entries that match term pOrTerm. */
  1638   1633             sqlite3WhereEnd(pSubWInfo);
  1639   1634           }
................................................................................
  1687   1682       Expr *pE;
  1688   1683       int skipLikeAddr = 0;
  1689   1684       testcase( pTerm->wtFlags & TERM_VIRTUAL );
  1690   1685       testcase( pTerm->wtFlags & TERM_CODED );
  1691   1686       if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
  1692   1687       if( (pTerm->prereqAll & pLevel->notReady)!=0 ){
  1693   1688         testcase( pWInfo->untestedTerms==0
  1694         -               && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 );
         1689  +               && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
  1695   1690         pWInfo->untestedTerms = 1;
  1696   1691         continue;
  1697   1692       }
  1698   1693       pE = pTerm->pExpr;
  1699   1694       assert( pE!=0 );
  1700   1695       if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){
  1701   1696         continue;

Changes to test/fts3expr.test.

   509    509   do_test fts3expr-8.8 { test_fts3expr "(,(blah-),)" } {PHRASE 3 0 blah}
   510    510   
   511    511   set sqlite_fts3_enable_parentheses 0
   512    512   
   513    513   do_test fts3expr-9.1 {
   514    514     test_fts3expr "f (e NEAR/2 a)"
   515    515   } {AND {PHRASE 3 0 f} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}}
          516  +
          517  +do_test fts3expr-10.1 { test_fts3expr "abc *" } {PHRASE 3 0 abc}
          518  +do_test fts3expr-10.2 { test_fts3expr "*" } {}
          519  +do_test fts3expr-10.3 { test_fts3expr "abc*" } {PHRASE 3 0 abc+}
   516    520   
   517    521   finish_test

Added test/limit2.test.

            1  +# 2016-05-20
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the LIMIT in combination with ORDER BY
           13  +# and in particular, the optimizations in the inner loop that cause an
           14  +# early exit of the inner loop when the LIMIT is reached and the inner
           15  +# loop is emitting rows in ORDER BY order.
           16  +
           17  +
           18  +set testdir [file dirname $argv0]
           19  +source $testdir/tester.tcl
           20  +
           21  +do_execsql_test limit2-100 {
           22  +  CREATE TABLE t1(a,b);
           23  +  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
           24  +    INSERT INTO t1(a,b) SELECT 1, (x*17)%1000 + 1000 FROM c;
           25  +  INSERT INTO t1(a,b) VALUES(2,2),(3,1006),(4,4),(5,9999);
           26  +  CREATE INDEX t1ab ON t1(a,b);
           27  +}
           28  +set sqlite_search_count 0
           29  +do_execsql_test limit2-100.1 {
           30  +  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b LIMIT 5;
           31  +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
           32  +set fast_count $sqlite_search_count
           33  +set sqlite_search_count 0
           34  +do_execsql_test limit2-100.2 {
           35  +  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b LIMIT 5;
           36  +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
           37  +do_test limit2-100.3 {
           38  +  set slow_count $sqlite_search_count
           39  +  expr {$fast_count < 0.02*$slow_count}
           40  +} {1}
           41  +
           42  +do_execsql_test limit2-110 {
           43  +  CREATE TABLE t2(x,y);
           44  +  INSERT INTO t2(x,y) VALUES('a',1),('a',2),('a',3),('a',4);
           45  +  INSERT INTO t2(x,y) VALUES('b',1),('c',2),('d',3),('e',4);
           46  +  CREATE INDEX t2xy ON t2(x,y);
           47  +}
           48  +set sqlite_search_count 0
           49  +do_execsql_test limit2-110.1 {
           50  +  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY t1.b LIMIT 5;
           51  +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
           52  +set fast_count $sqlite_search_count
           53  +set sqlite_search_count 0
           54  +do_execsql_test limit2-110.2 {
           55  +  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY +t1.b LIMIT 5;
           56  +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
           57  +set slow_count $sqlite_search_count
           58  +do_test limit2-110.3 {
           59  +  expr {$fast_count < 0.02*$slow_count}
           60  +} {1}
           61  +
           62  +do_execsql_test limit2-120 {
           63  +  DROP INDEX t1ab;
           64  +  CREATE INDEX t1ab ON t1(a,b DESC);
           65  +}
           66  +set sqlite_search_count 0
           67  +do_execsql_test limit2-120.1 {
           68  +  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b DESC LIMIT 5;
           69  +} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
           70  +set fast_count $sqlite_search_count
           71  +set sqlite_search_count 0
           72  +do_execsql_test limit2-120.2 {
           73  +  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b DESC LIMIT 5;
           74  +} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
           75  +do_test limit2-120.3 {
           76  +  set slow_count $sqlite_search_count
           77  +  expr {$fast_count < 0.02*$slow_count}
           78  +} {1}
           79  +
           80  +
           81  +
           82  +finish_test

Added tool/opcodesum.tcl.

            1  +#!/usr/bin/tclsh
            2  +#
            3  +# Run this script, redirecting input from cachegrind output, to compute the
            4  +# number of CPU cycles used by each VDBE opcode.
            5  +#
            6  +# The cachegrind output should be configured so that it reports a single
            7  +# column of Ir at the left margin. Ex:
            8  +#
            9  +#    cg_annotation --show=Ir --auto=yes cachegrind.out.* | tclsh opcodesum.tcl
           10  +#
           11  +set currentop x
           12  +set ncycle(x) 0
           13  +while {![eof stdin]} {
           14  +  set line [string map {\173 x \175 x \042 x} [gets stdin]]
           15  +  if {[regexp {  \.  case OP_.*:} $line]} {
           16  +    regexp {OP_(.+):} $line all currentop
           17  +    set ncycle($currentop) 0
           18  +  } elseif {[lindex $line 1]=="default:"
           19  +            && [regexp {really OP_Noop and OP_Explain} $line]} {
           20  +    break
           21  +  } elseif {[lindex $line 0]!="."} {
           22  +    regsub -all {[^0-9]} [lindex $line 0] {} n
           23  +    if {$n!=""} {incr ncycle($currentop) $n}
           24  +  }
           25  +}
           26  +unset ncycle(x)
           27  +set results {}
           28  +foreach op [lsort [array names ncycle]] {
           29  +  if {$ncycle($op)==0} continue
           30  +  lappend results [list $ncycle($op) $op]
           31  +}
           32  +foreach entry [lsort -index 0 -int -decr $results] {
           33  +  puts [format {%-16s %10d} [lindex $entry 1] [lindex $entry 0]]
           34  +}

Added tool/speed-check.sh.

            1  +#!/bin/bash
            2  +#
            3  +# This is a template for a script used for day-to-day size and 
            4  +# performance monitoring of SQLite.  Typical usage:
            5  +#
            6  +#     sh run-speed-test.sh trunk  #  Baseline measurement of trunk
            7  +#     sh run-speed-test.sh x1     # Measure some experimental change
            8  +#     fossil test-diff --tk cout-trunk.txt cout-x1.txt   # View chanages
            9  +#
           10  +# There are multiple output files, all with a base name given by
           11  +# the first argument:
           12  +#
           13  +#     summary-$BASE.txt           # Copy of standard output
           14  +#     cout-$BASE.txt              # cachegrind output
           15  +#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
           16  +#
           17  +if test "$1" = ""
           18  +then
           19  +  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
           20  +  exit
           21  +fi
           22  +NAME=$1
           23  +shift
           24  +CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
           25  +SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64"
           26  +SIZE=5
           27  +doExplain=0
           28  +doCachegrind=1
           29  +while test "$1" != ""; do
           30  +  case $1 in
           31  +    --reprepare)
           32  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
           33  +        ;;
           34  +    --autovacuum)
           35  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
           36  +        ;;
           37  +    --utf16be)
           38  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
           39  +        ;;
           40  +    --stats)
           41  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
           42  +        ;;
           43  +    --without-rowid)
           44  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
           45  +        ;;
           46  +    --nomemstat)
           47  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
           48  +        ;;
           49  +    --temp)
           50  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
           51  +        ;;
           52  +    --wal)
           53  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
           54  +        ;;
           55  +    --size)
           56  +        shift; SIZE=$1
           57  +        ;;
           58  +    --explain)
           59  +        doExplain=1
           60  +        ;;
           61  +    --vdbeprofile)
           62  +        rm -f vdbe_profile.out
           63  +        CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
           64  +        doCachegrind=0
           65  +        ;;
           66  +    --heap)
           67  +        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
           68  +        shift;
           69  +        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
           70  +        ;;
           71  +    *)
           72  +        CC_OPTS="$CC_OPTS $1"
           73  +        ;;
           74  +  esac
           75  +  shift
           76  +done
           77  +SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
           78  +echo "NAME           = $NAME" | tee summary-$NAME.txt
           79  +echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
           80  +echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
           81  +rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
           82  +gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
           83  +size sqlite3.o | tee -a summary-$NAME.txt
           84  +if test $doExplain -eq 1; then
           85  +  gcc -g -Os -Wall -I. $CC_OPTS \
           86  +     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
           87  +    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
           88  +fi
           89  +SRC=./speedtest1.c
           90  +gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
           91  +ls -l speedtest1 | tee -a summary-$NAME.txt
           92  +if test $doCachegrind -eq 1; then
           93  +  valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
           94  +      $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
           95  +else
           96  +  ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
           97  +fi
           98  +size sqlite3.o | tee -a summary-$NAME.txt
           99  +wc sqlite3.c
          100  +if test $doCachegrind -eq 1; then
          101  +  cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
          102  +fi
          103  +if test $doExplain -eq 1; then
          104  +  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
          105  +fi