/ Check-in [dd4b77c8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge updates from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | toTypeFuncs
Files: files | file ages | folders
SHA1: dd4b77c82af07bdcc92ed743f050e70887e5956e
User & Date: mistachkin 2013-07-20 00:39:33
Context
2013-08-19
21:15
Add tointeger() and toreal() SQL functions. check-in: af497072 user: mistachkin tags: toTypeFuncs
2013-07-20
00:39
Merge updates from trunk. check-in: dd4b77c8 user: mistachkin tags: toTypeFuncs
00:34
Add 'queryplantest' target to the MSVC makefile. check-in: ad0551e0 user: mistachkin tags: trunk
2013-06-21
19:29
Merge in the latest changes from trunk. Simplify the implementation of the tointeger() and toreal() functions. Fix test cases and put unambiguous labels on all testcase names. check-in: 9b837b05 user: drh tags: toTypeFuncs
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.msc.

  1235   1235   
  1236   1236   soaktest:	testfixture.exe sqlite3.exe
  1237   1237   	.\testfixture.exe $(TOP)\test\all.test -soak=1
  1238   1238   
  1239   1239   fulltestonly:	testfixture.exe sqlite3.exe
  1240   1240   	.\testfixture.exe $(TOP)\test\full.test
  1241   1241   
         1242  +queryplantest:	testfixture.exe sqlite3.exe
         1243  +	.\testfixture.exe $(TOP)\test\permutations.test queryplanner
         1244  +
  1242   1245   test:	testfixture.exe sqlite3.exe
  1243   1246   	.\testfixture.exe $(TOP)\test\veryquick.test
  1244   1247   
  1245   1248   sqlite3_analyzer.c: sqlite3.c $(TOP)\src\test_stat.c $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
  1246   1249   	copy sqlite3.c + $(TOP)\src\test_stat.c + $(TOP)\src\tclsqlite.c $@
  1247   1250   	echo static const char *tclsh_main_loop(void){ >> $@
  1248   1251   	echo static const char *zMainloop = >> $@

Changes to VERSION.

     1         -3.7.17
            1  +3.8.0

Changes to ext/fts1/fts1.c.

  3331   3331   int sqlite3Fts1Init(sqlite3 *db){
  3332   3332     sqlite3_overload_function(db, "snippet", -1);
  3333   3333     sqlite3_overload_function(db, "offsets", -1);
  3334   3334     return sqlite3_create_module(db, "fts1", &fulltextModule, 0);
  3335   3335   }
  3336   3336   
  3337   3337   #if !SQLITE_CORE
  3338         -int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
  3339         -                           const sqlite3_api_routines *pApi){
         3338  +#ifdef _WIN32
         3339  +__declspec(dllexport)
         3340  +#endif
         3341  +int sqlite3_fts1_init(sqlite3 *db, char **pzErrMsg,
         3342  +                      const sqlite3_api_routines *pApi){
  3340   3343     SQLITE_EXTENSION_INIT2(pApi)
  3341   3344     return sqlite3Fts1Init(db);
  3342   3345   }
  3343   3346   #endif
  3344   3347   
  3345   3348   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */

Changes to ext/fts1/fulltext.c.

   848    848   /* Current interface:
   849    849   ** argv[0] - module name
   850    850   ** argv[1] - database name
   851    851   ** argv[2] - table name
   852    852   ** argv[3] - tokenizer name (optional, a sensible default is provided)
   853    853   ** argv[4..] - passed to tokenizer (optional based on tokenizer)
   854    854   **/
   855         -static int fulltextConnect(sqlite3 *db, void *pAux, int argc, char **argv,
   856         -                           sqlite3_vtab **ppVTab){
          855  +static int fulltextConnect(
          856  +  sqlite3 *db,
          857  +  void *pAux,
          858  +  int argc,
          859  +  const char * const *argv,
          860  +  sqlite3_vtab **ppVTab,
          861  +  char **pzErr
          862  +){
   857    863     int rc;
   858    864     fulltext_vtab *v;
   859    865     sqlite3_tokenizer_module *m = NULL;
   860    866   
   861    867     assert( argc>=3 );
   862    868     v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab));
   863    869     /* sqlite will initialize v->base */
................................................................................
   894    900   
   895    901     memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
   896    902   
   897    903     *ppVTab = &v->base;
   898    904     return SQLITE_OK;
   899    905   }
   900    906   
   901         -static int fulltextCreate(sqlite3 *db, void *pAux, int argc, char **argv,
   902         -                          sqlite3_vtab **ppVTab){
          907  +static int fulltextCreate(
          908  +  sqlite3 *db,
          909  +  void *pAux,
          910  +  int argc,
          911  +  const char * const *argv,
          912  +  sqlite3_vtab **ppVTab,
          913  +  char **pzErr
          914  +){
   903    915     int rc;
   904    916     assert( argc>=3 );
   905    917   
   906    918     /* The %_content table holds the text of each full-text item, with
   907    919     ** the rowid used as the docid.
   908    920     **
   909    921     ** The %_term table maps each term to a document list blob
................................................................................
   930    942     */
   931    943     rc = sql_exec(db, argv[2],
   932    944       "create table %_content(content text);"
   933    945       "create table %_term(term text, first integer, doclist blob);"
   934    946       "create index %_index on %_term(term, first)");
   935    947     if( rc!=SQLITE_OK ) return rc;
   936    948   
   937         -  return fulltextConnect(db, pAux, argc, argv, ppVTab);
          949  +  return fulltextConnect(db, pAux, argc, argv, ppVTab, pzErr);
   938    950   }
   939    951   
   940    952   /* Decide how to handle an SQL query.
   941    953    * At the moment, MATCH queries can include implicit boolean ANDs; we
   942    954    * haven't implemented phrase searches or OR yet. */
   943    955   static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   944    956     int i;
................................................................................
  1484   1496   };
  1485   1497   
  1486   1498   int fulltext_init(sqlite3 *db){
  1487   1499    return sqlite3_create_module(db, "fulltext", &fulltextModule, 0);
  1488   1500   }
  1489   1501   
  1490   1502   #if !SQLITE_CORE
  1491         -int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
  1492         -                           const sqlite3_api_routines *pApi){
         1503  +#ifdef _WIN32
         1504  +__declspec(dllexport)
         1505  +#endif
         1506  +int sqlite3_fulltext_init(sqlite3 *db, char **pzErrMsg,
         1507  +                          const sqlite3_api_routines *pApi){
  1493   1508    SQLITE_EXTENSION_INIT2(pApi)
  1494   1509    return fulltext_init(db);
  1495   1510   }
  1496   1511   #endif

Changes to ext/fts2/fts2.c.

  6840   6840       sqlite3Fts2HashClear(pHash);
  6841   6841       sqlite3_free(pHash);
  6842   6842     }
  6843   6843     return rc;
  6844   6844   }
  6845   6845   
  6846   6846   #if !SQLITE_CORE
  6847         -int sqlite3_extension_init(
         6847  +#ifdef _WIN32
         6848  +__declspec(dllexport)
         6849  +#endif
         6850  +int sqlite3_fts2_init(
  6848   6851     sqlite3 *db, 
  6849   6852     char **pzErrMsg,
  6850   6853     const sqlite3_api_routines *pApi
  6851   6854   ){
  6852   6855     SQLITE_EXTENSION_INIT2(pApi)
  6853   6856     return sqlite3Fts2Init(db);
  6854   6857   }
  6855   6858   #endif
  6856   6859   
  6857   6860   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */

Changes to ext/fts2/fts2_hash.c.

    26     26   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
    27     27   
    28     28   #include <assert.h>
    29     29   #include <stdlib.h>
    30     30   #include <string.h>
    31     31   
    32     32   #include "sqlite3.h"
           33  +#include "sqlite3ext.h"
           34  +SQLITE_EXTENSION_INIT3
    33     35   #include "fts2_hash.h"
    34     36   
    35     37   /*
    36     38   ** Malloc and Free functions
    37     39   */
    38     40   static void *fts2HashMalloc(int n){
    39     41     void *p = sqlite3_malloc(n);

Changes to ext/fts2/fts2_porter.c.

    26     26   
    27     27   
    28     28   #include <assert.h>
    29     29   #include <stdlib.h>
    30     30   #include <stdio.h>
    31     31   #include <string.h>
    32     32   
           33  +#include "sqlite3.h"
           34  +#include "sqlite3ext.h"
           35  +SQLITE_EXTENSION_INIT3
    33     36   #include "fts2_tokenizer.h"
    34     37   
    35     38   /*
    36     39   ** Class derived from sqlite3_tokenizer
    37     40   */
    38     41   typedef struct porter_tokenizer {
    39     42     sqlite3_tokenizer base;      /* Base class */

Changes to ext/fts2/fts2_tokenizer.c.

    24     24   **       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
    25     25   */
    26     26   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
    27     27   
    28     28   
    29     29   #include "sqlite3.h"
    30     30   #include "sqlite3ext.h"
    31         -SQLITE_EXTENSION_INIT1
           31  +SQLITE_EXTENSION_INIT3
    32     32   
    33     33   #include "fts2_hash.h"
    34     34   #include "fts2_tokenizer.h"
    35     35   #include <assert.h>
    36     36   
    37     37   /*
    38     38   ** Implementation of the SQL scalar function for accessing the underlying 

Changes to ext/fts2/fts2_tokenizer1.c.

    26     26   
    27     27   
    28     28   #include <assert.h>
    29     29   #include <stdlib.h>
    30     30   #include <stdio.h>
    31     31   #include <string.h>
    32     32   
           33  +#include "sqlite3.h"
           34  +#include "sqlite3ext.h"
           35  +SQLITE_EXTENSION_INIT3
    33     36   #include "fts2_tokenizer.h"
    34     37   
    35     38   typedef struct simple_tokenizer {
    36     39     sqlite3_tokenizer base;
    37     40     char delim[128];             /* flag ASCII delimiters */
    38     41   } simple_tokenizer;
    39     42   

Changes to ext/fts3/fts3.c.

  1091   1091   
  1092   1092     nDb = (int)strlen(argv[1]) + 1;
  1093   1093     nName = (int)strlen(argv[2]) + 1;
  1094   1094   
  1095   1095     nByte = sizeof(const char *) * (argc-2);
  1096   1096     aCol = (const char **)sqlite3_malloc(nByte);
  1097   1097     if( aCol ){
  1098         -    memset(aCol, 0, nByte);
         1098  +    memset((void*)aCol, 0, nByte);
  1099   1099       azNotindexed = (char **)sqlite3_malloc(nByte);
  1100   1100     }
  1101   1101     if( azNotindexed ){
  1102   1102       memset(azNotindexed, 0, nByte);
  1103   1103     }
  1104   1104     if( !aCol || !azNotindexed ){
  1105   1105       rc = SQLITE_NOMEM;
................................................................................
  1342   1342       p->azColumn[iCol] = zCsr;
  1343   1343       zCsr += n+1;
  1344   1344       assert( zCsr <= &((char *)p)[nByte] );
  1345   1345     }
  1346   1346   
  1347   1347     /* Fill in the abNotindexed array */
  1348   1348     for(iCol=0; iCol<nCol; iCol++){
  1349         -    int n = strlen(p->azColumn[iCol]);
         1349  +    int n = (int)strlen(p->azColumn[iCol]);
  1350   1350       for(i=0; i<nNotindexed; i++){
  1351   1351         char *zNot = azNotindexed[i];
  1352   1352         if( zNot && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n) ){
  1353   1353           p->abNotindexed[iCol] = 1;
  1354   1354           sqlite3_free(zNot);
  1355   1355           azNotindexed[i] = 0;
  1356   1356         }
................................................................................
  1460   1460     int iLangidCons = -1;           /* Index of langid=x constraint, if present */
  1461   1461   
  1462   1462     /* By default use a full table scan. This is an expensive option,
  1463   1463     ** so search through the constraints to see if a more efficient 
  1464   1464     ** strategy is possible.
  1465   1465     */
  1466   1466     pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
  1467         -  pInfo->estimatedCost = 500000;
         1467  +  pInfo->estimatedCost = 5000000;
  1468   1468     for(i=0; i<pInfo->nConstraint; i++){
  1469   1469       struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
  1470   1470       if( pCons->usable==0 ) continue;
  1471   1471   
  1472   1472       /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
  1473   1473       if( iCons<0 
  1474   1474        && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
................................................................................
  5372   5372   }
  5373   5373   #endif
  5374   5374   
  5375   5375   #if !SQLITE_CORE
  5376   5376   /*
  5377   5377   ** Initialize API pointer table, if required.
  5378   5378   */
  5379         -int sqlite3_extension_init(
         5379  +#ifdef _WIN32
         5380  +__declspec(dllexport)
         5381  +#endif
         5382  +int sqlite3_fts3_init(
  5380   5383     sqlite3 *db, 
  5381   5384     char **pzErrMsg,
  5382   5385     const sqlite3_api_routines *pApi
  5383   5386   ){
  5384   5387     SQLITE_EXTENSION_INIT2(pApi)
  5385   5388     return sqlite3Fts3Init(db);
  5386   5389   }
  5387   5390   #endif
  5388   5391   
  5389   5392   #endif

Changes to ext/fts3/fts3Int.h.

    28     28   #endif
    29     29   
    30     30   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    31     31   
    32     32   /* If not building as part of the core, include sqlite3ext.h. */
    33     33   #ifndef SQLITE_CORE
    34     34   # include "sqlite3ext.h" 
    35         -extern const sqlite3_api_routines *sqlite3_api;
           35  +SQLITE_EXTENSION_INIT3
    36     36   #endif
    37     37   
    38     38   #include "sqlite3.h"
    39     39   #include "fts3_tokenizer.h"
    40     40   #include "fts3_hash.h"
    41     41   
    42     42   /*

Changes to ext/icu/icu.c.

   484    484       );
   485    485     }
   486    486   
   487    487     return rc;
   488    488   }
   489    489   
   490    490   #if !SQLITE_CORE
   491         -int sqlite3_extension_init(
          491  +#ifdef _WIN32
          492  +__declspec(dllexport)
          493  +#endif
          494  +int sqlite3_icu_init(
   492    495     sqlite3 *db, 
   493    496     char **pzErrMsg,
   494    497     const sqlite3_api_routines *pApi
   495    498   ){
   496    499     SQLITE_EXTENSION_INIT2(pApi)
   497    500     return sqlite3IcuInit(db);
   498    501   }
   499    502   #endif
   500    503   
   501    504   #endif

Changes to ext/misc/closure.c.

   492    492     while( isspace(zStr[i]) ){ i++; }
   493    493     return zStr+i;
   494    494   }
   495    495   
   496    496   /*
   497    497   ** xConnect/xCreate method for the closure module. Arguments are:
   498    498   **
   499         -**   argv[0]    -> module name  ("approximate_match")
          499  +**   argv[0]    -> module name  ("transitive_closure")
   500    500   **   argv[1]    -> database name
   501    501   **   argv[2]    -> table name
   502    502   **   argv[3...] -> arguments
   503    503   */
   504    504   static int closureConnect(
   505    505     sqlite3 *db,
   506    506     void *pAux,
................................................................................
   822    822   static int closureBestIndex(
   823    823     sqlite3_vtab *pTab,             /* The virtual table */
   824    824     sqlite3_index_info *pIdxInfo    /* Information about the query */
   825    825   ){
   826    826     int iPlan = 0;
   827    827     int i;
   828    828     int idx = 1;
          829  +  int seenMatch = 0;
   829    830     const struct sqlite3_index_constraint *pConstraint;
   830    831     closure_vtab *pVtab = (closure_vtab*)pTab;
          832  +  double rCost = 10000000.0;
   831    833   
   832    834     pConstraint = pIdxInfo->aConstraint;
   833    835     for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
          836  +    if( pConstraint->iColumn==CLOSURE_COL_ROOT
          837  +     && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
          838  +      seenMatch = 1;
          839  +    }
   834    840       if( pConstraint->usable==0 ) continue;
   835    841       if( (iPlan & 1)==0 
   836    842        && pConstraint->iColumn==CLOSURE_COL_ROOT
   837    843        && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ
   838    844       ){
   839    845         iPlan |= 1;
   840    846         pIdxInfo->aConstraintUsage[i].argvIndex = 1;
   841    847         pIdxInfo->aConstraintUsage[i].omit = 1;
          848  +      rCost /= 100.0;
   842    849       }
   843    850       if( (iPlan & 0x0000f0)==0
   844    851        && pConstraint->iColumn==CLOSURE_COL_DEPTH
   845    852        && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT
   846    853              || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE
   847    854              || pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ)
   848    855       ){
   849    856         iPlan |= idx<<4;
   850    857         pIdxInfo->aConstraintUsage[i].argvIndex = ++idx;
   851    858         if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ) iPlan |= 0x000002;
          859  +      rCost /= 5.0;
   852    860       }
   853    861       if( (iPlan & 0x000f00)==0
   854    862        && pConstraint->iColumn==CLOSURE_COL_TABLENAME
   855    863        && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ
   856    864       ){
   857    865         iPlan |= idx<<8;
   858    866         pIdxInfo->aConstraintUsage[i].argvIndex = ++idx;
   859    867         pIdxInfo->aConstraintUsage[i].omit = 1;
          868  +      rCost /= 5.0;
   860    869       }
   861    870       if( (iPlan & 0x00f000)==0
   862    871        && pConstraint->iColumn==CLOSURE_COL_IDCOLUMN
   863    872        && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ
   864    873       ){
   865    874         iPlan |= idx<<12;
   866    875         pIdxInfo->aConstraintUsage[i].argvIndex = ++idx;
................................................................................
   887    896     pIdxInfo->idxNum = iPlan;
   888    897     if( pIdxInfo->nOrderBy==1
   889    898      && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID
   890    899      && pIdxInfo->aOrderBy[0].desc==0
   891    900     ){
   892    901       pIdxInfo->orderByConsumed = 1;
   893    902     }
   894         -  pIdxInfo->estimatedCost = (double)10000;
          903  +  if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30;
          904  +  pIdxInfo->estimatedCost = rCost;
   895    905      
   896    906     return SQLITE_OK;
   897    907   }
   898    908   
   899    909   /*
   900         -** A virtual table module that implements the "approximate_match".
          910  +** A virtual table module that implements the "transitive_closure".
   901    911   */
   902    912   static sqlite3_module closureModule = {
   903    913     0,                      /* iVersion */
   904    914     closureConnect,         /* xCreate */
   905    915     closureConnect,         /* xConnect */
   906    916     closureBestIndex,       /* xBestIndex */
   907    917     closureDisconnect,      /* xDisconnect */

Changes to ext/misc/fuzzer.c.

  1073   1073   ** filter.argv[2] if both bit-1 and bit-2 are set.
  1074   1074   */
  1075   1075   static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
  1076   1076     int iPlan = 0;
  1077   1077     int iDistTerm = -1;
  1078   1078     int iRulesetTerm = -1;
  1079   1079     int i;
         1080  +  int seenMatch = 0;
  1080   1081     const struct sqlite3_index_constraint *pConstraint;
         1082  +  double rCost = 1e12;
         1083  +
  1081   1084     pConstraint = pIdxInfo->aConstraint;
  1082   1085     for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
         1086  +    if( pConstraint->iColumn==0
         1087  +     && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
         1088  +      seenMatch = 1;
         1089  +    }
  1083   1090       if( pConstraint->usable==0 ) continue;
  1084   1091       if( (iPlan & 1)==0 
  1085   1092        && pConstraint->iColumn==0
  1086   1093        && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH
  1087   1094       ){
  1088   1095         iPlan |= 1;
  1089   1096         pIdxInfo->aConstraintUsage[i].argvIndex = 1;
  1090   1097         pIdxInfo->aConstraintUsage[i].omit = 1;
         1098  +      rCost /= 1e6;
  1091   1099       }
  1092   1100       if( (iPlan & 2)==0
  1093   1101        && pConstraint->iColumn==1
  1094   1102        && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT
  1095   1103              || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE)
  1096   1104       ){
  1097   1105         iPlan |= 2;
  1098   1106         iDistTerm = i;
         1107  +      rCost /= 10.0;
  1099   1108       }
  1100   1109       if( (iPlan & 4)==0
  1101   1110        && pConstraint->iColumn==2
  1102   1111        && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ
  1103   1112       ){
  1104   1113         iPlan |= 4;
  1105   1114         pIdxInfo->aConstraintUsage[i].omit = 1;
  1106   1115         iRulesetTerm = i;
         1116  +      rCost /= 10.0;
  1107   1117       }
  1108   1118     }
  1109   1119     if( iPlan & 2 ){
  1110   1120       pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0);
  1111   1121     }
  1112   1122     if( iPlan & 4 ){
  1113   1123       int idx = 1;
................................................................................
  1118   1128     pIdxInfo->idxNum = iPlan;
  1119   1129     if( pIdxInfo->nOrderBy==1
  1120   1130      && pIdxInfo->aOrderBy[0].iColumn==1
  1121   1131      && pIdxInfo->aOrderBy[0].desc==0
  1122   1132     ){
  1123   1133       pIdxInfo->orderByConsumed = 1;
  1124   1134     }
  1125         -  pIdxInfo->estimatedCost = (double)10000;
         1135  +  if( seenMatch && (iPlan&1)==0 ) rCost = 1e99;
         1136  +  pIdxInfo->estimatedCost = rCost;
  1126   1137      
  1127   1138     return SQLITE_OK;
  1128   1139   }
  1129   1140   
  1130   1141   /*
  1131   1142   ** A virtual table module that implements the "fuzzer".
  1132   1143   */

Changes to ext/misc/ieee754.c.

    14     14   ** and input of IEEE754 Binary64 floating-point numbers.
    15     15   **
    16     16   **   ieee754(X)
    17     17   **   ieee754(Y,Z)
    18     18   **
    19     19   ** In the first form, the value X should be a floating-point number.
    20     20   ** The function will return a string of the form 'ieee754(Y,Z)' where
    21         -** Y and Z are integers such that X==Y*pow(w.0,Z).
           21  +** Y and Z are integers such that X==Y*pow(2,Z).
    22     22   **
    23     23   ** In the second form, Y and Z are integers which are the mantissa and
    24     24   ** base-2 exponent of a new floating point number.  The function returns
    25         -** a floating-point value equal to Y*pow(2.0,Z).
           25  +** a floating-point value equal to Y*pow(2,Z).
    26     26   **
    27     27   ** Examples:
    28     28   **
    29     29   **     ieee754(2.0)       ->     'ieee754(2,0)'
    30     30   **     ieee754(45.25)     ->     'ieee754(181,-2)'
    31     31   **     ieee754(2, 0)      ->     2.0
    32     32   **     ieee754(181, -2)   ->     45.25

Changes to ext/misc/nextchar.c.

     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13         -** This file contains code to implement the next_char(A,T,F,W) SQL function.
           13  +** This file contains code to implement the next_char(A,T,F,W,C) SQL function.
           14  +**
           15  +** The next_char(A,T,F,W,C) function finds all valid "next" characters for
           16  +** string A given the vocabulary in T.F.  If the W value exists and is a
           17  +** non-empty string, then it is an SQL expression that limits the entries
           18  +** in T.F that will be considered.  If C exists and is a non-empty string,
           19  +** then it is the name of the collating sequence to use for comparison.  If
           20  +** 
           21  +** Only the first three arguments are required.  If the C parameter is 
           22  +** omitted or is NULL or is an empty string, then the default collating 
           23  +** sequence of T.F is used for comparision.  If the W parameter is omitted
           24  +** or is NULL or is an empty string, then no filtering of the output is
           25  +** done.
    14     26   **
    15         -** The next_char(A,T,F,H) function finds all valid "next" characters for
    16         -** string A given the vocabulary in T.F.  The T.F field should be indexed.
    17         -** If the W value exists and is a non-empty string, then it is an SQL
    18         -** expression that limits the entries in T.F that will be considered.
           27  +** The T.F column should be indexed using collation C or else this routine
           28  +** will be quite slow.
    19     29   **
    20     30   ** For example, suppose an application has a dictionary like this:
    21     31   **
    22     32   **   CREATE TABLE dictionary(word TEXT UNIQUE);
    23     33   **
    24     34   ** Further suppose that for user keypad entry, it is desired to disable
    25     35   ** (gray out) keys that are not valid as the next character.  If the
................................................................................
   180    190     int argc,
   181    191     sqlite3_value **argv
   182    192   ){
   183    193     nextCharContext c;
   184    194     const unsigned char *zTable = sqlite3_value_text(argv[1]);
   185    195     const unsigned char *zField = sqlite3_value_text(argv[2]);
   186    196     const unsigned char *zWhere;
          197  +  const unsigned char *zCollName;
          198  +  char *zWhereClause = 0;
          199  +  char *zColl = 0;
   187    200     char *zSql;
   188    201     int rc;
   189    202   
   190    203     memset(&c, 0, sizeof(c));
   191    204     c.db = sqlite3_context_db_handle(context);
   192    205     c.zPrefix = sqlite3_value_text(argv[0]);
   193    206     c.nPrefix = sqlite3_value_bytes(argv[0]);
   194    207     if( zTable==0 || zField==0 || c.zPrefix==0 ) return;
   195         -  if( argc<4
   196         -   || (zWhere = sqlite3_value_text(argv[3]))==0
   197         -   || zWhere[0]==0
          208  +  if( argc>=4
          209  +   && (zWhere = sqlite3_value_text(argv[3]))!=0
          210  +   && zWhere[0]!=0
          211  +  ){
          212  +    zWhereClause = sqlite3_mprintf("AND (%s)", zWhere);
          213  +    if( zWhereClause==0 ){
          214  +      sqlite3_result_error_nomem(context);
          215  +      return;
          216  +    }
          217  +  }else{
          218  +    zWhereClause = "";
          219  +  }
          220  +  if( argc>=5
          221  +   && (zCollName = sqlite3_value_text(argv[4]))!=0
          222  +   && zCollName[0]!=0 
   198    223     ){
   199         -    zSql = sqlite3_mprintf(
   200         -        "SELECT \"%w\" FROM \"%w\""
   201         -        " WHERE \"%w\">=(?1 || ?2)"
   202         -        "   AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */
   203         -        " ORDER BY 1 ASC LIMIT 1",
   204         -        zField, zTable, zField, zField);
          224  +    zColl = sqlite3_mprintf("collate \"%w\"", zCollName);
          225  +    if( zColl==0 ){
          226  +      sqlite3_result_error_nomem(context);
          227  +      if( zWhereClause[0] ) sqlite3_free(zWhereClause);
          228  +      return;
          229  +    }
   205    230     }else{
   206         -    zSql = sqlite3_mprintf(
   207         -        "SELECT \"%w\" FROM \"%w\""
   208         -        " WHERE \"%w\">=(?1 || ?2)"
   209         -        "   AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */
   210         -        "   AND (%s)"
   211         -        " ORDER BY 1 ASC LIMIT 1",
   212         -        zField, zTable, zField, zField, zWhere);
          231  +    zColl = "";
   213    232     }
          233  +  zSql = sqlite3_mprintf(
          234  +    "SELECT \"%w\" FROM \"%w\""
          235  +    " WHERE \"%w\">=(?1 || ?2) %s"
          236  +    "   AND \"%w\"<=(?1 || char(1114111)) %s" /* 1114111 == 0x10ffff */
          237  +    "   %s"
          238  +    " ORDER BY 1 %s ASC LIMIT 1",
          239  +    zField, zTable, zField, zColl, zField, zColl, zWhereClause, zColl
          240  +  );
          241  +  if( zWhereClause[0] ) sqlite3_free(zWhereClause);
          242  +  if( zColl[0] ) sqlite3_free(zColl);
   214    243     if( zSql==0 ){
   215    244       sqlite3_result_error_nomem(context);
   216    245       return;
   217    246     }
   218    247   
   219    248     rc = sqlite3_prepare_v2(c.db, zSql, -1, &c.pStmt, 0);
   220    249     sqlite3_free(zSql);
................................................................................
   257    286     (void)pzErrMsg;  /* Unused parameter */
   258    287     rc = sqlite3_create_function(db, "next_char", 3, SQLITE_UTF8, 0,
   259    288                                  nextCharFunc, 0, 0);
   260    289     if( rc==SQLITE_OK ){
   261    290       rc = sqlite3_create_function(db, "next_char", 4, SQLITE_UTF8, 0,
   262    291                                    nextCharFunc, 0, 0);
   263    292     }
          293  +  if( rc==SQLITE_OK ){
          294  +    rc = sqlite3_create_function(db, "next_char", 5, SQLITE_UTF8, 0,
          295  +                                 nextCharFunc, 0, 0);
          296  +  }
   264    297     return rc;
   265    298   }

Changes to ext/misc/percentile.c.

   186    186     double ix, vx;
   187    187     p = (Percentile*)sqlite3_aggregate_context(pCtx, 0);
   188    188     if( p==0 ) return;
   189    189     if( p->a==0 ) return;
   190    190     if( p->nUsed ){
   191    191       qsort(p->a, p->nUsed, sizeof(double), doubleCmp);
   192    192       ix = (p->rPct-1.0)*(p->nUsed-1)*0.01;
   193         -    i1 = ix;
          193  +    i1 = (unsigned)ix;
   194    194       i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1;
   195    195       v1 = p->a[i1];
   196    196       v2 = p->a[i2];
   197    197       vx = v1 + (v2-v1)*(ix-i1);
   198    198       sqlite3_result_double(pCtx, vx);
   199    199     }
   200    200     sqlite3_free(p->a);

Changes to ext/misc/regexp.c.

   709    709     int argc, 
   710    710     sqlite3_value **argv
   711    711   ){
   712    712     ReCompiled *pRe;          /* Compiled regular expression */
   713    713     const char *zPattern;     /* The regular expression */
   714    714     const unsigned char *zStr;/* String being searched */
   715    715     const char *zErr;         /* Compile error message */
          716  +  int setAux = 0;           /* True to invoke sqlite3_set_auxdata() */
   716    717   
   717    718     pRe = sqlite3_get_auxdata(context, 0);
   718    719     if( pRe==0 ){
   719    720       zPattern = (const char*)sqlite3_value_text(argv[0]);
   720    721       if( zPattern==0 ) return;
   721    722       zErr = re_compile(&pRe, zPattern, 0);
   722    723       if( zErr ){
................................................................................
   724    725         sqlite3_result_error(context, zErr, -1);
   725    726         return;
   726    727       }
   727    728       if( pRe==0 ){
   728    729         sqlite3_result_error_nomem(context);
   729    730         return;
   730    731       }
   731         -    sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free);
          732  +    setAux = 1;
   732    733     }
   733    734     zStr = (const unsigned char*)sqlite3_value_text(argv[1]);
   734    735     if( zStr!=0 ){
   735    736       sqlite3_result_int(context, re_match(pRe, zStr, -1));
   736    737     }
          738  +  if( setAux ){
          739  +    sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free);
          740  +  }
   737    741   }
   738    742   
   739    743   /*
   740    744   ** Invoke this routine to register the regexp() function with the
   741    745   ** SQLite database connection.
   742    746   */
   743    747   #ifdef _WIN32

Changes to ext/misc/spellfix.c.

  2144   2144         pIdxInfo->aConstraintUsage[iScopeTerm].argvIndex = idx++;
  2145   2145         pIdxInfo->aConstraintUsage[iScopeTerm].omit = 1;
  2146   2146       }
  2147   2147       if( iPlan&(16|32) ){
  2148   2148         pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = idx++;
  2149   2149         pIdxInfo->aConstraintUsage[iDistTerm].omit = 1;
  2150   2150       }
  2151         -    pIdxInfo->estimatedCost = (double)10000;
         2151  +    pIdxInfo->estimatedCost = 1e5;
  2152   2152     }else{
  2153   2153       pIdxInfo->idxNum = 0;
  2154         -    pIdxInfo->estimatedCost = (double)10000000;
         2154  +    pIdxInfo->estimatedCost = 1e50;
  2155   2155     }
  2156   2156     return SQLITE_OK;
  2157   2157   }
  2158   2158   
  2159   2159   /*
  2160   2160   ** Open a new fuzzy-search cursor.
  2161   2161   */

Added ext/misc/vtshim.c.

            1  +/*
            2  +** 2013-06-12
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** A shim that sits between the SQLite virtual table interface and
           14  +** runtimes with garbage collector based memory management.
           15  +*/
           16  +#include "sqlite3ext.h"
           17  +SQLITE_EXTENSION_INIT1
           18  +#include <assert.h>
           19  +#include <string.h>
           20  +
           21  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           22  +
           23  +/* Forward references */
           24  +typedef struct vtshim_aux vtshim_aux;
           25  +typedef struct vtshim_vtab vtshim_vtab;
           26  +typedef struct vtshim_cursor vtshim_cursor;
           27  +
           28  +
           29  +/* The vtshim_aux argument is the auxiliary parameter that is passed
           30  +** into sqlite3_create_module_v2().
           31  +*/
           32  +struct vtshim_aux {
           33  +  void *pChildAux;              /* pAux for child virtual tables */
           34  +  void (*xChildDestroy)(void*); /* Destructor for pChildAux */
           35  +  sqlite3_module *pMod;         /* Methods for child virtual tables */
           36  +  sqlite3 *db;                  /* The database to which we are attached */
           37  +  char *zName;                  /* Name of the module */
           38  +  int bDisposed;                /* True if disposed */
           39  +  vtshim_vtab *pAllVtab;        /* List of all vtshim_vtab objects */
           40  +  sqlite3_module sSelf;         /* Methods used by this shim */
           41  +};
           42  +
           43  +/* A vtshim virtual table object */
           44  +struct vtshim_vtab {
           45  +  sqlite3_vtab base;       /* Base class - must be first */
           46  +  sqlite3_vtab *pChild;    /* Child virtual table */
           47  +  vtshim_aux *pAux;        /* Pointer to vtshim_aux object */
           48  +  vtshim_cursor *pAllCur;  /* List of all cursors */
           49  +  vtshim_vtab **ppPrev;    /* Previous on list */
           50  +  vtshim_vtab *pNext;      /* Next on list */
           51  +};
           52  +
           53  +/* A vtshim cursor object */
           54  +struct vtshim_cursor {
           55  +  sqlite3_vtab_cursor base;    /* Base class - must be first */
           56  +  sqlite3_vtab_cursor *pChild; /* Cursor generated by the managed subclass */
           57  +  vtshim_cursor **ppPrev;      /* Previous on list of all cursors */
           58  +  vtshim_cursor *pNext;        /* Next on list of all cursors */
           59  +};
           60  +
           61  +/* Macro used to copy the child vtable error message to outer vtable */
           62  +#define VTSHIM_COPY_ERRMSG()                                             \
           63  +  do {                                                                   \
           64  +    sqlite3_free(pVtab->base.zErrMsg);                                   \
           65  +    pVtab->base.zErrMsg = sqlite3_mprintf("%s", pVtab->pChild->zErrMsg); \
           66  +  } while (0)
           67  +
           68  +/* Methods for the vtshim module */
           69  +static int vtshimCreate(
           70  +  sqlite3 *db,
           71  +  void *ppAux,
           72  +  int argc,
           73  +  const char *const*argv,
           74  +  sqlite3_vtab **ppVtab,
           75  +  char **pzErr
           76  +){
           77  +  vtshim_aux *pAux = (vtshim_aux*)ppAux;
           78  +  vtshim_vtab *pNew;
           79  +  int rc;
           80  +
           81  +  assert( db==pAux->db );
           82  +  if( pAux->bDisposed ){
           83  +    if( pzErr ){
           84  +      *pzErr = sqlite3_mprintf("virtual table was disposed: \"%s\"",
           85  +                               pAux->zName);
           86  +    }
           87  +    return SQLITE_ERROR;
           88  +  }
           89  +  pNew = sqlite3_malloc( sizeof(*pNew) );
           90  +  *ppVtab = (sqlite3_vtab*)pNew;
           91  +  if( pNew==0 ) return SQLITE_NOMEM;
           92  +  memset(pNew, 0, sizeof(*pNew));
           93  +  rc = pAux->pMod->xCreate(db, pAux->pChildAux, argc, argv,
           94  +                           &pNew->pChild, pzErr);
           95  +  if( rc ){
           96  +    sqlite3_free(pNew);
           97  +    *ppVtab = 0;
           98  +  }
           99  +  pNew->pAux = pAux;
          100  +  pNew->ppPrev = &pAux->pAllVtab;
          101  +  pNew->pNext = pAux->pAllVtab;
          102  +  if( pAux->pAllVtab ) pAux->pAllVtab->ppPrev = &pNew->pNext;
          103  +  pAux->pAllVtab = pNew;
          104  +  return rc;
          105  +}
          106  +
          107  +static int vtshimConnect(
          108  +  sqlite3 *db,
          109  +  void *ppAux,
          110  +  int argc,
          111  +  const char *const*argv,
          112  +  sqlite3_vtab **ppVtab,
          113  +  char **pzErr
          114  +){
          115  +  vtshim_aux *pAux = (vtshim_aux*)ppAux;
          116  +  vtshim_vtab *pNew;
          117  +  int rc;
          118  +
          119  +  assert( db==pAux->db );
          120  +  if( pAux->bDisposed ){
          121  +    if( pzErr ){
          122  +      *pzErr = sqlite3_mprintf("virtual table was disposed: \"%s\"",
          123  +                               pAux->zName);
          124  +    }
          125  +    return SQLITE_ERROR;
          126  +  }
          127  +  pNew = sqlite3_malloc( sizeof(*pNew) );
          128  +  *ppVtab = (sqlite3_vtab*)pNew;
          129  +  if( pNew==0 ) return SQLITE_NOMEM;
          130  +  memset(pNew, 0, sizeof(*pNew));
          131  +  rc = pAux->pMod->xConnect(db, pAux->pChildAux, argc, argv,
          132  +                            &pNew->pChild, pzErr);
          133  +  if( rc ){
          134  +    sqlite3_free(pNew);
          135  +    *ppVtab = 0;
          136  +  }
          137  +  pNew->pAux = pAux;
          138  +  pNew->ppPrev = &pAux->pAllVtab;
          139  +  pNew->pNext = pAux->pAllVtab;
          140  +  if( pAux->pAllVtab ) pAux->pAllVtab->ppPrev = &pNew->pNext;
          141  +  pAux->pAllVtab = pNew;
          142  +  return rc;
          143  +}
          144  +
          145  +static int vtshimBestIndex(
          146  +  sqlite3_vtab *pBase,
          147  +  sqlite3_index_info *pIdxInfo
          148  +){
          149  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          150  +  vtshim_aux *pAux = pVtab->pAux;
          151  +  int rc;
          152  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          153  +  rc = pAux->pMod->xBestIndex(pVtab->pChild, pIdxInfo);
          154  +  if( rc!=SQLITE_OK ){
          155  +    VTSHIM_COPY_ERRMSG();
          156  +  }
          157  +  return rc;
          158  +}
          159  +
          160  +static int vtshimDisconnect(sqlite3_vtab *pBase){
          161  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          162  +  vtshim_aux *pAux = pVtab->pAux;
          163  +  int rc = SQLITE_OK;
          164  +  if( !pAux->bDisposed ){
          165  +    rc = pAux->pMod->xDisconnect(pVtab->pChild);
          166  +  }
          167  +  if( pVtab->pNext ) pVtab->pNext->ppPrev = pVtab->ppPrev;
          168  +  *pVtab->ppPrev = pVtab->pNext;
          169  +  sqlite3_free(pVtab);
          170  +  return rc;
          171  +}
          172  +
          173  +static int vtshimDestroy(sqlite3_vtab *pBase){
          174  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          175  +  vtshim_aux *pAux = pVtab->pAux;
          176  +  int rc = SQLITE_OK;
          177  +  if( !pAux->bDisposed ){
          178  +    rc = pAux->pMod->xDestroy(pVtab->pChild);
          179  +  }
          180  +  if( pVtab->pNext ) pVtab->pNext->ppPrev = pVtab->ppPrev;
          181  +  *pVtab->ppPrev = pVtab->pNext;
          182  +  sqlite3_free(pVtab);
          183  +  return rc;
          184  +}
          185  +
          186  +static int vtshimOpen(sqlite3_vtab *pBase, sqlite3_vtab_cursor **ppCursor){
          187  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          188  +  vtshim_aux *pAux = pVtab->pAux;
          189  +  vtshim_cursor *pCur;
          190  +  int rc;
          191  +  *ppCursor = 0;
          192  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          193  +  pCur = sqlite3_malloc( sizeof(*pCur) );
          194  +  if( pCur==0 ) return SQLITE_NOMEM;
          195  +  memset(pCur, 0, sizeof(*pCur));
          196  +  rc = pAux->pMod->xOpen(pVtab->pChild, &pCur->pChild);
          197  +  if( rc ){
          198  +    sqlite3_free(pCur);
          199  +    VTSHIM_COPY_ERRMSG();
          200  +    return rc;
          201  +  }
          202  +  pCur->pChild->pVtab = pVtab->pChild;
          203  +  *ppCursor = &pCur->base;
          204  +  pCur->ppPrev = &pVtab->pAllCur;
          205  +  if( pVtab->pAllCur ) pVtab->pAllCur->ppPrev = &pCur->pNext;
          206  +  pCur->pNext = pVtab->pAllCur;
          207  +  pVtab->pAllCur = pCur;
          208  +  return SQLITE_OK;
          209  +}
          210  +
          211  +static int vtshimClose(sqlite3_vtab_cursor *pX){
          212  +  vtshim_cursor *pCur = (vtshim_cursor*)pX;
          213  +  vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab;
          214  +  vtshim_aux *pAux = pVtab->pAux;
          215  +  int rc = SQLITE_OK;
          216  +  if( !pAux->bDisposed ){
          217  +    rc = pAux->pMod->xClose(pCur->pChild);
          218  +    if( rc!=SQLITE_OK ){
          219  +      VTSHIM_COPY_ERRMSG();
          220  +    }
          221  +  }
          222  +  if( pCur->pNext ) pCur->pNext->ppPrev = pCur->ppPrev;
          223  +  *pCur->ppPrev = pCur->pNext;
          224  +  sqlite3_free(pCur);
          225  +  return rc;
          226  +}
          227  +
          228  +static int vtshimFilter(
          229  +  sqlite3_vtab_cursor *pX,
          230  +  int idxNum,
          231  +  const char *idxStr,
          232  +  int argc,
          233  +  sqlite3_value **argv
          234  +){
          235  +  vtshim_cursor *pCur = (vtshim_cursor*)pX;
          236  +  vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab;
          237  +  vtshim_aux *pAux = pVtab->pAux;
          238  +  int rc;
          239  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          240  +  rc = pAux->pMod->xFilter(pCur->pChild, idxNum, idxStr, argc, argv);
          241  +  if( rc!=SQLITE_OK ){
          242  +    VTSHIM_COPY_ERRMSG();
          243  +  }
          244  +  return rc;
          245  +}
          246  +
          247  +static int vtshimNext(sqlite3_vtab_cursor *pX){
          248  +  vtshim_cursor *pCur = (vtshim_cursor*)pX;
          249  +  vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab;
          250  +  vtshim_aux *pAux = pVtab->pAux;
          251  +  int rc;
          252  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          253  +  rc = pAux->pMod->xNext(pCur->pChild);
          254  +  if( rc!=SQLITE_OK ){
          255  +    VTSHIM_COPY_ERRMSG();
          256  +  }
          257  +  return rc;
          258  +}
          259  +
          260  +static int vtshimEof(sqlite3_vtab_cursor *pX){
          261  +  vtshim_cursor *pCur = (vtshim_cursor*)pX;
          262  +  vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab;
          263  +  vtshim_aux *pAux = pVtab->pAux;
          264  +  int rc;
          265  +  if( pAux->bDisposed ) return 1;
          266  +  rc = pAux->pMod->xEof(pCur->pChild);
          267  +  VTSHIM_COPY_ERRMSG();
          268  +  return rc;
          269  +}
          270  +
          271  +static int vtshimColumn(sqlite3_vtab_cursor *pX, sqlite3_context *ctx, int i){
          272  +  vtshim_cursor *pCur = (vtshim_cursor*)pX;
          273  +  vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab;
          274  +  vtshim_aux *pAux = pVtab->pAux;
          275  +  int rc;
          276  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          277  +  rc = pAux->pMod->xColumn(pCur->pChild, ctx, i);
          278  +  if( rc!=SQLITE_OK ){
          279  +    VTSHIM_COPY_ERRMSG();
          280  +  }
          281  +  return rc;
          282  +}
          283  +
          284  +static int vtshimRowid(sqlite3_vtab_cursor *pX, sqlite3_int64 *pRowid){
          285  +  vtshim_cursor *pCur = (vtshim_cursor*)pX;
          286  +  vtshim_vtab *pVtab = (vtshim_vtab*)pCur->base.pVtab;
          287  +  vtshim_aux *pAux = pVtab->pAux;
          288  +  int rc;
          289  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          290  +  rc = pAux->pMod->xRowid(pCur->pChild, pRowid);
          291  +  if( rc!=SQLITE_OK ){
          292  +    VTSHIM_COPY_ERRMSG();
          293  +  }
          294  +  return rc;
          295  +}
          296  +
          297  +static int vtshimUpdate(
          298  +  sqlite3_vtab *pBase,
          299  +  int argc,
          300  +  sqlite3_value **argv,
          301  +  sqlite3_int64 *pRowid
          302  +){
          303  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          304  +  vtshim_aux *pAux = pVtab->pAux;
          305  +  int rc;
          306  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          307  +  rc = pAux->pMod->xUpdate(pVtab->pChild, argc, argv, pRowid);
          308  +  if( rc!=SQLITE_OK ){
          309  +    VTSHIM_COPY_ERRMSG();
          310  +  }
          311  +  return rc;
          312  +}
          313  +
          314  +static int vtshimBegin(sqlite3_vtab *pBase){
          315  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          316  +  vtshim_aux *pAux = pVtab->pAux;
          317  +  int rc;
          318  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          319  +  rc = pAux->pMod->xBegin(pVtab->pChild);
          320  +  if( rc!=SQLITE_OK ){
          321  +    VTSHIM_COPY_ERRMSG();
          322  +  }
          323  +  return rc;
          324  +}
          325  +
          326  +static int vtshimSync(sqlite3_vtab *pBase){
          327  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          328  +  vtshim_aux *pAux = pVtab->pAux;
          329  +  int rc;
          330  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          331  +  rc = pAux->pMod->xSync(pVtab->pChild);
          332  +  if( rc!=SQLITE_OK ){
          333  +    VTSHIM_COPY_ERRMSG();
          334  +  }
          335  +  return rc;
          336  +}
          337  +
          338  +static int vtshimCommit(sqlite3_vtab *pBase){
          339  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          340  +  vtshim_aux *pAux = pVtab->pAux;
          341  +  int rc;
          342  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          343  +  rc = pAux->pMod->xCommit(pVtab->pChild);
          344  +  if( rc!=SQLITE_OK ){
          345  +    VTSHIM_COPY_ERRMSG();
          346  +  }
          347  +  return rc;
          348  +}
          349  +
          350  +static int vtshimRollback(sqlite3_vtab *pBase){
          351  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          352  +  vtshim_aux *pAux = pVtab->pAux;
          353  +  int rc;
          354  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          355  +  rc = pAux->pMod->xRollback(pVtab->pChild);
          356  +  if( rc!=SQLITE_OK ){
          357  +    VTSHIM_COPY_ERRMSG();
          358  +  }
          359  +  return rc;
          360  +}
          361  +
          362  +static int vtshimFindFunction(
          363  +  sqlite3_vtab *pBase,
          364  +  int nArg,
          365  +  const char *zName,
          366  +  void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
          367  +  void **ppArg
          368  +){
          369  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          370  +  vtshim_aux *pAux = pVtab->pAux;
          371  +  int rc;
          372  +  if( pAux->bDisposed ) return 0;
          373  +  rc = pAux->pMod->xFindFunction(pVtab->pChild, nArg, zName, pxFunc, ppArg);
          374  +  VTSHIM_COPY_ERRMSG();
          375  +  return rc;
          376  +}
          377  +
          378  +static int vtshimRename(sqlite3_vtab *pBase, const char *zNewName){
          379  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          380  +  vtshim_aux *pAux = pVtab->pAux;
          381  +  int rc;
          382  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          383  +  rc = pAux->pMod->xRename(pVtab->pChild, zNewName);
          384  +  if( rc!=SQLITE_OK ){
          385  +    VTSHIM_COPY_ERRMSG();
          386  +  }
          387  +  return rc;
          388  +}
          389  +
          390  +static int vtshimSavepoint(sqlite3_vtab *pBase, int n){
          391  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          392  +  vtshim_aux *pAux = pVtab->pAux;
          393  +  int rc;
          394  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          395  +  rc = pAux->pMod->xSavepoint(pVtab->pChild, n);
          396  +  if( rc!=SQLITE_OK ){
          397  +    VTSHIM_COPY_ERRMSG();
          398  +  }
          399  +  return rc;
          400  +}
          401  +
          402  +static int vtshimRelease(sqlite3_vtab *pBase, int n){
          403  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          404  +  vtshim_aux *pAux = pVtab->pAux;
          405  +  int rc;
          406  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          407  +  rc = pAux->pMod->xRelease(pVtab->pChild, n);
          408  +  if( rc!=SQLITE_OK ){
          409  +    VTSHIM_COPY_ERRMSG();
          410  +  }
          411  +  return rc;
          412  +}
          413  +
          414  +static int vtshimRollbackTo(sqlite3_vtab *pBase, int n){
          415  +  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
          416  +  vtshim_aux *pAux = pVtab->pAux;
          417  +  int rc;
          418  +  if( pAux->bDisposed ) return SQLITE_ERROR;
          419  +  rc = pAux->pMod->xRollbackTo(pVtab->pChild, n);
          420  +  if( rc!=SQLITE_OK ){
          421  +    VTSHIM_COPY_ERRMSG();
          422  +  }
          423  +  return rc;
          424  +}
          425  +
          426  +/* The destructor function for a disposible module */
          427  +static void vtshimAuxDestructor(void *pXAux){
          428  +  vtshim_aux *pAux = (vtshim_aux*)pXAux;
          429  +  assert( pAux->pAllVtab==0 );
          430  +  if( !pAux->bDisposed && pAux->xChildDestroy ){
          431  +    pAux->xChildDestroy(pAux->pChildAux);
          432  +  }
          433  +  sqlite3_free(pAux->zName);
          434  +  sqlite3_free(pAux->pMod);
          435  +  sqlite3_free(pAux);
          436  +}
          437  +
          438  +static int vtshimCopyModule(
          439  +  const sqlite3_module *pMod,   /* Source module to be copied */
          440  +  sqlite3_module **ppMod        /* Destination for copied module */
          441  +){
          442  +  sqlite3_module *p;
          443  +  if( !pMod || !ppMod ) return SQLITE_ERROR;
          444  +  p = sqlite3_malloc( sizeof(*p) );
          445  +  if( p==0 ) return SQLITE_NOMEM;
          446  +  memcpy(p, pMod, sizeof(*p));
          447  +  *ppMod = p;
          448  +  return SQLITE_OK;
          449  +}
          450  +
          451  +#ifdef _WIN32
          452  +__declspec(dllexport)
          453  +#endif
          454  +void *sqlite3_create_disposable_module(
          455  +  sqlite3 *db,               /* SQLite connection to register module with */
          456  +  const char *zName,         /* Name of the module */
          457  +  const sqlite3_module *p,   /* Methods for the module */
          458  +  void *pClientData,         /* Client data for xCreate/xConnect */
          459  +  void(*xDestroy)(void*)     /* Module destructor function */
          460  +){
          461  +  vtshim_aux *pAux;
          462  +  sqlite3_module *pMod;
          463  +  int rc;
          464  +  pAux = sqlite3_malloc( sizeof(*pAux) );
          465  +  if( pAux==0 ){
          466  +    if( xDestroy ) xDestroy(pClientData);
          467  +    return 0;
          468  +  }
          469  +  rc = vtshimCopyModule(p, &pMod);
          470  +  if( rc!=SQLITE_OK ){
          471  +    sqlite3_free(pAux);
          472  +    return 0;
          473  +  }
          474  +  pAux->pChildAux = pClientData;
          475  +  pAux->xChildDestroy = xDestroy;
          476  +  pAux->pMod = pMod;
          477  +  pAux->db = db;
          478  +  pAux->zName = sqlite3_mprintf("%s", zName);
          479  +  pAux->bDisposed = 0;
          480  +  pAux->pAllVtab = 0;
          481  +  pAux->sSelf.iVersion = p->iVersion<=2 ? p->iVersion : 2;
          482  +  pAux->sSelf.xCreate = p->xCreate ? vtshimCreate : 0;
          483  +  pAux->sSelf.xConnect = p->xConnect ? vtshimConnect : 0;
          484  +  pAux->sSelf.xBestIndex = p->xBestIndex ? vtshimBestIndex : 0;
          485  +  pAux->sSelf.xDisconnect = p->xDisconnect ? vtshimDisconnect : 0;
          486  +  pAux->sSelf.xDestroy = p->xDestroy ? vtshimDestroy : 0;
          487  +  pAux->sSelf.xOpen = p->xOpen ? vtshimOpen : 0;
          488  +  pAux->sSelf.xClose = p->xClose ? vtshimClose : 0;
          489  +  pAux->sSelf.xFilter = p->xFilter ? vtshimFilter : 0;
          490  +  pAux->sSelf.xNext = p->xNext ? vtshimNext : 0;
          491  +  pAux->sSelf.xEof = p->xEof ? vtshimEof : 0;
          492  +  pAux->sSelf.xColumn = p->xColumn ? vtshimColumn : 0;
          493  +  pAux->sSelf.xRowid = p->xRowid ? vtshimRowid : 0;
          494  +  pAux->sSelf.xUpdate = p->xUpdate ? vtshimUpdate : 0;
          495  +  pAux->sSelf.xBegin = p->xBegin ? vtshimBegin : 0;
          496  +  pAux->sSelf.xSync = p->xSync ? vtshimSync : 0;
          497  +  pAux->sSelf.xCommit = p->xCommit ? vtshimCommit : 0;
          498  +  pAux->sSelf.xRollback = p->xRollback ? vtshimRollback : 0;
          499  +  pAux->sSelf.xFindFunction = p->xFindFunction ? vtshimFindFunction : 0;
          500  +  pAux->sSelf.xRename = p->xRename ? vtshimRename : 0;
          501  +  if( p->iVersion>=2 ){
          502  +    pAux->sSelf.xSavepoint = p->xSavepoint ? vtshimSavepoint : 0;
          503  +    pAux->sSelf.xRelease = p->xRelease ? vtshimRelease : 0;
          504  +    pAux->sSelf.xRollbackTo = p->xRollbackTo ? vtshimRollbackTo : 0;
          505  +  }else{
          506  +    pAux->sSelf.xSavepoint = 0;
          507  +    pAux->sSelf.xRelease = 0;
          508  +    pAux->sSelf.xRollbackTo = 0;
          509  +  }
          510  +  rc = sqlite3_create_module_v2(db, zName, &pAux->sSelf,
          511  +                                pAux, vtshimAuxDestructor);
          512  +  return rc==SQLITE_OK ? (void*)pAux : 0;
          513  +}
          514  +
          515  +#ifdef _WIN32
          516  +__declspec(dllexport)
          517  +#endif
          518  +void sqlite3_dispose_module(void *pX){
          519  +  vtshim_aux *pAux = (vtshim_aux*)pX;
          520  +  if( !pAux->bDisposed ){
          521  +    vtshim_vtab *pVtab;
          522  +    vtshim_cursor *pCur;
          523  +    for(pVtab=pAux->pAllVtab; pVtab; pVtab=pVtab->pNext){
          524  +      for(pCur=pVtab->pAllCur; pCur; pCur=pCur->pNext){
          525  +        pAux->pMod->xClose(pCur->pChild);
          526  +      }
          527  +      pAux->pMod->xDisconnect(pVtab->pChild);
          528  +    }
          529  +    pAux->bDisposed = 1;
          530  +    if( pAux->xChildDestroy ) pAux->xChildDestroy(pAux->pChildAux);
          531  +  }
          532  +}
          533  +
          534  +
          535  +#endif /* SQLITE_OMIT_VIRTUALTABLE */
          536  +
          537  +#ifdef _WIN32
          538  +__declspec(dllexport)
          539  +#endif
          540  +int sqlite3_vtshim_init(
          541  +  sqlite3 *db,
          542  +  char **pzErrMsg,
          543  +  const sqlite3_api_routines *pApi
          544  +){
          545  +  SQLITE_EXTENSION_INIT2(pApi);
          546  +  return SQLITE_OK;
          547  +}

Changes to ext/rtree/rtree.c.

  3346   3346     ** the context object when it is no longer required.  */
  3347   3347     return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, 
  3348   3348         (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free
  3349   3349     );
  3350   3350   }
  3351   3351   
  3352   3352   #if !SQLITE_CORE
  3353         -int sqlite3_extension_init(
         3353  +#ifdef _WIN32
         3354  +__declspec(dllexport)
         3355  +#endif
         3356  +int sqlite3_rtree_init(
  3354   3357     sqlite3 *db,
  3355   3358     char **pzErrMsg,
  3356   3359     const sqlite3_api_routines *pApi
  3357   3360   ){
  3358   3361     SQLITE_EXTENSION_INIT2(pApi)
  3359   3362     return sqlite3RtreeInit(db);
  3360   3363   }
  3361   3364   #endif
  3362   3365   
  3363   3366   #endif

Changes to ext/rtree/rtree6.test.

    70     70   do_test rtree6-1.5 {
    71     71     rtree_strategy {SELECT * FROM t1,t2 WHERE k=+ii AND x1<10}
    72     72   } {Ca}
    73     73   
    74     74   do_eqp_test rtree6.2.1 {
    75     75     SELECT * FROM t1,t2 WHERE k=+ii AND x1<10
    76     76   } {
    77         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)} 
    78         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
           77  +  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} 
           78  +  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
    79     79   }
    80     80   
    81     81   do_eqp_test rtree6.2.2 {
    82     82     SELECT * FROM t1,t2 WHERE k=ii AND x1<10
    83     83   } {
    84         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca (~0 rows)} 
    85         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
           84  +  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} 
           85  +  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
    86     86   }
    87     87   
    88     88   do_eqp_test rtree6.2.3 {
    89     89     SELECT * FROM t1,t2 WHERE k=ii
    90     90   } {
    91         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~0 rows)} 
    92         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
           91  +  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} 
           92  +  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
    93     93   }
    94     94   
    95     95   do_eqp_test rtree6.2.4 {
    96     96     SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10
    97     97   } {
    98         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb (~0 rows)} 
    99         -  0 1 1 {SCAN TABLE t2 (~100000 rows)}
           98  +  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb} 
           99  +  0 1 1 {SCAN TABLE t2}
   100    100   }
   101    101   
   102    102   do_eqp_test rtree6.2.5 {
   103    103     SELECT * FROM t1,t2 WHERE k=ii AND x1<v
   104    104   } {
   105         -  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2: (~0 rows)} 
   106         -  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?) (~1 rows)}
          105  +  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:} 
          106  +  0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)}
   107    107   }
   108    108   
   109    109   do_execsql_test rtree6-3.1 {
   110    110     CREATE VIRTUAL TABLE t3 USING rtree(id, x1, x2, y1, y2);
   111    111     INSERT INTO t3 VALUES(NULL, 1, 1, 2, 2);
   112    112     SELECT * FROM t3 WHERE 
   113    113       x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND 

Changes to ext/rtree/rtree8.test.

   164    164       execsql { DELETE FROM t2 WHERE id = $i }
   165    165     }
   166    166     execsql COMMIT
   167    167   } {}
   168    168   
   169    169   
   170    170   finish_test
   171         -

Changes to main.mk.

   589    589   
   590    590   soaktest:	testfixture$(EXE) sqlite3$(EXE)
   591    591   	./testfixture$(EXE) $(TOP)/test/all.test -soak=1
   592    592   
   593    593   fulltestonly:	testfixture$(EXE) sqlite3$(EXE)
   594    594   	./testfixture$(EXE) $(TOP)/test/full.test
   595    595   
          596  +queryplantest:	testfixture$(EXE) sqlite3$(EXE)
          597  +	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner
          598  +
   596    599   test:	testfixture$(EXE) sqlite3$(EXE)
   597    600   	./testfixture$(EXE) $(TOP)/test/veryquick.test
   598    601   
   599    602   # The next two rules are used to support the "threadtest" target. Building
   600    603   # threadtest runs a few thread-safety tests that are implemented in C. This
   601    604   # target is invoked by the releasetest.tcl script.
   602    605   # 

Changes to mkextw.sh.

     7      7   OPTS='-DTHREADSAFE=1 -DBUILD_sqlite=1 -DSQLITE_OS_WIN=1'
     8      8   CC="i386-mingw32msvc-gcc -O2 $OPTS -Itsrc"
     9      9   NM="i386-mingw32msvc-nm"
    10     10   CMD="$CC -c fts2amal.c"
    11     11   echo $CMD
    12     12   $CMD
    13     13   echo 'EXPORTS' >fts2.def
    14         -echo 'sqlite3_extension_init' >>fts2.def
           14  +echo 'sqlite3_fts2_init' >>fts2.def
    15     15   i386-mingw32msvc-dllwrap \
    16     16        --def fts2.def -v --export-all \
    17     17        --driver-name i386-mingw32msvc-gcc \
    18     18        --dlltool-name i386-mingw32msvc-dlltool \
    19     19        --as i386-mingw32msvc-as \
    20     20        --target i386-mingw32 \
    21     21        -dllname fts2.dll -lmsvcrt fts2amal.o
    22     22   zip fts2dll.zip fts2.dll fts2.def

Changes to src/backup.c.

    11     11   *************************************************************************
    12     12   ** This file contains the implementation of the sqlite3_backup_XXX() 
    13     13   ** API functions and the related features.
    14     14   */
    15     15   #include "sqliteInt.h"
    16     16   #include "btreeInt.h"
    17     17   
    18         -/* Macro to find the minimum of two numeric values.
    19         -*/
    20         -#ifndef MIN
    21         -# define MIN(x,y) ((x)<(y)?(x):(y))
    22         -#endif
    23         -
    24     18   /*
    25     19   ** Structure allocated for each backup operation.
    26     20   */
    27     21   struct sqlite3_backup {
    28     22     sqlite3* pDestDb;        /* Destination database handle */
    29     23     Btree *pDest;            /* Destination b-tree file */
    30     24     u32 iDestSchema;         /* Original schema cookie in destination */

Changes to src/btree.c.

  3269   3269   
  3270   3270   /*
  3271   3271   ** This function is called from both BtreeCommitPhaseTwo() and BtreeRollback()
  3272   3272   ** at the conclusion of a transaction.
  3273   3273   */
  3274   3274   static void btreeEndTransaction(Btree *p){
  3275   3275     BtShared *pBt = p->pBt;
         3276  +  sqlite3 *db = p->db;
  3276   3277     assert( sqlite3BtreeHoldsMutex(p) );
  3277   3278   
  3278   3279   #ifndef SQLITE_OMIT_AUTOVACUUM
  3279   3280     pBt->bDoTruncate = 0;
  3280   3281   #endif
  3281         -  if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){
         3282  +  if( p->inTrans>TRANS_NONE && db->nVdbeRead>1 ){
  3282   3283       /* If there are other active statements that belong to this database
  3283   3284       ** handle, downgrade to a read-only transaction. The other statements
  3284   3285       ** may still be reading from the database.  */
  3285   3286       downgradeAllSharedCacheTableLocks(p);
  3286   3287       p->inTrans = TRANS_READ;
  3287   3288     }else{
  3288   3289       /* If the handle had any kind of transaction open, decrement the 

Changes to src/build.c.

  2692   2692     pIndex->aSortOrder = (u8 *)(&pIndex->aiColumn[nCol]);
  2693   2693     pIndex->zName = (char *)(&pIndex->aSortOrder[nCol]);
  2694   2694     zExtra = (char *)(&pIndex->zName[nName+1]);
  2695   2695     memcpy(pIndex->zName, zName, nName+1);
  2696   2696     pIndex->pTable = pTab;
  2697   2697     pIndex->nColumn = pList->nExpr;
  2698   2698     pIndex->onError = (u8)onError;
         2699  +  pIndex->uniqNotNull = onError==OE_Abort;
  2699   2700     pIndex->autoIndex = (u8)(pName==0);
  2700   2701     pIndex->pSchema = db->aDb[iDb].pSchema;
  2701   2702     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
  2702   2703   
  2703   2704     /* Check to see if we should honor DESC requests on index columns
  2704   2705     */
  2705   2706     if( pDb->pSchema->file_format>=4 ){
................................................................................
  2750   2751       }
  2751   2752       if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){
  2752   2753         goto exit_create_index;
  2753   2754       }
  2754   2755       pIndex->azColl[i] = zColl;
  2755   2756       requestedSortOrder = pListItem->sortOrder & sortOrderMask;
  2756   2757       pIndex->aSortOrder[i] = (u8)requestedSortOrder;
         2758  +    if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0;
  2757   2759     }
  2758   2760     sqlite3DefaultRowEst(pIndex);
  2759   2761   
  2760   2762     if( pTab==pParse->pNewTable ){
  2761   2763       /* This routine has been called to create an automatic index as a
  2762   2764       ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or
  2763   2765       ** a PRIMARY KEY or UNIQUE clause following the column definitions.
................................................................................
  3181   3183                  sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) );
  3182   3184       if( pNew==0 ){
  3183   3185         assert( db->mallocFailed );
  3184   3186         return pSrc;
  3185   3187       }
  3186   3188       pSrc = pNew;
  3187   3189       nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1;
  3188         -    pSrc->nAlloc = (u16)nGot;
         3190  +    pSrc->nAlloc = (u8)nGot;
  3189   3191     }
  3190   3192   
  3191   3193     /* Move existing slots that come after the newly inserted slots
  3192   3194     ** out of the way */
  3193   3195     for(i=pSrc->nSrc-1; i>=iStart; i--){
  3194   3196       pSrc->a[i+nExtra] = pSrc->a[i];
  3195   3197     }
  3196         -  pSrc->nSrc += (i16)nExtra;
         3198  +  pSrc->nSrc += (i8)nExtra;
  3197   3199   
  3198   3200     /* Zero the newly allocated slots */
  3199   3201     memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra);
  3200   3202     for(i=iStart; i<iStart+nExtra; i++){
  3201   3203       pSrc->a[i].iCursor = -1;
  3202   3204     }
  3203   3205   

Changes to src/expr.c.

   916    916     for(i=0; i<p->nExpr; i++, pItem++, pOldItem++){
   917    917       Expr *pOldExpr = pOldItem->pExpr;
   918    918       pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags);
   919    919       pItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
   920    920       pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan);
   921    921       pItem->sortOrder = pOldItem->sortOrder;
   922    922       pItem->done = 0;
          923  +    pItem->bSpanIsTab = pOldItem->bSpanIsTab;
   923    924       pItem->iOrderByCol = pOldItem->iOrderByCol;
   924    925       pItem->iAlias = pOldItem->iAlias;
   925    926     }
   926    927     return pNew;
   927    928   }
   928    929   
   929    930   /*
................................................................................
  1592   1593       }
  1593   1594     }
  1594   1595   
  1595   1596     if( eType==0 ){
  1596   1597       /* Could not found an existing table or index to use as the RHS b-tree.
  1597   1598       ** We will have to generate an ephemeral table to do the job.
  1598   1599       */
  1599         -    double savedNQueryLoop = pParse->nQueryLoop;
         1600  +    u32 savedNQueryLoop = pParse->nQueryLoop;
  1600   1601       int rMayHaveNull = 0;
  1601   1602       eType = IN_INDEX_EPH;
  1602   1603       if( prNotFound ){
  1603   1604         *prNotFound = rMayHaveNull = ++pParse->nMem;
  1604   1605         sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound);
  1605   1606       }else{
  1606         -      testcase( pParse->nQueryLoop>(double)1 );
  1607         -      pParse->nQueryLoop = (double)1;
         1607  +      testcase( pParse->nQueryLoop>0 );
         1608  +      pParse->nQueryLoop = 0;
  1608   1609         if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){
  1609   1610           eType = IN_INDEX_ROWID;
  1610   1611         }
  1611   1612       }
  1612   1613       sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID);
  1613   1614       pParse->nQueryLoop = savedNQueryLoop;
  1614   1615     }else{

Changes to src/fkey.c.

   418    418         sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0);
   419    419     
   420    420         sqlite3ReleaseTempReg(pParse, regRec);
   421    421         sqlite3ReleaseTempRange(pParse, regTemp, nCol);
   422    422       }
   423    423     }
   424    424   
   425         -  if( !pFKey->isDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){
          425  +  if( !pFKey->isDeferred && !(pParse->db->flags & SQLITE_DeferFKs)
          426  +   && !pParse->pToplevel 
          427  +   && !pParse->isMultiWrite 
          428  +  ){
   426    429       /* Special case: If this is an INSERT statement that will insert exactly
   427    430       ** one row into the table, raise a constraint immediately instead of
   428    431       ** incrementing a counter. This is necessary as the VM code is being
   429    432       ** generated for will not open a statement transaction.  */
   430    433       assert( nIncr==1 );
   431    434       sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY,
   432    435           OE_Abort, "foreign key constraint failed", P4_STATIC
................................................................................
   809    812   
   810    813     /* Loop through all the foreign key constraints that refer to this table */
   811    814     for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){
   812    815       Index *pIdx = 0;              /* Foreign key index for pFKey */
   813    816       SrcList *pSrc;
   814    817       int *aiCol = 0;
   815    818   
   816         -    if( !pFKey->isDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){
          819  +    if( !pFKey->isDeferred && !(db->flags & SQLITE_DeferFKs) 
          820  +     && !pParse->pToplevel && !pParse->isMultiWrite 
          821  +    ){
   817    822         assert( regOld==0 && regNew!=0 );
   818    823         /* Inserting a single row into a parent table cannot cause an immediate
   819    824         ** foreign key violation. So do nothing in this case.  */
   820    825         continue;
   821    826       }
   822    827   
   823    828       if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){

Changes to src/loadext.c.

   663    663         }
   664    664       }
   665    665       sqlite3_mutex_leave(mutex);
   666    666       assert( (rc&0xff)==rc );
   667    667       return rc;
   668    668     }
   669    669   }
          670  +
          671  +/*
          672  +** Cancel a prior call to sqlite3_auto_extension.  Remove xInit from the
          673  +** set of routines that is invoked for each new database connection, if it
          674  +** is currently on the list.  If xInit is not on the list, then this
          675  +** routine is a no-op.
          676  +**
          677  +** Return 1 if xInit was found on the list and removed.  Return 0 if xInit
          678  +** was not on the list.
          679  +*/
          680  +int sqlite3_cancel_auto_extension(void (*xInit)(void)){
          681  +#if SQLITE_THREADSAFE
          682  +  sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);
          683  +#endif
          684  +  int i;
          685  +  int n = 0;
          686  +  wsdAutoextInit;
          687  +  sqlite3_mutex_enter(mutex);
          688  +  for(i=wsdAutoext.nExt-1; i>=0; i--){
          689  +    if( wsdAutoext.aExt[i]==xInit ){
          690  +      wsdAutoext.nExt--;
          691  +      wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt];
          692  +      n++;
          693  +      break;
          694  +    }
          695  +  }
          696  +  sqlite3_mutex_leave(mutex);
          697  +  return n;
          698  +}
   670    699   
   671    700   /*
   672    701   ** Reset the automatic extension loading mechanism.
   673    702   */
   674    703   void sqlite3_reset_auto_extension(void){
   675    704   #ifndef SQLITE_OMIT_AUTOINIT
   676    705     if( sqlite3_initialize()==SQLITE_OK )

Changes to src/main.c.

  1033   1033       sqlite3ExpirePreparedStatements(db);
  1034   1034       sqlite3ResetAllSchemasOfConnection(db);
  1035   1035     }
  1036   1036     sqlite3BtreeLeaveAll(db);
  1037   1037   
  1038   1038     /* Any deferred constraint violations have now been resolved. */
  1039   1039     db->nDeferredCons = 0;
         1040  +  db->nDeferredImmCons = 0;
         1041  +  db->flags &= ~SQLITE_DeferFKs;
  1040   1042   
  1041   1043     /* If one has been configured, invoke the rollback-hook callback */
  1042   1044     if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
  1043   1045       db->xRollbackCallback(db->pRollbackArg);
  1044   1046     }
  1045   1047   }
  1046   1048   
................................................................................
  1059   1061         case SQLITE_ERROR:              zName = "SQLITE_ERROR";             break;
  1060   1062         case SQLITE_INTERNAL:           zName = "SQLITE_INTERNAL";          break;
  1061   1063         case SQLITE_PERM:               zName = "SQLITE_PERM";              break;
  1062   1064         case SQLITE_ABORT:              zName = "SQLITE_ABORT";             break;
  1063   1065         case SQLITE_ABORT_ROLLBACK:     zName = "SQLITE_ABORT_ROLLBACK";    break;
  1064   1066         case SQLITE_BUSY:               zName = "SQLITE_BUSY";              break;
  1065   1067         case SQLITE_BUSY_RECOVERY:      zName = "SQLITE_BUSY_RECOVERY";     break;
         1068  +      case SQLITE_BUSY_SNAPSHOT:      zName = "SQLITE_BUSY_SNAPSHOT";     break;
  1066   1069         case SQLITE_LOCKED:             zName = "SQLITE_LOCKED";            break;
  1067   1070         case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break;
  1068   1071         case SQLITE_NOMEM:              zName = "SQLITE_NOMEM";             break;
  1069   1072         case SQLITE_READONLY:           zName = "SQLITE_READONLY";          break;
  1070   1073         case SQLITE_READONLY_RECOVERY:  zName = "SQLITE_READONLY_RECOVERY"; break;
  1071   1074         case SQLITE_READONLY_CANTLOCK:  zName = "SQLITE_READONLY_CANTLOCK"; break;
  1072   1075         case SQLITE_READONLY_ROLLBACK:  zName = "SQLITE_READONLY_ROLLBACK"; break;
................................................................................
  1132   1135         case SQLITE_NOTADB:             zName = "SQLITE_NOTADB";            break;
  1133   1136         case SQLITE_ROW:                zName = "SQLITE_ROW";               break;
  1134   1137         case SQLITE_NOTICE:             zName = "SQLITE_NOTICE";            break;
  1135   1138         case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break;
  1136   1139         case SQLITE_NOTICE_RECOVER_ROLLBACK:
  1137   1140                                   zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break;
  1138   1141         case SQLITE_WARNING:            zName = "SQLITE_WARNING";           break;
         1142  +      case SQLITE_WARNING_AUTOINDEX:  zName = "SQLITE_WARNING_AUTOINDEX"; break;
  1139   1143         case SQLITE_DONE:               zName = "SQLITE_DONE";              break;
  1140   1144       }
  1141   1145     }
  1142   1146     if( zName==0 ){
  1143   1147       static char zBuf[50];
  1144   1148       sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc);
  1145   1149       zName = zBuf;
................................................................................
  1292   1296     int nOps,
  1293   1297     int (*xProgress)(void*), 
  1294   1298     void *pArg
  1295   1299   ){
  1296   1300     sqlite3_mutex_enter(db->mutex);
  1297   1301     if( nOps>0 ){
  1298   1302       db->xProgress = xProgress;
  1299         -    db->nProgressOps = nOps;
         1303  +    db->nProgressOps = (unsigned)nOps;
  1300   1304       db->pProgressArg = pArg;
  1301   1305     }else{
  1302   1306       db->xProgress = 0;
  1303   1307       db->nProgressOps = 0;
  1304   1308       db->pProgressArg = 0;
  1305   1309     }
  1306   1310     sqlite3_mutex_leave(db->mutex);
................................................................................
  1390   1394     /* Check if an existing function is being overridden or deleted. If so,
  1391   1395     ** and there are active VMs, then return SQLITE_BUSY. If a function
  1392   1396     ** is being overridden/deleted but there are no active VMs, allow the
  1393   1397     ** operation to continue but invalidate all precompiled statements.
  1394   1398     */
  1395   1399     p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0);
  1396   1400     if( p && p->iPrefEnc==enc && p->nArg==nArg ){
  1397         -    if( db->activeVdbeCnt ){
         1401  +    if( db->nVdbeActive ){
  1398   1402         sqlite3Error(db, SQLITE_BUSY, 
  1399   1403           "unable to delete/modify user-function due to active statements");
  1400   1404         assert( !db->mallocFailed );
  1401   1405         return SQLITE_BUSY;
  1402   1406       }else{
  1403   1407         sqlite3ExpirePreparedStatements(db);
  1404   1408       }
................................................................................
  1971   1975   
  1972   1976     /* Check if this call is removing or replacing an existing collation 
  1973   1977     ** sequence. If so, and there are active VMs, return busy. If there
  1974   1978     ** are no active VMs, invalidate any pre-compiled statements.
  1975   1979     */
  1976   1980     pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0);
  1977   1981     if( pColl && pColl->xCmp ){
  1978         -    if( db->activeVdbeCnt ){
         1982  +    if( db->nVdbeActive ){
  1979   1983         sqlite3Error(db, SQLITE_BUSY, 
  1980   1984           "unable to delete/modify collation sequence due to active statements");
  1981   1985         return SQLITE_BUSY;
  1982   1986       }
  1983   1987       sqlite3ExpirePreparedStatements(db);
  1984   1988   
  1985   1989       /* If collation sequence pColl was created directly by a call to
................................................................................
  2446   2450   
  2447   2451     assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
  2448   2452     memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
  2449   2453     db->autoCommit = 1;
  2450   2454     db->nextAutovac = -1;
  2451   2455     db->szMmap = sqlite3GlobalConfig.szMmap;
  2452   2456     db->nextPagesize = 0;
  2453         -  db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger
         2457  +  db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger
         2458  +#if !defined(SQLITE_DEAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX
         2459  +                 | SQLITE_AutoIndex
         2460  +#endif
  2454   2461   #if SQLITE_DEFAULT_FILE_FORMAT<4
  2455   2462                    | SQLITE_LegacyFileFmt
  2456   2463   #endif
  2457   2464   #ifdef SQLITE_ENABLE_LOAD_EXTENSION
  2458   2465                    | SQLITE_LoadExtension
  2459   2466   #endif
  2460   2467   #if SQLITE_DEFAULT_RECURSIVE_TRIGGERS

Changes to src/memjournal.c.

    27     27   ** The size chosen is a little less than a power of two.  That way,
    28     28   ** the FileChunk object will have a size that almost exactly fills
    29     29   ** a power-of-two allocation.  This mimimizes wasted space in power-of-two
    30     30   ** memory allocators.
    31     31   */
    32     32   #define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*)))
    33     33   
    34         -/* Macro to find the minimum of two numeric values.
    35         -*/
    36         -#ifndef MIN
    37         -# define MIN(x,y) ((x)<(y)?(x):(y))
    38         -#endif
    39         -
    40     34   /*
    41     35   ** The rollback journal is composed of a linked list of these structures.
    42     36   */
    43     37   struct FileChunk {
    44     38     FileChunk *pNext;               /* Next chunk in the journal */
    45     39     u8 zChunk[JOURNAL_CHUNKSIZE];   /* Content of this chunk */
    46     40   };

Changes to src/os_win.c.

    80     80   
    81     81   /*
    82     82   ** This file mapping API is common to both Win32 and WinRT.
    83     83   */
    84     84   WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID);
    85     85   #endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */
    86     86   
    87         -/*
    88         -** Macro to find the minimum of two numeric values.
    89         -*/
    90         -#ifndef MIN
    91         -# define MIN(x,y) ((x)<(y)?(x):(y))
    92         -#endif
    93         -
    94     87   /*
    95     88   ** Some Microsoft compilers lack this definition.
    96     89   */
    97     90   #ifndef INVALID_FILE_ATTRIBUTES
    98     91   # define INVALID_FILE_ATTRIBUTES ((DWORD)-1) 
    99     92   #endif
   100     93   

Changes to src/pager.c.

  1808   1808     */
  1809   1809     if( pPager->errCode ){
  1810   1810       assert( !MEMDB );
  1811   1811       pager_reset(pPager);
  1812   1812       pPager->changeCountDone = pPager->tempFile;
  1813   1813       pPager->eState = PAGER_OPEN;
  1814   1814       pPager->errCode = SQLITE_OK;
         1815  +    if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0);
  1815   1816     }
  1816   1817   
  1817   1818     pPager->journalOff = 0;
  1818   1819     pPager->journalHdr = 0;
  1819   1820     pPager->setMaster = 0;
  1820   1821   }
  1821   1822   
................................................................................
  3374   3375   
  3375   3376   /*
  3376   3377   ** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap.
  3377   3378   */
  3378   3379   static void pagerFixMaplimit(Pager *pPager){
  3379   3380   #if SQLITE_MAX_MMAP_SIZE>0
  3380   3381     sqlite3_file *fd = pPager->fd;
  3381         -  if( isOpen(fd) ){
         3382  +  if( isOpen(fd) && fd->pMethods->iVersion>=3 ){
  3382   3383       sqlite3_int64 sz;
  3383         -    pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->szMmap>0;
  3384   3384       sz = pPager->szMmap;
         3385  +    pPager->bUseFetch = (sz>0);
  3385   3386       sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz);
  3386   3387     }
  3387   3388   #endif
  3388   3389   }
  3389   3390   
  3390   3391   /*
  3391   3392   ** Change the maximum size of any memory mapping made of the database file.

Changes to src/pragma.c.

   173    173       { "short_column_names",       SQLITE_ShortColNames },
   174    174       { "count_changes",            SQLITE_CountRows     },
   175    175       { "empty_result_callbacks",   SQLITE_NullCallback  },
   176    176       { "legacy_file_format",       SQLITE_LegacyFileFmt },
   177    177       { "fullfsync",                SQLITE_FullFSync     },
   178    178       { "checkpoint_fullfsync",     SQLITE_CkptFullFSync },
   179    179       { "reverse_unordered_selects", SQLITE_ReverseOrder  },
          180  +    { "query_only",               SQLITE_QueryOnly     },
   180    181   #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
   181    182       { "automatic_index",          SQLITE_AutoIndex     },
   182    183   #endif
   183    184   #ifdef SQLITE_DEBUG
   184    185       { "sql_trace",                SQLITE_SqlTrace      },
   185    186       { "vdbe_listing",             SQLITE_VdbeListing   },
   186    187       { "vdbe_trace",               SQLITE_VdbeTrace     },
................................................................................
   193    194   #endif
   194    195       /* The following is VERY experimental */
   195    196       { "writable_schema",          SQLITE_WriteSchema|SQLITE_RecoveryMode },
   196    197   
   197    198       /* TODO: Maybe it shouldn't be possible to change the ReadUncommitted
   198    199       ** flag if there are any active statements. */
   199    200       { "read_uncommitted",         SQLITE_ReadUncommitted },
   200         -    { "recursive_triggers",       SQLITE_RecTriggers },
          201  +    { "recursive_triggers",       SQLITE_RecTriggers   },
   201    202   
   202    203       /* This flag may only be set if both foreign-key and trigger support
   203    204       ** are present in the build.  */
   204    205   #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER)
   205         -    { "foreign_keys",             SQLITE_ForeignKeys },
          206  +    { "foreign_keys",             SQLITE_ForeignKeys   },
          207  +    { "defer_foreign_keys",       SQLITE_DeferFKs      },
   206    208   #endif
   207    209     };
   208    210     int i;
   209    211     const struct sPragmaType *p;
   210    212     for(i=0, p=aPragma; i<ArraySize(aPragma); i++, p++){
   211    213       if( sqlite3StrICmp(zLeft, p->zName)==0 ){
   212    214         sqlite3 *db = pParse->db;
................................................................................
   224    226               mask &= ~(SQLITE_ForeignKeys);
   225    227             }
   226    228   
   227    229             if( sqlite3GetBoolean(zRight, 0) ){
   228    230               db->flags |= mask;
   229    231             }else{
   230    232               db->flags &= ~mask;
          233  +            if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0;
   231    234             }
   232    235   
   233    236             /* Many of the flag-pragmas modify the code generated by the SQL 
   234    237             ** compiler (eg. count_changes). So add an opcode to expire all
   235    238             ** compiled SQL statements after modifying a pragma value.
   236    239             */
   237    240             sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);

Changes to src/prepare.c.

   588    588         }
   589    589       }
   590    590     }
   591    591   
   592    592     sqlite3VtabUnlockList(db);
   593    593   
   594    594     pParse->db = db;
   595         -  pParse->nQueryLoop = (double)1;
          595  +  pParse->nQueryLoop = 0;  /* Logarithmic, so 0 really means 1 */
   596    596     if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){
   597    597       char *zSqlCopy;
   598    598       int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH];
   599    599       testcase( nBytes==mxLen );
   600    600       testcase( nBytes==mxLen+1 );
   601    601       if( nBytes>mxLen ){
   602    602         sqlite3Error(db, SQLITE_TOOBIG, "statement too long");
................................................................................
   610    610         pParse->zTail = &zSql[pParse->zTail-zSqlCopy];
   611    611       }else{
   612    612         pParse->zTail = &zSql[nBytes];
   613    613       }
   614    614     }else{
   615    615       sqlite3RunParser(pParse, zSql, &zErrMsg);
   616    616     }
   617         -  assert( 1==(int)pParse->nQueryLoop );
          617  +  assert( 0==pParse->nQueryLoop );
   618    618   
   619    619     if( db->mallocFailed ){
   620    620       pParse->rc = SQLITE_NOMEM;
   621    621     }
   622    622     if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK;
   623    623     if( pParse->checkSchema ){
   624    624       schemaIsValid(pParse);
................................................................................
   805    805     const char *zTail8 = 0;
   806    806     int rc = SQLITE_OK;
   807    807   
   808    808     assert( ppStmt );
   809    809     *ppStmt = 0;
   810    810     if( !sqlite3SafetyCheckOk(db) ){
   811    811       return SQLITE_MISUSE_BKPT;
          812  +  }
          813  +  if( nBytes>=0 ){
          814  +    int sz;
          815  +    const char *z = (const char*)zSql;
          816  +    for(sz=0; sz<nBytes && (z[sz]!=0 || z[sz+1]!=0); sz += 2){}
          817  +    nBytes = sz;
   812    818     }
   813    819     sqlite3_mutex_enter(db->mutex);
   814    820     zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE);
   815    821     if( zSql8 ){
   816    822       rc = sqlite3LockAndPrepare(db, zSql8, -1, saveSqlFlag, 0, ppStmt, &zTail8);
   817    823     }
   818    824   

Changes to src/printf.c.

   464    464             flag_rtz = flag_altform2;
   465    465           }
   466    466           if( xtype==etEXP ){
   467    467             e2 = 0;
   468    468           }else{
   469    469             e2 = exp;
   470    470           }
   471         -        if( e2+precision+width > etBUFSIZE - 15 ){
   472         -          bufpt = zExtra = sqlite3Malloc( e2+precision+width+15 );
          471  +        if( MAX(e2,0)+precision+width > etBUFSIZE - 15 ){
          472  +          bufpt = zExtra = sqlite3Malloc( MAX(e2,0)+precision+width+15 );
   473    473             if( bufpt==0 ){
   474    474               pAccum->mallocFailed = 1;
   475    475               return;
   476    476             }
   477    477           }
   478    478           zOut = bufpt;
   479    479           nsd = 16 + flag_altform2*10;

Changes to src/select.c.

  1534   1534       v = sqlite3GetVdbe(pParse);
  1535   1535       if( NEVER(v==0) ) return;  /* VDBE should have already been allocated */
  1536   1536       if( sqlite3ExprIsInteger(p->pLimit, &n) ){
  1537   1537         sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
  1538   1538         VdbeComment((v, "LIMIT counter"));
  1539   1539         if( n==0 ){
  1540   1540           sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
  1541         -      }else{
  1542         -        if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n;
         1541  +      }else if( n>=0 && p->nSelectRow>(u64)n ){
         1542  +        p->nSelectRow = n;
  1543   1543         }
  1544   1544       }else{
  1545   1545         sqlite3ExprCode(pParse, p->pLimit, iLimit);
  1546   1546         sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
  1547   1547         VdbeComment((v, "LIMIT counter"));
  1548   1548         sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
  1549   1549       }
................................................................................
  1729   1729         rc = sqlite3Select(pParse, p, &dest);
  1730   1730         testcase( rc!=SQLITE_OK );
  1731   1731         pDelete = p->pPrior;
  1732   1732         p->pPrior = pPrior;
  1733   1733         p->nSelectRow += pPrior->nSelectRow;
  1734   1734         if( pPrior->pLimit
  1735   1735          && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit)
  1736         -       && p->nSelectRow > (double)nLimit 
         1736  +       && nLimit>0 && p->nSelectRow > (u64)nLimit 
  1737   1737         ){
  1738         -        p->nSelectRow = (double)nLimit;
         1738  +        p->nSelectRow = nLimit;
  1739   1739         }
  1740   1740         if( addr ){
  1741   1741           sqlite3VdbeJumpHere(v, addr);
  1742   1742         }
  1743   1743         break;
  1744   1744       }
  1745   1745       case TK_EXCEPT:
................................................................................
  3880   3880   #ifndef SQLITE_OMIT_EXPLAIN
  3881   3881   static void explainSimpleCount(
  3882   3882     Parse *pParse,                  /* Parse context */
  3883   3883     Table *pTab,                    /* Table being queried */
  3884   3884     Index *pIdx                     /* Index used to optimize scan, or NULL */
  3885   3885   ){
  3886   3886     if( pParse->explain==2 ){
  3887         -    char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s %s%s(~%d rows)",
         3887  +    char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s%s%s",
  3888   3888           pTab->zName, 
  3889         -        pIdx ? "USING COVERING INDEX " : "",
  3890         -        pIdx ? pIdx->zName : "",
  3891         -        pTab->nRowEst
         3889  +        pIdx ? " USING COVERING INDEX " : "",
         3890  +        pIdx ? pIdx->zName : ""
  3892   3891       );
  3893   3892       sqlite3VdbeAddOp4(
  3894   3893           pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC
  3895   3894       );
  3896   3895     }
  3897   3896   }
  3898   3897   #else
................................................................................
  4235   4234     if( pDest->eDest==SRT_EphemTab ){
  4236   4235       sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr);
  4237   4236     }
  4238   4237   
  4239   4238     /* Set the limiter.
  4240   4239     */
  4241   4240     iEnd = sqlite3VdbeMakeLabel(v);
  4242         -  p->nSelectRow = (double)LARGEST_INT64;
         4241  +  p->nSelectRow = LARGEST_INT64;
  4243   4242     computeLimitRegisters(pParse, p, iEnd);
  4244   4243     if( p->iLimit==0 && addrSortIndex>=0 ){
  4245   4244       sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen;
  4246   4245       p->selFlags |= SF_UseSorter;
  4247   4246     }
  4248   4247   
  4249   4248     /* Open a virtual index to use for the distinct set.
................................................................................
  4258   4257       sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED;
  4259   4258     }else{
  4260   4259       sDistinct.eTnctType = WHERE_DISTINCT_NOOP;
  4261   4260     }
  4262   4261   
  4263   4262     if( !isAgg && pGroupBy==0 ){
  4264   4263       /* No aggregate functions and no GROUP BY clause */
  4265         -    ExprList *pDist = (sDistinct.isTnct ? p->pEList : 0);
         4264  +    u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0);
  4266   4265   
  4267   4266       /* Begin the database scan. */
  4268         -    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, pDist, 0,0);
         4267  +    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, p->pEList,
         4268  +                               wctrlFlags, 0);
  4269   4269       if( pWInfo==0 ) goto select_end;
  4270         -    if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
  4271         -    if( pWInfo->eDistinct ) sDistinct.eTnctType = pWInfo->eDistinct;
  4272         -    if( pOrderBy && pWInfo->nOBSat==pOrderBy->nExpr ) pOrderBy = 0;
         4270  +    if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){
         4271  +      p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo);
         4272  +    }
         4273  +    if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){
         4274  +      sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo);
         4275  +    }
         4276  +    if( pOrderBy && sqlite3WhereIsOrdered(pWInfo) ) pOrderBy = 0;
  4273   4277   
  4274   4278       /* If sorting index that was created by a prior OP_OpenEphemeral 
  4275   4279       ** instruction ended up not being needed, then change the OP_OpenEphemeral
  4276   4280       ** into an OP_Noop.
  4277   4281       */
  4278   4282       if( addrSortIndex>=0 && pOrderBy==0 ){
  4279   4283         sqlite3VdbeChangeToNoop(v, addrSortIndex);
  4280   4284         p->addrOpenEphm[2] = -1;
  4281   4285       }
  4282   4286   
  4283   4287       /* Use the standard inner loop. */
  4284   4288       selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest,
  4285         -                    pWInfo->iContinue, pWInfo->iBreak);
         4289  +                    sqlite3WhereContinueLabel(pWInfo),
         4290  +                    sqlite3WhereBreakLabel(pWInfo));
  4286   4291   
  4287   4292       /* End the database scan loop.
  4288   4293       */
  4289   4294       sqlite3WhereEnd(pWInfo);
  4290   4295     }else{
  4291   4296       /* This case when there exist aggregate functions or a GROUP BY clause
  4292   4297       ** or both */
................................................................................
  4311   4316   
  4312   4317         for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){
  4313   4318           pItem->iAlias = 0;
  4314   4319         }
  4315   4320         for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){
  4316   4321           pItem->iAlias = 0;
  4317   4322         }
  4318         -      if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100;
         4323  +      if( p->nSelectRow>100 ) p->nSelectRow = 100;
  4319   4324       }else{
  4320         -      p->nSelectRow = (double)1;
         4325  +      p->nSelectRow = 1;
  4321   4326       }
  4322   4327   
  4323   4328    
  4324   4329       /* Create a label to jump to when we want to abort the query */
  4325   4330       addrEnd = sqlite3VdbeMakeLabel(v);
  4326   4331   
  4327   4332       /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in
................................................................................
  4393   4398   
  4394   4399         /* Begin a loop that will extract all source rows in GROUP BY order.
  4395   4400         ** This might involve two separate loops with an OP_Sort in between, or
  4396   4401         ** it might be a single loop that uses an index to extract information
  4397   4402         ** in the right order to begin with.
  4398   4403         */
  4399   4404         sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
  4400         -      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 0, 0);
         4405  +      pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 
         4406  +                                 WHERE_GROUPBY, 0);
  4401   4407         if( pWInfo==0 ) goto select_end;
  4402         -      if( pWInfo->nOBSat==pGroupBy->nExpr ){
         4408  +      if( sqlite3WhereIsOrdered(pWInfo) ){
  4403   4409           /* The optimizer is able to deliver rows in group by order so
  4404   4410           ** we do not have to sort.  The OP_OpenEphemeral table will be
  4405   4411           ** cancelled later because we still need to use the pKeyInfo
  4406   4412           */
  4407   4413           groupBySort = 0;
  4408   4414         }else{
  4409   4415           /* Rows are coming out in undetermined order.  We have to push
................................................................................
  4676   4682           pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0);
  4677   4683           if( pWInfo==0 ){
  4678   4684             sqlite3ExprListDelete(db, pDel);
  4679   4685             goto select_end;
  4680   4686           }
  4681   4687           updateAccumulator(pParse, &sAggInfo);
  4682   4688           assert( pMinMax==0 || pMinMax->nExpr==1 );
  4683         -        if( pWInfo->nOBSat>0 ){
  4684         -          sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak);
         4689  +        if( sqlite3WhereIsOrdered(pWInfo) ){
         4690  +          sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo));
  4685   4691             VdbeComment((v, "%s() by index",
  4686   4692                   (flag==WHERE_ORDERBY_MIN?"min":"max")));
  4687   4693           }
  4688   4694           sqlite3WhereEnd(pWInfo);
  4689   4695           finalizeAggFunctions(pParse, &sAggInfo);
  4690   4696         }
  4691   4697   

Changes to src/shell.c.

    63     63   #if defined(_WIN32) || defined(WIN32)
    64     64   # include <io.h>
    65     65   #define isatty(h) _isatty(h)
    66     66   #define access(f,m) _access((f),(m))
    67     67   #undef popen
    68     68   #define popen(a,b) _popen((a),(b))
    69     69   #undef pclose
    70         -#define pclose(x) _pclose(x)
           70  +#define pclose _pclose
    71     71   #else
    72     72   /* Make sure isatty() has a prototype.
    73     73   */
    74     74   extern int isatty(int);
    75     75   #endif
    76     76   
    77     77   #if defined(_WIN32_WCE)
................................................................................
  1105   1105     if( pArg && pArg->out && db && pArg->pStmt ){
  1106   1106       iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset);
  1107   1107       fprintf(pArg->out, "Fullscan Steps:                      %d\n", iCur);
  1108   1108       iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset);
  1109   1109       fprintf(pArg->out, "Sort Operations:                     %d\n", iCur);
  1110   1110       iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX, bReset);
  1111   1111       fprintf(pArg->out, "Autoindex Inserts:                   %d\n", iCur);
         1112  +    iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset);
         1113  +    fprintf(pArg->out, "Virtual Machine Steps:               %d\n", iCur);
  1112   1114     }
  1113   1115   
  1114   1116     return 0;
  1115   1117   }
  1116   1118   
  1117   1119   /*
  1118   1120   ** Execute a statement or set of statements.  Print 
................................................................................
  1486   1488   
  1487   1489   /*
  1488   1490   ** Do C-language style dequoting.
  1489   1491   **
  1490   1492   **    \t    -> tab
  1491   1493   **    \n    -> newline
  1492   1494   **    \r    -> carriage return
         1495  +**    \"    -> "
  1493   1496   **    \NNN  -> ascii character NNN in octal
  1494   1497   **    \\    -> backslash
  1495   1498   */
  1496   1499   static void resolve_backslashes(char *z){
  1497   1500     int i, j;
  1498   1501     char c;
  1499   1502     for(i=j=0; (c = z[i])!=0; i++, j++){
................................................................................
  1501   1504         c = z[++i];
  1502   1505         if( c=='n' ){
  1503   1506           c = '\n';
  1504   1507         }else if( c=='t' ){
  1505   1508           c = '\t';
  1506   1509         }else if( c=='r' ){
  1507   1510           c = '\r';
         1511  +      }else if( c=='\\' ){
         1512  +        c = '\\';
  1508   1513         }else if( c>='0' && c<='7' ){
  1509   1514           c -= '0';
  1510   1515           if( z[i+1]>='0' && z[i+1]<='7' ){
  1511   1516             i++;
  1512   1517             c = (c<<3) + z[i] - '0';
  1513   1518             if( z[i+1]>='0' && z[i+1]<='7' ){
  1514   1519               i++;
................................................................................
  1643   1648   ** A no-op routine that runs with the ".breakpoint" doc-command.  This is
  1644   1649   ** a useful spot to set a debugger breakpoint.
  1645   1650   */
  1646   1651   static void test_breakpoint(void){
  1647   1652     static int nCall = 0;
  1648   1653     nCall++;
  1649   1654   }
         1655  +
         1656  +/*
         1657  +** An object used to read a CSV file
         1658  +*/
         1659  +typedef struct CSVReader CSVReader;
         1660  +struct CSVReader {
         1661  +  const char *zFile;  /* Name of the input file */
         1662  +  FILE *in;           /* Read the CSV text from this input stream */
         1663  +  char *z;            /* Accumulated text for a field */
         1664  +  int n;              /* Number of bytes in z */
         1665  +  int nAlloc;         /* Space allocated for z[] */
         1666  +  int nLine;          /* Current line number */
         1667  +  int cTerm;          /* Character that terminated the most recent field */
         1668  +  int cSeparator;     /* The separator character.  (Usually ",") */
         1669  +};
         1670  +
         1671  +/* Append a single byte to z[] */
         1672  +static void csv_append_char(CSVReader *p, int c){
         1673  +  if( p->n+1>=p->nAlloc ){
         1674  +    p->nAlloc += p->nAlloc + 100;
         1675  +    p->z = sqlite3_realloc(p->z, p->nAlloc);
         1676  +    if( p->z==0 ){
         1677  +      fprintf(stderr, "out of memory\n");
         1678  +      exit(1);
         1679  +    }
         1680  +  }
         1681  +  p->z[p->n++] = (char)c;
         1682  +}
         1683  +
         1684  +/* Read a single field of CSV text.  Compatible with rfc4180 and extended
         1685  +** with the option of having a separator other than ",".
         1686  +**
         1687  +**   +  Input comes from p->in.
         1688  +**   +  Store results in p->z of length p->n.  Space to hold p->z comes
         1689  +**      from sqlite3_malloc().
         1690  +**   +  Use p->cSep as the separator.  The default is ",".
         1691  +**   +  Keep track of the line number in p->nLine.
         1692  +**   +  Store the character that terminates the field in p->cTerm.  Store
         1693  +**      EOF on end-of-file.
         1694  +**   +  Report syntax errors on stderr
         1695  +*/
         1696  +static char *csv_read_one_field(CSVReader *p){
         1697  +  int c, pc;
         1698  +  int cSep = p->cSeparator;
         1699  +  p->n = 0;
         1700  +  c = fgetc(p->in);
         1701  +  if( c==EOF || seenInterrupt ){
         1702  +    p->cTerm = EOF;
         1703  +    return 0;
         1704  +  }
         1705  +  if( c=='"' ){
         1706  +    int startLine = p->nLine;
         1707  +    int cQuote = c;
         1708  +    pc = 0;
         1709  +    while( 1 ){
         1710  +      c = fgetc(p->in);
         1711  +      if( c=='\n' ) p->nLine++;
         1712  +      if( c==cQuote ){
         1713  +        if( pc==cQuote ){
         1714  +          pc = 0;
         1715  +          continue;
         1716  +        }
         1717  +      }
         1718  +      if( (c==cSep && pc==cQuote)
         1719  +       || (c=='\n' && pc==cQuote)
         1720  +       || (c=='\n' && pc=='\r' && p->n>2 && p->z[p->n-2]==cQuote)
         1721  +       || (c==EOF && pc==cQuote)
         1722  +      ){
         1723  +        do{ p->n--; }while( p->z[p->n]!=cQuote );
         1724  +        p->cTerm = c;
         1725  +        break;
         1726  +      }
         1727  +      if( pc==cQuote && c!='\r' ){
         1728  +        fprintf(stderr, "%s:%d: unescaped %c character\n",
         1729  +                p->zFile, p->nLine, cQuote);
         1730  +      }
         1731  +      if( c==EOF ){
         1732  +        fprintf(stderr, "%s:%d: unterminated %c-quoted field\n",
         1733  +                p->zFile, startLine, cQuote);
         1734  +        p->cTerm = EOF;
         1735  +        break;
         1736  +      }
         1737  +      csv_append_char(p, c);
         1738  +      pc = c;
         1739  +    }
         1740  +  }else{
         1741  +    while( c!=EOF && c!=cSep && c!='\n' ){
         1742  +      csv_append_char(p, c);
         1743  +      c = fgetc(p->in);
         1744  +    }
         1745  +    if( c=='\n' ){
         1746  +      p->nLine++;
         1747  +      if( p->n>1 && p->z[p->n-1]=='\r' ) p->n--;
         1748  +    }
         1749  +    p->cTerm = c;
         1750  +  }
         1751  +  if( p->z ) p->z[p->n] = 0;
         1752  +  return p->z;
         1753  +}
  1650   1754   
  1651   1755   /*
  1652   1756   ** If an input line begins with "." then invoke this routine to
  1653   1757   ** process that line.
  1654   1758   **
  1655   1759   ** Return 1 on error, 2 to exit, and 0 otherwise.
  1656   1760   */
................................................................................
  1665   1769     */
  1666   1770     while( zLine[i] && nArg<ArraySize(azArg) ){
  1667   1771       while( IsSpace(zLine[i]) ){ i++; }
  1668   1772       if( zLine[i]==0 ) break;
  1669   1773       if( zLine[i]=='\'' || zLine[i]=='"' ){
  1670   1774         int delim = zLine[i++];
  1671   1775         azArg[nArg++] = &zLine[i];
  1672         -      while( zLine[i] && zLine[i]!=delim ){ i++; }
         1776  +      while( zLine[i] && zLine[i]!=delim ){ 
         1777  +        if( zLine[i]=='\\' && delim=='"' && zLine[i+1]!=0 ) i++;
         1778  +        i++; 
         1779  +      }
  1673   1780         if( zLine[i]==delim ){
  1674   1781           zLine[i++] = 0;
  1675   1782         }
  1676   1783         if( delim=='"' ) resolve_backslashes(azArg[nArg-1]);
  1677   1784       }else{
  1678   1785         azArg[nArg++] = &zLine[i];
  1679   1786         while( zLine[i] && !IsSpace(zLine[i]) ){ i++; }
................................................................................
  1686   1793     */
  1687   1794     if( nArg==0 ) return 0; /* no tokens, no error */
  1688   1795     n = strlen30(azArg[0]);
  1689   1796     c = azArg[0][0];
  1690   1797     if( c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0 ){
  1691   1798       const char *zDestFile = 0;
  1692   1799       const char *zDb = 0;
  1693         -    const char *zKey = 0;
  1694   1800       sqlite3 *pDest;
  1695   1801       sqlite3_backup *pBackup;
  1696   1802       int j;
  1697   1803       for(j=1; j<nArg; j++){
  1698   1804         const char *z = azArg[j];
  1699   1805         if( z[0]=='-' ){
  1700   1806           while( z[0]=='-' ) z++;
  1701         -        if( strcmp(z,"key")==0 && j<nArg-1 ){
  1702         -          zKey = azArg[++j];
  1703         -        }else
         1807  +        /* No options to process at this time */
  1704   1808           {
  1705   1809             fprintf(stderr, "unknown option: %s\n", azArg[j]);
  1706   1810             return 1;
  1707   1811           }
  1708   1812         }else if( zDestFile==0 ){
  1709   1813           zDestFile = azArg[j];
  1710   1814         }else if( zDb==0 ){
................................................................................
  1722   1826       if( zDb==0 ) zDb = "main";
  1723   1827       rc = sqlite3_open(zDestFile, &pDest);
  1724   1828       if( rc!=SQLITE_OK ){
  1725   1829         fprintf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
  1726   1830         sqlite3_close(pDest);
  1727   1831         return 1;
  1728   1832       }
  1729         -#ifdef SQLITE_HAS_CODEC
  1730         -    sqlite3_key(pDest, zKey, (int)strlen(zKey));
  1731         -#else
  1732         -    (void)zKey;
  1733         -#endif
  1734   1833       open_db(p);
  1735   1834       pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
  1736   1835       if( pBackup==0 ){
  1737   1836         fprintf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
  1738   1837         sqlite3_close(pDest);
  1739   1838         return 1;
  1740   1839       }
................................................................................
  1882   1981       if( HAS_TIMER ){
  1883   1982         fprintf(stderr,"%s",zTimerHelp);
  1884   1983       }
  1885   1984     }else
  1886   1985   
  1887   1986     if( c=='i' && strncmp(azArg[0], "import", n)==0 && nArg==3 ){
  1888   1987       char *zTable = azArg[2];    /* Insert data into this table */
  1889         -    char *zFile = azArg[1];     /* The file from which to extract data */
         1988  +    char *zFile = azArg[1];     /* Name of file to extra content from */
  1890   1989       sqlite3_stmt *pStmt = NULL; /* A statement */
  1891   1990       int nCol;                   /* Number of columns in the table */
  1892   1991       int nByte;                  /* Number of bytes in an SQL string */
  1893   1992       int i, j;                   /* Loop counters */
  1894   1993       int nSep;                   /* Number of bytes in p->separator[] */
  1895   1994       char *zSql;                 /* An SQL statement */
  1896         -    char *zLine;                /* A single line of input from the file */
  1897         -    char **azCol;               /* zLine[] broken up into columns */
  1898         -    char *zCommit;              /* How to commit changes */   
  1899         -    FILE *in;                   /* The input file */
  1900         -    int lineno = 0;             /* Line number of input file */
         1995  +    CSVReader sCsv;             /* Reader context */
         1996  +    int (*xCloser)(FILE*);      /* Procedure to close th3 connection */
  1901   1997   
         1998  +    seenInterrupt = 0;
         1999  +    memset(&sCsv, 0, sizeof(sCsv));
  1902   2000       open_db(p);
  1903   2001       nSep = strlen30(p->separator);
  1904   2002       if( nSep==0 ){
  1905   2003         fprintf(stderr, "Error: non-null separator required for import\n");
  1906   2004         return 1;
  1907   2005       }
         2006  +    if( nSep>1 ){
         2007  +      fprintf(stderr, "Error: multi-character separators not allowed"
         2008  +                      " for import\n");
         2009  +      return 1;
         2010  +    }
         2011  +    sCsv.zFile = zFile;
         2012  +    sCsv.nLine = 1;
         2013  +    if( sCsv.zFile[0]=='|' ){
         2014  +      sCsv.in = popen(sCsv.zFile+1, "r");
         2015  +      sCsv.zFile = "<pipe>";
         2016  +      xCloser = pclose;
         2017  +    }else{
         2018  +      sCsv.in = fopen(sCsv.zFile, "rb");
         2019  +      xCloser = fclose;
         2020  +    }
         2021  +    if( sCsv.in==0 ){
         2022  +      fprintf(stderr, "Error: cannot open \"%s\"\n", zFile);
         2023  +      return 1;
         2024  +    }
         2025  +    sCsv.cSeparator = p->separator[0];
  1908   2026       zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
  1909   2027       if( zSql==0 ){
  1910   2028         fprintf(stderr, "Error: out of memory\n");
         2029  +      xCloser(sCsv.in);
  1911   2030         return 1;
  1912   2031       }
  1913   2032       nByte = strlen30(zSql);
  1914   2033       rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
         2034  +    if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(db))==0 ){
         2035  +      char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable);
         2036  +      char cSep = '(';
         2037  +      while( csv_read_one_field(&sCsv) ){
         2038  +        zCreate = sqlite3_mprintf("%z%c\n  \"%s\" TEXT", zCreate, cSep, sCsv.z);
         2039  +        cSep = ',';
         2040  +        if( sCsv.cTerm!=sCsv.cSeparator ) break;
         2041  +      }
         2042  +      if( cSep=='(' ){
         2043  +        sqlite3_free(zCreate);
         2044  +        sqlite3_free(sCsv.z);
         2045  +        xCloser(sCsv.in);
         2046  +        fprintf(stderr,"%s: empty file\n", sCsv.zFile);
         2047  +        return 1;
         2048  +      }
         2049  +      zCreate = sqlite3_mprintf("%z\n)", zCreate);
         2050  +      rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
         2051  +      sqlite3_free(zCreate);
         2052  +      if( rc ){
         2053  +        fprintf(stderr, "CREATE TABLE %s(...) failed: %s\n", zTable,
         2054  +                sqlite3_errmsg(db));
         2055  +        sqlite3_free(sCsv.z);
         2056  +        xCloser(sCsv.in);
         2057  +        return 1;
         2058  +      }
         2059  +      rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
         2060  +    }
  1915   2061       sqlite3_free(zSql);
  1916   2062       if( rc ){
  1917   2063         if (pStmt) sqlite3_finalize(pStmt);
  1918   2064         fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db));
         2065  +      xCloser(sCsv.in);
  1919   2066         return 1;
  1920   2067       }
  1921   2068       nCol = sqlite3_column_count(pStmt);
  1922   2069       sqlite3_finalize(pStmt);
  1923   2070       pStmt = 0;
  1924   2071       if( nCol==0 ) return 0; /* no columns, no error */
  1925         -    zSql = malloc( nByte + 20 + nCol*2 );
         2072  +    zSql = sqlite3_malloc( nByte*2 + 20 + nCol*2 );
  1926   2073       if( zSql==0 ){
  1927   2074         fprintf(stderr, "Error: out of memory\n");
         2075  +      xCloser(sCsv.in);
  1928   2076         return 1;
  1929   2077       }
  1930         -    sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zTable);
         2078  +    sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
  1931   2079       j = strlen30(zSql);
  1932   2080       for(i=1; i<nCol; i++){
  1933   2081         zSql[j++] = ',';
  1934   2082         zSql[j++] = '?';
  1935   2083       }
  1936   2084       zSql[j++] = ')';
  1937   2085       zSql[j] = 0;
  1938   2086       rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
  1939         -    free(zSql);
         2087  +    sqlite3_free(zSql);
  1940   2088       if( rc ){
  1941   2089         fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db));
  1942   2090         if (pStmt) sqlite3_finalize(pStmt);
  1943         -      return 1;
  1944         -    }
  1945         -    in = fopen(zFile, "rb");
  1946         -    if( in==0 ){
  1947         -      fprintf(stderr, "Error: cannot open \"%s\"\n", zFile);
  1948         -      sqlite3_finalize(pStmt);
         2091  +      xCloser(sCsv.in);
  1949   2092         return 1;
  1950   2093       }
  1951         -    azCol = malloc( sizeof(azCol[0])*(nCol+1) );
  1952         -    if( azCol==0 ){
  1953         -      fprintf(stderr, "Error: out of memory\n");
  1954         -      fclose(in);
  1955         -      sqlite3_finalize(pStmt);
  1956         -      return 1;
  1957         -    }
  1958         -    sqlite3_exec(p->db, "BEGIN", 0, 0, 0);
  1959         -    zCommit = "COMMIT";
  1960         -    while( (zLine = local_getline(0, in, 1))!=0 ){
  1961         -      char *z, c;
  1962         -      int inQuote = 0;
  1963         -      lineno++;
  1964         -      azCol[0] = zLine;
  1965         -      for(i=0, z=zLine; (c = *z)!=0; z++){
  1966         -        if( c=='"' ) inQuote = !inQuote;
  1967         -        if( c=='\n' ) lineno++;
  1968         -        if( !inQuote && c==p->separator[0] && strncmp(z,p->separator,nSep)==0 ){
  1969         -          *z = 0;
         2094  +    do{
         2095  +      int startLine = sCsv.nLine;
         2096  +      for(i=0; i<nCol; i++){
         2097  +        char *z = csv_read_one_field(&sCsv);
         2098  +        if( z==0 && i==0 ) break;
         2099  +        sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT);
         2100  +        if( i<nCol-1 && sCsv.cTerm!=sCsv.cSeparator ){
         2101  +          fprintf(stderr, "%s:%d: expected %d columns but found %d - "
         2102  +                          "filling the rest with NULL\n",
         2103  +                          sCsv.zFile, startLine, nCol, i+1);
         2104  +          i++;
         2105  +          while( i<nCol ){ sqlite3_bind_null(pStmt, i); i++; }
         2106  +        }
         2107  +      }
         2108  +      if( sCsv.cTerm==sCsv.cSeparator ){
         2109  +        do{
         2110  +          csv_read_one_field(&sCsv);
  1970   2111             i++;
  1971         -          if( i<nCol ){
  1972         -            azCol[i] = &z[nSep];
  1973         -            z += nSep-1;
  1974         -          }
         2112  +        }while( sCsv.cTerm==sCsv.cSeparator );
         2113  +        fprintf(stderr, "%s:%d: expected %d columns but found %d - "
         2114  +                        "extras ignored\n",
         2115  +                        sCsv.zFile, startLine, nCol, i);
         2116  +      }
         2117  +      if( i>=nCol ){
         2118  +        sqlite3_step(pStmt);
         2119  +        rc = sqlite3_reset(pStmt);
         2120  +        if( rc!=SQLITE_OK ){
         2121  +          fprintf(stderr, "%s:%d: INSERT failed: %s\n", sCsv.zFile, startLine,
         2122  +                  sqlite3_errmsg(db));
  1975   2123           }
  1976         -      } /* end for */
  1977         -      *z = 0;
  1978         -      if( i+1!=nCol ){
  1979         -        fprintf(stderr,
  1980         -                "Error: %s line %d: expected %d columns of data but found %d\n",
  1981         -                zFile, lineno, nCol, i+1);
  1982         -        zCommit = "ROLLBACK";
  1983         -        free(zLine);
  1984         -        rc = 1;
  1985         -        break; /* from while */
  1986   2124         }
  1987         -      for(i=0; i<nCol; i++){
  1988         -        if( azCol[i][0]=='"' ){
  1989         -          int k;
  1990         -          for(z=azCol[i], j=1, k=0; z[j]; j++){
  1991         -            if( z[j]=='"' ){ j++; if( z[j]==0 ) break; }
  1992         -            z[k++] = z[j];
  1993         -          }
  1994         -          z[k] = 0;
  1995         -        }
  1996         -        sqlite3_bind_text(pStmt, i+1, azCol[i], -1, SQLITE_STATIC);
  1997         -      }
  1998         -      sqlite3_step(pStmt);
  1999         -      rc = sqlite3_reset(pStmt);
  2000         -      free(zLine);
  2001         -      if( rc!=SQLITE_OK ){
  2002         -        fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db));
  2003         -        zCommit = "ROLLBACK";
  2004         -        rc = 1;
  2005         -        break; /* from while */
  2006         -      }
  2007         -    } /* end while */
  2008         -    free(azCol);
  2009         -    fclose(in);
         2125  +    }while( sCsv.cTerm!=EOF );
         2126  +
         2127  +    xCloser(sCsv.in);
         2128  +    sqlite3_free(sCsv.z);
  2010   2129       sqlite3_finalize(pStmt);
  2011         -    sqlite3_exec(p->db, zCommit, 0, 0, 0);
         2130  +    sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
  2012   2131     }else
  2013   2132   
  2014   2133     if( c=='i' && strncmp(azArg[0], "indices", n)==0 && nArg<3 ){
  2015   2134       struct callback_data data;
  2016   2135       char *zErrMsg = 0;
  2017   2136       open_db(p);
  2018   2137       memcpy(&data, p, sizeof(data));
................................................................................
  2326   2445         fprintf(stderr,"Error: querying schema information\n");
  2327   2446         rc = 1;
  2328   2447       }else{
  2329   2448         rc = 0;
  2330   2449       }
  2331   2450     }else
  2332   2451   
         2452  +#ifdef SQLITE_DEBUG
  2333   2453     /* Undocumented commands for internal testing.  Subject to change
  2334   2454     ** without notice. */
  2335   2455     if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){
  2336   2456       if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){
  2337   2457         int i, v;
  2338   2458         for(i=1; i<nArg; i++){
  2339   2459           v = booleanValue(azArg[i]);
  2340   2460           fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v);
  2341   2461         }
  2342   2462       }
  2343   2463       if( strncmp(azArg[0]+9, "integer", n-9)==0 ){
  2344   2464         int i; sqlite3_int64 v;
  2345   2465         for(i=1; i<nArg; i++){
         2466  +        char zBuf[200];
  2346   2467           v = integerValue(azArg[i]);
  2347         -        fprintf(p->out, "%s: %lld 0x%llx\n", azArg[i], v, v);
         2468  +        sqlite3_snprintf(sizeof(zBuf), zBuf, "%s: %lld 0x%llx\n", azArg[i], v, v);
         2469  +        fprintf(p->out, "%s", zBuf);
  2348   2470         }
  2349   2471       }
  2350   2472     }else
         2473  +#endif
  2351   2474   
  2352   2475     if( c=='s' && strncmp(azArg[0], "separator", n)==0 && nArg==2 ){
  2353   2476       sqlite3_snprintf(sizeof(p->separator), p->separator,
  2354   2477                        "%.*s", (int)sizeof(p->separator)-1, azArg[1]);
  2355   2478     }else
  2356   2479   
  2357   2480     if( c=='s' && strncmp(azArg[0], "show", n)==0 && nArg==1 ){
................................................................................
  2532   2655               fprintf(stderr,"Error: testctrl %s takes no options\n", azArg[1]);
  2533   2656             }
  2534   2657             break;
  2535   2658   
  2536   2659           /* sqlite3_test_control(int, uint) */
  2537   2660           case SQLITE_TESTCTRL_PENDING_BYTE:        
  2538   2661             if( nArg==3 ){
  2539         -            unsigned int opt = (unsigned int)integerValue(azArg[2]);        
         2662  +            unsigned int opt = (unsigned int)integerValue(azArg[2]);
  2540   2663               rc = sqlite3_test_control(testctrl, opt);
  2541   2664               fprintf(p->out, "%d (0x%08x)\n", rc, rc);
  2542   2665             } else {
  2543   2666               fprintf(stderr,"Error: testctrl %s takes a single unsigned"
  2544   2667                              " int option\n", azArg[1]);
  2545   2668             }
  2546   2669             break;

Changes to src/sqlite.h.in.

   471    471   #define SQLITE_IOERR_SHMLOCK           (SQLITE_IOERR | (20<<8))
   472    472   #define SQLITE_IOERR_SHMMAP            (SQLITE_IOERR | (21<<8))
   473    473   #define SQLITE_IOERR_SEEK              (SQLITE_IOERR | (22<<8))
   474    474   #define SQLITE_IOERR_DELETE_NOENT      (SQLITE_IOERR | (23<<8))
   475    475   #define SQLITE_IOERR_MMAP              (SQLITE_IOERR | (24<<8))
   476    476   #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
   477    477   #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
          478  +#define SQLITE_BUSY_SNAPSHOT           (SQLITE_BUSY   |  (2<<8))
   478    479   #define SQLITE_CANTOPEN_NOTEMPDIR      (SQLITE_CANTOPEN | (1<<8))
   479    480   #define SQLITE_CANTOPEN_ISDIR          (SQLITE_CANTOPEN | (2<<8))
   480    481   #define SQLITE_CANTOPEN_FULLPATH       (SQLITE_CANTOPEN | (3<<8))
   481    482   #define SQLITE_CORRUPT_VTAB            (SQLITE_CORRUPT | (1<<8))
   482    483   #define SQLITE_READONLY_RECOVERY       (SQLITE_READONLY | (1<<8))
   483    484   #define SQLITE_READONLY_CANTLOCK       (SQLITE_READONLY | (2<<8))
   484    485   #define SQLITE_READONLY_ROLLBACK       (SQLITE_READONLY | (3<<8))
................................................................................
   490    491   #define SQLITE_CONSTRAINT_NOTNULL      (SQLITE_CONSTRAINT | (5<<8))
   491    492   #define SQLITE_CONSTRAINT_PRIMARYKEY   (SQLITE_CONSTRAINT | (6<<8))
   492    493   #define SQLITE_CONSTRAINT_TRIGGER      (SQLITE_CONSTRAINT | (7<<8))
   493    494   #define SQLITE_CONSTRAINT_UNIQUE       (SQLITE_CONSTRAINT | (8<<8))
   494    495   #define SQLITE_CONSTRAINT_VTAB         (SQLITE_CONSTRAINT | (9<<8))
   495    496   #define SQLITE_NOTICE_RECOVER_WAL      (SQLITE_NOTICE | (1<<8))
   496    497   #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8))
          498  +#define SQLITE_WARNING_AUTOINDEX       (SQLITE_WARNING | (1<<8))
   497    499   
   498    500   /*
   499    501   ** CAPI3REF: Flags For File Open Operations
   500    502   **
   501    503   ** These bit values are intended for use in the
   502    504   ** 3rd parameter to the [sqlite3_open_v2()] interface and
   503    505   ** in the 4th parameter to the [sqlite3_vfs.xOpen] method.
................................................................................
  2548   2550   ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback
  2549   2551   ** function X to be invoked periodically during long running calls to
  2550   2552   ** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for
  2551   2553   ** database connection D.  An example use for this
  2552   2554   ** interface is to keep a GUI updated during a large query.
  2553   2555   **
  2554   2556   ** ^The parameter P is passed through as the only parameter to the 
  2555         -** callback function X.  ^The parameter N is the number of 
         2557  +** callback function X.  ^The parameter N is the approximate number of 
  2556   2558   ** [virtual machine instructions] that are evaluated between successive
  2557   2559   ** invocations of the callback X.
  2558   2560   **
  2559   2561   ** ^Only a single progress handler may be defined at one time per
  2560   2562   ** [database connection]; setting a new progress handler cancels the
  2561   2563   ** old one.  ^Setting parameter X to NULL disables the progress handler.
  2562   2564   ** ^The progress handler is also disabled by setting N to a value less
................................................................................
  4170   4172   ** registered the application defined function.
  4171   4173   */
  4172   4174   sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
  4173   4175   
  4174   4176   /*
  4175   4177   ** CAPI3REF: Function Auxiliary Data
  4176   4178   **
  4177         -** The following two functions may be used by scalar SQL functions to
         4179  +** These functions may be used by (non-aggregate) SQL functions to
  4178   4180   ** associate metadata with argument values. If the same value is passed to
  4179   4181   ** multiple invocations of the same SQL function during query execution, under
  4180         -** some circumstances the associated metadata may be preserved. This may
  4181         -** be used, for example, to add a regular-expression matching scalar
  4182         -** function. The compiled version of the regular expression is stored as
  4183         -** metadata associated with the SQL value passed as the regular expression
  4184         -** pattern.  The compiled regular expression can be reused on multiple
  4185         -** invocations of the same function so that the original pattern string
  4186         -** does not need to be recompiled on each invocation.
         4182  +** some circumstances the associated metadata may be preserved.  An example
         4183  +** of where this might be useful is in a regular-expression matching
         4184  +** function. The compiled version of the regular expression can be stored as
         4185  +** metadata associated with the pattern string.  
         4186  +** Then as long as the pattern string remains the same,
         4187  +** the compiled regular expression can be reused on multiple
         4188  +** invocations of the same function.
  4187   4189   **
  4188   4190   ** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata
  4189   4191   ** associated by the sqlite3_set_auxdata() function with the Nth argument
  4190         -** value to the application-defined function. ^If no metadata has been ever
  4191         -** been set for the Nth argument of the function, or if the corresponding
  4192         -** function parameter has changed since the meta-data was set,
  4193         -** then sqlite3_get_auxdata() returns a NULL pointer.
  4194         -**
  4195         -** ^The sqlite3_set_auxdata() interface saves the metadata
  4196         -** pointed to by its 3rd parameter as the metadata for the N-th
  4197         -** argument of the application-defined function.  Subsequent
  4198         -** calls to sqlite3_get_auxdata() might return this data, if it has
  4199         -** not been destroyed.
  4200         -** ^If it is not NULL, SQLite will invoke the destructor
  4201         -** function given by the 4th parameter to sqlite3_set_auxdata() on
  4202         -** the metadata when the corresponding function parameter changes
  4203         -** or when the SQL statement completes, whichever comes first.
  4204         -**
  4205         -** SQLite is free to call the destructor and drop metadata on any
  4206         -** parameter of any function at any time.  ^The only guarantee is that
  4207         -** the destructor will be called before the metadata is dropped.
         4192  +** value to the application-defined function. ^If there is no metadata
         4193  +** associated with the function argument, this sqlite3_get_auxdata() interface
         4194  +** returns a NULL pointer.
         4195  +**
         4196  +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th
         4197  +** argument of the application-defined function.  ^Subsequent
         4198  +** calls to sqlite3_get_auxdata(C,N) return P from the most recent
         4199  +** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or
         4200  +** NULL if the metadata has been discarded.
         4201  +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL,
         4202  +** SQLite will invoke the destructor function X with parameter P exactly
         4203  +** once, when the metadata is discarded.
         4204  +** SQLite is free to discard the metadata at any time, including: <ul>
         4205  +** <li> when the corresponding function parameter changes, or
         4206  +** <li> when [sqlite3_reset()] or [sqlite3_finalize()] is called for the
         4207  +**      SQL statement, or
         4208  +** <li> when sqlite3_set_auxdata() is invoked again on the same parameter, or
         4209  +** <li> during the original sqlite3_set_auxdata() call when a memory 
         4210  +**      allocation error occurs. </ul>)^
         4211  +**
         4212  +** Note the last bullet in particular.  The destructor X in 
         4213  +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the
         4214  +** sqlite3_set_auxdata() interface even returns.  Hence sqlite3_set_auxdata()
         4215  +** should be called near the end of the function implementation and the
         4216  +** function implementation should not make any use of P after
         4217  +** sqlite3_set_auxdata() has been called.
  4208   4218   **
  4209   4219   ** ^(In practice, metadata is preserved between function calls for
  4210         -** expressions that are constant at compile time. This includes literal
  4211         -** values and [parameters].)^
         4220  +** function parameters that are compile-time constants, including literal
         4221  +** values and [parameters] and expressions composed from the same.)^
  4212   4222   **
  4213   4223   ** These routines must be called from the same thread in which
  4214   4224   ** the SQL function is running.
  4215   4225   */
  4216   4226   void *sqlite3_get_auxdata(sqlite3_context*, int N);
  4217   4227   void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*));
  4218   4228   
................................................................................
  5117   5127   ** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()],
  5118   5128   ** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail.
  5119   5129   **
  5120   5130   ** ^Calling sqlite3_auto_extension(X) with an entry point X that is already
  5121   5131   ** on the list of automatic extensions is a harmless no-op. ^No entry point
  5122   5132   ** will be called more than once for each database connection that is opened.
  5123   5133   **
  5124         -** See also: [sqlite3_reset_auto_extension()].
         5134  +** See also: [sqlite3_reset_auto_extension()]
         5135  +** and [sqlite3_cancel_auto_extension()]
  5125   5136   */
  5126   5137   int sqlite3_auto_extension(void (*xEntryPoint)(void));
  5127   5138   
         5139  +/*
         5140  +** CAPI3REF: Cancel Automatic Extension Loading
         5141  +**
         5142  +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the
         5143  +** initialization routine X that was registered using a prior call to
         5144  +** [sqlite3_auto_extension(X)].  ^The [sqlite3_cancel_auto_extension(X)]
         5145  +** routine returns 1 if initialization routine X was successfully 
         5146  +** unregistered and it returns 0 if X was not on the list of initialization
         5147  +** routines.
         5148  +*/
         5149  +int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void));
         5150  +
  5128   5151   /*
  5129   5152   ** CAPI3REF: Reset Automatic Extension Loading
  5130   5153   **
  5131   5154   ** ^This interface disables all automatic extensions previously
  5132   5155   ** registered using [sqlite3_auto_extension()].
  5133   5156   */
  5134   5157   void sqlite3_reset_auto_extension(void);
................................................................................
  6233   6256   ** wal file in wal mode databases, or the number of pages written to the
  6234   6257   ** database file in rollback mode databases. Any pages written as part of
  6235   6258   ** transaction rollback or database recovery operations are not included.
  6236   6259   ** If an IO or other error occurs while writing a page to disk, the effect
  6237   6260   ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The
  6238   6261   ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0.
  6239   6262   ** </dd>
         6263  +**
         6264  +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(<dt>SQLITE_DBSTATUS_DEFERRED_FKS</dt>
         6265  +** <dd>This parameter returns the zero for the current value if and only if
         6266  +** there all foreign key constraints (deferred or immediate) have been
         6267  +** resolved.  The highwater mark is always 0.
         6268  +** </dd>
  6240   6269   ** </dl>
  6241   6270   */
  6242   6271   #define SQLITE_DBSTATUS_LOOKASIDE_USED       0
  6243   6272   #define SQLITE_DBSTATUS_CACHE_USED           1
  6244   6273   #define SQLITE_DBSTATUS_SCHEMA_USED          2
  6245   6274   #define SQLITE_DBSTATUS_STMT_USED            3
  6246   6275   #define SQLITE_DBSTATUS_LOOKASIDE_HIT        4
  6247   6276   #define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE  5
  6248   6277   #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL  6
  6249   6278   #define SQLITE_DBSTATUS_CACHE_HIT            7
  6250   6279   #define SQLITE_DBSTATUS_CACHE_MISS           8
  6251   6280   #define SQLITE_DBSTATUS_CACHE_WRITE          9
  6252         -#define SQLITE_DBSTATUS_MAX                  9   /* Largest defined DBSTATUS */
         6281  +#define SQLITE_DBSTATUS_DEFERRED_FKS        10
         6282  +#define SQLITE_DBSTATUS_MAX                 10   /* Largest defined DBSTATUS */
  6253   6283   
  6254   6284   
  6255   6285   /*
  6256   6286   ** CAPI3REF: Prepared Statement Status
  6257   6287   **
  6258   6288   ** ^(Each prepared statement maintains various
  6259   6289   ** [SQLITE_STMTSTATUS counters] that measure the number
................................................................................
  6299   6329   **
  6300   6330   ** [[SQLITE_STMTSTATUS_AUTOINDEX]] <dt>SQLITE_STMTSTATUS_AUTOINDEX</dt>
  6301   6331   ** <dd>^This is the number of rows inserted into transient indices that
  6302   6332   ** were created automatically in order to help joins run faster.
  6303   6333   ** A non-zero value in this counter may indicate an opportunity to
  6304   6334   ** improvement performance by adding permanent indices that do not
  6305   6335   ** need to be reinitialized each time the statement is run.</dd>
         6336  +**
         6337  +** [[SQLITE_STMTSTATUS_VM_STEP]] <dt>SQLITE_STMTSTATUS_VM_STEP</dt>
         6338  +** <dd>^This is the number of virtual machine operations executed
         6339  +** by the prepared statement if that number is less than or equal
         6340  +** to 2147483647.  The number of virtual machine operations can be 
         6341  +** used as a proxy for the total work done by the prepared statement.
         6342  +** If the number of virtual machine operations exceeds 2147483647
         6343  +** then the value returned by this statement status code is undefined.
         6344  +** </dd>
  6306   6345   ** </dl>
  6307   6346   */
  6308   6347   #define SQLITE_STMTSTATUS_FULLSCAN_STEP     1
  6309   6348   #define SQLITE_STMTSTATUS_SORT              2
  6310   6349   #define SQLITE_STMTSTATUS_AUTOINDEX         3
         6350  +#define SQLITE_STMTSTATUS_VM_STEP           4
  6311   6351   
  6312   6352   /*
  6313   6353   ** CAPI3REF: Custom Page Cache Object
  6314   6354   **
  6315   6355   ** The sqlite3_pcache type is opaque.  It is implemented by
  6316   6356   ** the pluggable module.  The SQLite core has no knowledge of
  6317   6357   ** its size or internal structure and never deals with the

Changes to src/sqlite3ext.h.

   470    470   #endif /* SQLITE_CORE */
   471    471   
   472    472   #ifndef SQLITE_CORE
   473    473     /* This case when the file really is being compiled as a loadable 
   474    474     ** extension */
   475    475   # define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api=0;
   476    476   # define SQLITE_EXTENSION_INIT2(v)  sqlite3_api=v;
          477  +# define SQLITE_EXTENSION_INIT3     \
          478  +    extern const sqlite3_api_routines *sqlite3_api;
   477    479   #else
   478    480     /* This case when the file is being statically linked into the 
   479    481     ** application */
   480    482   # define SQLITE_EXTENSION_INIT1     /*no-op*/
   481    483   # define SQLITE_EXTENSION_INIT2(v)  (void)v; /* unused parameter */
          484  +# define SQLITE_EXTENSION_INIT3     /*no-op*/
   482    485   #endif
   483    486   
   484    487   #endif /* _SQLITE3EXT_H_ */

Changes to src/sqliteInt.h.

   391    391   ** GCC does not define the offsetof() macro so we'll have to do it
   392    392   ** ourselves.
   393    393   */
   394    394   #ifndef offsetof
   395    395   #define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD))
   396    396   #endif
   397    397   
          398  +/*
          399  +** Macros to compute minimum and maximum of two numbers.
          400  +*/
          401  +#define MIN(A,B) ((A)<(B)?(A):(B))
          402  +#define MAX(A,B) ((A)>(B)?(A):(B))
          403  +
   398    404   /*
   399    405   ** Check to see if this machine uses EBCDIC.  (Yes, believe it or
   400    406   ** not, there are still machines out there that use EBCDIC.)
   401    407   */
   402    408   #if 'A' == '\301'
   403    409   # define SQLITE_EBCDIC 1
   404    410   #else
................................................................................
   716    722   typedef struct Trigger Trigger;
   717    723   typedef struct TriggerPrg TriggerPrg;
   718    724   typedef struct TriggerStep TriggerStep;
   719    725   typedef struct UnpackedRecord UnpackedRecord;
   720    726   typedef struct VTable VTable;
   721    727   typedef struct VtabCtx VtabCtx;
   722    728   typedef struct Walker Walker;
   723         -typedef struct WherePlan WherePlan;
   724    729   typedef struct WhereInfo WhereInfo;
   725         -typedef struct WhereLevel WhereLevel;
   726    730   
   727    731   /*
   728    732   ** Defer sourcing vdbe.h and btree.h until after the "u8" and 
   729    733   ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque
   730    734   ** pointer types (i.e. FuncDef) defined above.
   731    735   */
   732    736   #include "btree.h"
................................................................................
   889    893     int aLimit[SQLITE_N_LIMIT];   /* Limits */
   890    894     struct sqlite3InitInfo {      /* Information used during initialization */
   891    895       int newTnum;                /* Rootpage of table being initialized */
   892    896       u8 iDb;                     /* Which db file is being initialized */
   893    897       u8 busy;                    /* TRUE if currently initializing */
   894    898       u8 orphanTrigger;           /* Last statement is orphaned TEMP trigger */
   895    899     } init;
   896         -  int activeVdbeCnt;            /* Number of VDBEs currently executing */
   897         -  int writeVdbeCnt;             /* Number of active VDBEs that are writing */
   898         -  int vdbeExecCnt;              /* Number of nested calls to VdbeExec() */
          900  +  int nVdbeActive;              /* Number of VDBEs currently running */
          901  +  int nVdbeRead;                /* Number of active VDBEs that read or write */
          902  +  int nVdbeWrite;               /* Number of active VDBEs that read and write */
          903  +  int nVdbeExec;                /* Number of nested calls to VdbeExec() */
   899    904     int nExtension;               /* Number of loaded extensions */
   900    905     void **aExtension;            /* Array of shared library handles */
   901    906     void (*xTrace)(void*,const char*);        /* Trace function */
   902    907     void *pTraceArg;                          /* Argument to the trace function */
   903    908     void (*xProfile)(void*,const char*,u64);  /* Profiling function */
   904    909     void *pProfileArg;                        /* Argument to profile function */
   905    910     void *pCommitArg;                 /* Argument to xCommitCallback() */   
................................................................................
   927    932     int (*xAuth)(void*,int,const char*,const char*,const char*,const char*);
   928    933                                   /* Access authorization function */
   929    934     void *pAuthArg;               /* 1st argument to the access auth function */
   930    935   #endif
   931    936   #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   932    937     int (*xProgress)(void *);     /* The progress callback */
   933    938     void *pProgressArg;           /* Argument to the progress callback */
   934         -  int nProgressOps;             /* Number of opcodes for progress callback */
          939  +  unsigned nProgressOps;        /* Number of opcodes for progress callback */
   935    940   #endif
   936    941   #ifndef SQLITE_OMIT_VIRTUALTABLE
   937    942     int nVTrans;                  /* Allocated size of aVTrans */
   938    943     Hash aModule;                 /* populated by sqlite3_create_module() */
   939    944     VtabCtx *pVtabCtx;            /* Context for active vtab connect/create */
   940    945     VTable **aVTrans;             /* Virtual tables with open transactions */
   941    946     VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
................................................................................
   945    950     BusyHandler busyHandler;      /* Busy callback */
   946    951     Db aDbStatic[2];              /* Static space for the 2 default backends */
   947    952     Savepoint *pSavepoint;        /* List of active savepoints */
   948    953     int busyTimeout;              /* Busy handler timeout, in msec */
   949    954     int nSavepoint;               /* Number of non-transaction savepoints */
   950    955     int nStatement;               /* Number of nested statement-transactions  */
   951    956     i64 nDeferredCons;            /* Net deferred constraints this transaction. */
          957  +  i64 nDeferredImmCons;         /* Net deferred immediate constraints */
   952    958     int *pnBytesFreed;            /* If not NULL, increment this in DbFree() */
   953    959   
   954    960   #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
   955    961     /* The following variables are all protected by the STATIC_MASTER 
   956    962     ** mutex, not by sqlite3.mutex. They are used by code in notify.c. 
   957    963     **
   958    964     ** When X.pUnlockConnection==Y, that means that X is waiting for Y to
................................................................................
  1000   1006   #define SQLITE_ReverseOrder   0x00010000  /* Reverse unordered SELECTs */
  1001   1007   #define SQLITE_RecTriggers    0x00020000  /* Enable recursive triggers */
  1002   1008   #define SQLITE_ForeignKeys    0x00040000  /* Enforce foreign key constraints  */
  1003   1009   #define SQLITE_AutoIndex      0x00080000  /* Enable automatic indexes */
  1004   1010   #define SQLITE_PreferBuiltin  0x00100000  /* Preference to built-in funcs */
  1005   1011   #define SQLITE_LoadExtension  0x00200000  /* Enable load_extension */
  1006   1012   #define SQLITE_EnableTrigger  0x00400000  /* True to enable triggers */
         1013  +#define SQLITE_DeferFKs       0x00800000  /* Defer all FK constraints */
         1014  +#define SQLITE_QueryOnly      0x01000000  /* Disable database changes */
         1015  +
  1007   1016   
  1008   1017   /*
  1009   1018   ** Bits of the sqlite3.dbOptFlags field that are used by the
  1010   1019   ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to
  1011   1020   ** selectively disable various optimizations.
  1012   1021   */
  1013   1022   #define SQLITE_QueryFlattener 0x0001   /* Query flattening */
................................................................................
  1016   1025   #define SQLITE_FactorOutConst 0x0008   /* Constant factoring */
  1017   1026   #define SQLITE_IdxRealAsInt   0x0010   /* Store REAL as INT in indices */
  1018   1027   #define SQLITE_DistinctOpt    0x0020   /* DISTINCT using indexes */
  1019   1028   #define SQLITE_CoverIdxScan   0x0040   /* Covering index scans */
  1020   1029   #define SQLITE_OrderByIdxJoin 0x0080   /* ORDER BY of joins via index */
  1021   1030   #define SQLITE_SubqCoroutine  0x0100   /* Evaluate subqueries as coroutines */
  1022   1031   #define SQLITE_Transitive     0x0200   /* Transitive constraints */
         1032  +#define SQLITE_OmitNoopJoin   0x0400   /* Omit unused tables in joins */
         1033  +#define SQLITE_Stat3          0x0800   /* Use the SQLITE_STAT3 table */
  1023   1034   #define SQLITE_AllOpts        0xffff   /* All optimizations */
  1024   1035   
  1025   1036   /*
  1026   1037   ** Macros for testing whether or not optimizations are enabled or disabled.
  1027   1038   */
  1028   1039   #ifndef SQLITE_OMIT_BUILTIN_TEST
  1029   1040   #define OptimizationDisabled(db, mask)  (((db)->dbOptFlags&(mask))!=0)
................................................................................
  1144   1155   ** sqlite3.pSavepoint. The first element in the list is the most recently
  1145   1156   ** opened savepoint. Savepoints are added to the list by the vdbe
  1146   1157   ** OP_Savepoint instruction.
  1147   1158   */
  1148   1159   struct Savepoint {
  1149   1160     char *zName;                        /* Savepoint name (nul-terminated) */
  1150   1161     i64 nDeferredCons;                  /* Number of deferred fk violations */
         1162  +  i64 nDeferredImmCons;               /* Number of deferred imm fk. */
  1151   1163     Savepoint *pNext;                   /* Parent savepoint (if any) */
  1152   1164   };
  1153   1165   
  1154   1166   /*
  1155   1167   ** The following are used as the second parameter to sqlite3Savepoint(),
  1156   1168   ** and as the P1 argument to the OP_Savepoint instruction.
  1157   1169   */
................................................................................
  1541   1553     u8 *aSortOrder;          /* for each column: True==DESC, False==ASC */
  1542   1554     char **azColl;           /* Array of collation sequence names for index */
  1543   1555     int tnum;                /* DB Page containing root of this index */
  1544   1556     u16 nColumn;             /* Number of columns in table used by this index */
  1545   1557     u8 onError;              /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
  1546   1558     unsigned autoIndex:2;    /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */
  1547   1559     unsigned bUnordered:1;   /* Use this index for == or IN queries only */
         1560  +  unsigned uniqNotNull:1;  /* True if UNIQUE and NOT NULL for all columns */
  1548   1561   #ifdef SQLITE_ENABLE_STAT3
  1549   1562     int nSample;             /* Number of elements in aSample[] */
  1550   1563     tRowcnt avgEq;           /* Average nEq value for key values not in aSample */
  1551   1564     IndexSample *aSample;    /* Samples of the left-most key */
  1552   1565   #endif
  1553   1566   };
  1554   1567   
................................................................................
  1886   1899   typedef u64 Bitmask;
  1887   1900   
  1888   1901   /*
  1889   1902   ** The number of bits in a Bitmask.  "BMS" means "BitMask Size".
  1890   1903   */
  1891   1904   #define BMS  ((int)(sizeof(Bitmask)*8))
  1892   1905   
         1906  +/*
         1907  +** A bit in a Bitmask
         1908  +*/
         1909  +#define MASKBIT(n)   (((Bitmask)1)<<(n))
         1910  +
  1893   1911   /*
  1894   1912   ** The following structure describes the FROM clause of a SELECT statement.
  1895   1913   ** Each table or subquery in the FROM clause is a separate element of
  1896   1914   ** the SrcList.a[] array.
  1897   1915   **
  1898   1916   ** With the addition of multiple database support, the following structure
  1899   1917   ** can also be used to describe a particular table such as the table that
................................................................................
  1906   1924   ** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each
  1907   1925   ** jointype expresses the join between the table and the previous table.
  1908   1926   **
  1909   1927   ** In the colUsed field, the high-order bit (bit 63) is set if the table
  1910   1928   ** contains more than 63 columns and the 64-th or later column is used.
  1911   1929   */
  1912   1930   struct SrcList {
  1913         -  i16 nSrc;        /* Number of tables or subqueries in the FROM clause */
  1914         -  i16 nAlloc;      /* Number of entries allocated in a[] below */
         1931  +  u8 nSrc;        /* Number of tables or subqueries in the FROM clause */
         1932  +  u8 nAlloc;      /* Number of entries allocated in a[] below */
  1915   1933     struct SrcList_item {
  1916   1934       Schema *pSchema;  /* Schema to which this item is fixed */
  1917   1935       char *zDatabase;  /* Name of database holding this table */
  1918   1936       char *zName;      /* Name of the table */
  1919   1937       char *zAlias;     /* The "B" part of a "A AS B" phrase.  zName is the "A" */
  1920   1938       Table *pTab;      /* An SQL table corresponding to zName */
  1921   1939       Select *pSelect;  /* A SELECT statement used in place of a table name */
................................................................................
  1945   1963   #define JT_NATURAL   0x0004    /* True for a "natural" join */
  1946   1964   #define JT_LEFT      0x0008    /* Left outer join */
  1947   1965   #define JT_RIGHT     0x0010    /* Right outer join */
  1948   1966   #define JT_OUTER     0x0020    /* The "OUTER" keyword is present */
  1949   1967   #define JT_ERROR     0x0040    /* unknown or unsupported join type */
  1950   1968   
  1951   1969   
  1952         -/*
  1953         -** A WherePlan object holds information that describes a lookup
  1954         -** strategy.
  1955         -**
  1956         -** This object is intended to be opaque outside of the where.c module.
  1957         -** It is included here only so that that compiler will know how big it
  1958         -** is.  None of the fields in this object should be used outside of
  1959         -** the where.c module.
  1960         -**
  1961         -** Within the union, pIdx is only used when wsFlags&WHERE_INDEXED is true.
  1962         -** pTerm is only used when wsFlags&WHERE_MULTI_OR is true.  And pVtabIdx
  1963         -** is only used when wsFlags&WHERE_VIRTUALTABLE is true.  It is never the
  1964         -** case that more than one of these conditions is true.
  1965         -*/
  1966         -struct WherePlan {
  1967         -  u32 wsFlags;                   /* WHERE_* flags that describe the strategy */
  1968         -  u16 nEq;                       /* Number of == constraints */
  1969         -  u16 nOBSat;                    /* Number of ORDER BY terms satisfied */
  1970         -  double nRow;                   /* Estimated number of rows (for EQP) */
  1971         -  union {
  1972         -    Index *pIdx;                   /* Index when WHERE_INDEXED is true */
  1973         -    struct WhereTerm *pTerm;       /* WHERE clause term for OR-search */
  1974         -    sqlite3_index_info *pVtabIdx;  /* Virtual table index to use */
  1975         -  } u;
  1976         -};
  1977         -
  1978         -/*
  1979         -** For each nested loop in a WHERE clause implementation, the WhereInfo
  1980         -** structure contains a single instance of this structure.  This structure
  1981         -** is intended to be private to the where.c module and should not be
  1982         -** access or modified by other modules.
  1983         -**
  1984         -** The pIdxInfo field is used to help pick the best index on a
  1985         -** virtual table.  The pIdxInfo pointer contains indexing
  1986         -** information for the i-th table in the FROM clause before reordering.
  1987         -** All the pIdxInfo pointers are freed by whereInfoFree() in where.c.
  1988         -** All other information in the i-th WhereLevel object for the i-th table
  1989         -** after FROM clause ordering.
  1990         -*/
  1991         -struct WhereLevel {
  1992         -  WherePlan plan;       /* query plan for this element of the FROM clause */
  1993         -  int iLeftJoin;        /* Memory cell used to implement LEFT OUTER JOIN */
  1994         -  int iTabCur;          /* The VDBE cursor used to access the table */
  1995         -  int iIdxCur;          /* The VDBE cursor used to access pIdx */
  1996         -  int addrBrk;          /* Jump here to break out of the loop */
  1997         -  int addrNxt;          /* Jump here to start the next IN combination */
  1998         -  int addrCont;         /* Jump here to continue with the next loop cycle */
  1999         -  int addrFirst;        /* First instruction of interior of the loop */
  2000         -  u8 iFrom;             /* Which entry in the FROM clause */
  2001         -  u8 op, p5;            /* Opcode and P5 of the opcode that ends the loop */
  2002         -  int p1, p2;           /* Operands of the opcode used to ends the loop */
  2003         -  union {               /* Information that depends on plan.wsFlags */
  2004         -    struct {
  2005         -      int nIn;              /* Number of entries in aInLoop[] */
  2006         -      struct InLoop {
  2007         -        int iCur;              /* The VDBE cursor used by this IN operator */
  2008         -        int addrInTop;         /* Top of the IN loop */
  2009         -        u8 eEndLoopOp;         /* IN Loop terminator. OP_Next or OP_Prev */
  2010         -      } *aInLoop;           /* Information about each nested IN operator */
  2011         -    } in;                 /* Used when plan.wsFlags&WHERE_IN_ABLE */
  2012         -    Index *pCovidx;       /* Possible covering index for WHERE_MULTI_OR */
  2013         -  } u;
  2014         -  double rOptCost;      /* "Optimal" cost for this level */
  2015         -
  2016         -  /* The following field is really not part of the current level.  But
  2017         -  ** we need a place to cache virtual table index information for each
  2018         -  ** virtual table in the FROM clause and the WhereLevel structure is
  2019         -  ** a convenient place since there is one WhereLevel for each FROM clause
  2020         -  ** element.
  2021         -  */
  2022         -  sqlite3_index_info *pIdxInfo;  /* Index info for n-th source table */
  2023         -};
  2024         -
  2025   1970   /*
  2026   1971   ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin()
  2027   1972   ** and the WhereInfo.wctrlFlags member.
  2028   1973   */
  2029   1974   #define WHERE_ORDERBY_NORMAL   0x0000 /* No-op */
  2030   1975   #define WHERE_ORDERBY_MIN      0x0001 /* ORDER BY processing for min() func */
  2031   1976   #define WHERE_ORDERBY_MAX      0x0002 /* ORDER BY processing for max() func */
  2032   1977   #define WHERE_ONEPASS_DESIRED  0x0004 /* Want to do one-pass UPDATE/DELETE */
  2033   1978   #define WHERE_DUPLICATES_OK    0x0008 /* Ok to return a row more than once */
  2034   1979   #define WHERE_OMIT_OPEN_CLOSE  0x0010 /* Table cursors are already open */
  2035   1980   #define WHERE_FORCE_TABLE      0x0020 /* Do not use an index-only search */
  2036   1981   #define WHERE_ONETABLE_ONLY    0x0040 /* Only code the 1st table in pTabList */
  2037   1982   #define WHERE_AND_ONLY         0x0080 /* Don't use indices for OR terms */
         1983  +#define WHERE_GROUPBY          0x0100 /* pOrderBy is really a GROUP BY */
         1984  +#define WHERE_DISTINCTBY       0x0200 /* pOrderby is really a DISTINCT clause */
         1985  +#define WHERE_WANT_DISTINCT    0x0400 /* All output needs to be distinct */
  2038   1986   
  2039         -/*
  2040         -** The WHERE clause processing routine has two halves.  The
  2041         -** first part does the start of the WHERE loop and the second
  2042         -** half does the tail of the WHERE loop.  An instance of
  2043         -** this structure is returned by the first half and passed
  2044         -** into the second half to give some continuity.
         1987  +/* Allowed return values from sqlite3WhereIsDistinct()
  2045   1988   */
  2046         -struct WhereInfo {
  2047         -  Parse *pParse;            /* Parsing and code generating context */
  2048         -  SrcList *pTabList;        /* List of tables in the join */
  2049         -  u16 nOBSat;               /* Number of ORDER BY terms satisfied by indices */
  2050         -  u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
  2051         -  u8 okOnePass;             /* Ok to use one-pass algorithm for UPDATE/DELETE */
  2052         -  u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
  2053         -  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
  2054         -  int iTop;                 /* The very beginning of the WHERE loop */
  2055         -  int iContinue;            /* Jump here to continue with next record */
  2056         -  int iBreak;               /* Jump here to break out of the loop */
  2057         -  int nLevel;               /* Number of nested loop */
  2058         -  struct WhereClause *pWC;  /* Decomposition of the WHERE clause */
  2059         -  double savedNQueryLoop;   /* pParse->nQueryLoop outside the WHERE loop */
  2060         -  double nRowOut;           /* Estimated number of output rows */
  2061         -  WhereLevel a[1];          /* Information about each nest loop in WHERE */
  2062         -};
  2063         -
  2064         -/* Allowed values for WhereInfo.eDistinct and DistinctCtx.eTnctType */
  2065   1989   #define WHERE_DISTINCT_NOOP      0  /* DISTINCT keyword not used */
  2066   1990   #define WHERE_DISTINCT_UNIQUE    1  /* No duplicates */
  2067   1991   #define WHERE_DISTINCT_ORDERED   2  /* All duplicates are adjacent */
  2068   1992   #define WHERE_DISTINCT_UNORDERED 3  /* Duplicates are scattered */
  2069   1993   
  2070   1994   /*
  2071   1995   ** A NameContext defines a context in which to resolve table and column
................................................................................
  2131   2055   */
  2132   2056   struct Select {
  2133   2057     ExprList *pEList;      /* The fields of the result */
  2134   2058     u8 op;                 /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */
  2135   2059     u16 selFlags;          /* Various SF_* values */
  2136   2060     int iLimit, iOffset;   /* Memory registers holding LIMIT & OFFSET counters */
  2137   2061     int addrOpenEphm[3];   /* OP_OpenEphem opcodes related to this select */
  2138         -  double nSelectRow;     /* Estimated number of result rows */
         2062  +  u64 nSelectRow;        /* Estimated number of result rows */
  2139   2063     SrcList *pSrc;         /* The FROM clause */
  2140   2064     Expr *pWhere;          /* The WHERE clause */
  2141   2065     ExprList *pGroupBy;    /* The GROUP BY clause */
  2142   2066     Expr *pHaving;         /* The HAVING clause */
  2143   2067     ExprList *pOrderBy;    /* The ORDER BY clause */
  2144   2068     Select *pPrior;        /* Prior select in a compound select statement */
  2145   2069     Select *pNext;         /* Next select to the left in a compound */
................................................................................
  2315   2239     TableLock *aTableLock; /* Required table locks for shared-cache mode */
  2316   2240   #endif
  2317   2241     AutoincInfo *pAinc;  /* Information about AUTOINCREMENT counters */
  2318   2242   
  2319   2243     /* Information used while coding trigger programs. */
  2320   2244     Parse *pToplevel;    /* Parse structure for main program (or NULL) */
  2321   2245     Table *pTriggerTab;  /* Table triggers are being coded for */
  2322         -  double nQueryLoop;   /* Estimated number of iterations of a query */
         2246  +  u32 nQueryLoop;      /* Est number of iterations of a query (10*log2(N)) */
  2323   2247     u32 oldmask;         /* Mask of old.* columns referenced */
  2324   2248     u32 newmask;         /* Mask of new.* columns referenced */
  2325   2249     u8 eTriggerOp;       /* TK_UPDATE, TK_INSERT or TK_DELETE */
  2326   2250     u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
  2327   2251     u8 disableTriggers;  /* True to disable triggers */
  2328   2252   
  2329   2253     /* Above is constant between recursions.  Below is reset before and after
................................................................................
  2885   2809   #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
  2886   2810   Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*);
  2887   2811   #endif
  2888   2812   void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
  2889   2813   void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
  2890   2814   WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
  2891   2815   void sqlite3WhereEnd(WhereInfo*);
         2816  +u64 sqlite3WhereOutputRowCount(WhereInfo*);
         2817  +int sqlite3WhereIsDistinct(WhereInfo*);
         2818  +int sqlite3WhereIsOrdered(WhereInfo*);
         2819  +int sqlite3WhereContinueLabel(WhereInfo*);
         2820  +int sqlite3WhereBreakLabel(WhereInfo*);
         2821  +int sqlite3WhereOkOnePass(WhereInfo*);
  2892   2822   int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8);
  2893   2823   void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int);
  2894   2824   void sqlite3ExprCodeMove(Parse*, int, int, int);
  2895   2825   void sqlite3ExprCacheStore(Parse*, int, int, int);
  2896   2826   void sqlite3ExprCachePush(Parse*);
  2897   2827   void sqlite3ExprCachePop(Parse*, int);
  2898   2828   void sqlite3ExprCacheRemove(Parse*, int, int);

Changes to src/status.c.

   238    238             sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet);
   239    239           }
   240    240         }
   241    241         *pHighwater = 0;
   242    242         *pCurrent = nRet;
   243    243         break;
   244    244       }
          245  +
          246  +    /* Set *pCurrent to non-zero if there are unresolved deferred foreign
          247  +    ** key constraints.  Set *pCurrent to zero if all foreign key constraints
          248  +    ** have been satisfied.  The *pHighwater is always set to zero.
          249  +    */
          250  +    case SQLITE_DBSTATUS_DEFERRED_FKS: {
          251  +      *pHighwater = 0;
          252  +      *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0;
          253  +      break;
          254  +    }
   245    255   
   246    256       default: {
   247    257         rc = SQLITE_ERROR;
   248    258       }
   249    259     }
   250    260     sqlite3_mutex_leave(db->mutex);
   251    261     return rc;
   252    262   }

Changes to src/test1.c.

  2214   2214     static const struct {
  2215   2215       const char *zName;
  2216   2216       int op;
  2217   2217     } aOp[] = {
  2218   2218       { "SQLITE_STMTSTATUS_FULLSCAN_STEP",   SQLITE_STMTSTATUS_FULLSCAN_STEP   },
  2219   2219       { "SQLITE_STMTSTATUS_SORT",            SQLITE_STMTSTATUS_SORT            },
  2220   2220       { "SQLITE_STMTSTATUS_AUTOINDEX",       SQLITE_STMTSTATUS_AUTOINDEX       },
         2221  +    { "SQLITE_STMTSTATUS_VM_STEP",         SQLITE_STMTSTATUS_VM_STEP         },
  2221   2222     };
  2222   2223     if( objc!=4 ){
  2223   2224       Tcl_WrongNumArgs(interp, 1, objv, "STMT PARAMETER RESETFLAG");
  2224   2225       return TCL_ERROR;
  2225   2226     }
  2226   2227     if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
  2227   2228     zOpName = Tcl_GetString(objv[2]);
................................................................................
  5954   5955     const char *zOpt;
  5955   5956     int onoff;
  5956   5957     int mask = 0;
  5957   5958     static const struct {
  5958   5959       const char *zOptName;
  5959   5960       int mask;
  5960   5961     } aOpt[] = {
  5961         -    { "all",              SQLITE_AllOpts        },
  5962         -    { "query-flattener",  SQLITE_QueryFlattener },
  5963         -    { "column-cache",     SQLITE_ColumnCache    },
  5964         -    { "groupby-order",    SQLITE_GroupByOrder   },
  5965         -    { "factor-constants", SQLITE_FactorOutConst },
  5966         -    { "real-as-int",      SQLITE_IdxRealAsInt   },
  5967         -    { "distinct-opt",     SQLITE_DistinctOpt    },
  5968         -    { "cover-idx-scan",   SQLITE_CoverIdxScan   },
  5969         -    { "order-by-idx-join",SQLITE_OrderByIdxJoin },
         5962  +    { "all",                 SQLITE_AllOpts        },
         5963  +    { "none",                0                     },
         5964  +    { "query-flattener",     SQLITE_QueryFlattener },
         5965  +    { "column-cache",        SQLITE_ColumnCache    },
         5966  +    { "groupby-order",       SQLITE_GroupByOrder   },
         5967  +    { "factor-constants",    SQLITE_FactorOutConst },
         5968  +    { "real-as-int",         SQLITE_IdxRealAsInt   },
         5969  +    { "distinct-opt",        SQLITE_DistinctOpt    },
         5970  +    { "cover-idx-scan",      SQLITE_CoverIdxScan   },
         5971  +    { "order-by-idx-join",   SQLITE_OrderByIdxJoin },
         5972  +    { "transitive",          SQLITE_Transitive     },
         5973  +    { "subquery-coroutine",  SQLITE_SubqCoroutine  },
         5974  +    { "omit-noop-join",      SQLITE_OmitNoopJoin   },
         5975  +    { "stat3",               SQLITE_Stat3          },
  5970   5976     };
  5971   5977   
  5972   5978     if( objc!=4 ){
  5973   5979       Tcl_WrongNumArgs(interp, 1, objv, "DB OPT BOOLEAN");
  5974   5980       return TCL_ERROR;
  5975   5981     }
  5976   5982     if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
................................................................................
  5983   5989       }
  5984   5990     }
  5985   5991     if( onoff ) mask = ~mask;
  5986   5992     if( i>=sizeof(aOpt)/sizeof(aOpt[0]) ){
  5987   5993       Tcl_AppendResult(interp, "unknown optimization - should be one of:",
  5988   5994                        (char*)0);
  5989   5995       for(i=0; i<sizeof(aOpt)/sizeof(aOpt[0]); i++){
  5990         -      Tcl_AppendResult(interp, " ", aOpt[i].zOptName);
         5996  +      Tcl_AppendResult(interp, " ", aOpt[i].zOptName, (char*)0);
  5991   5997       }
  5992   5998       return TCL_ERROR;
  5993   5999     }
  5994   6000     sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, db, mask);
  5995   6001     return TCL_OK;
  5996   6002   }
  5997   6003   
................................................................................
  6298   6304   #endif
  6299   6305   #ifdef SQLITE_DEBUG
  6300   6306     extern int sqlite3WhereTrace;
  6301   6307     extern int sqlite3OSTrace;
  6302   6308     extern int sqlite3WalTrace;
  6303   6309   #endif
  6304   6310   #ifdef SQLITE_TEST
  6305         -  extern char sqlite3_query_plan[];
  6306         -  static char *query_plan = sqlite3_query_plan;
  6307   6311   #ifdef SQLITE_ENABLE_FTS3
  6308   6312     extern int sqlite3_fts3_enable_parentheses;
  6309   6313   #endif
  6310   6314   #endif
  6311   6315   
  6312   6316     for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
  6313   6317       Tcl_CreateCommand(interp, aCmd[i].zName, aCmd[i].xProc, 0, 0);
................................................................................
  6353   6357         (char*)&pzNeededCollation, TCL_LINK_STRING|TCL_LINK_READ_ONLY);
  6354   6358   #endif
  6355   6359   #if SQLITE_OS_WIN
  6356   6360     Tcl_LinkVar(interp, "sqlite_os_type",
  6357   6361         (char*)&sqlite3_os_type, TCL_LINK_INT);
  6358   6362   #endif
  6359   6363   #ifdef SQLITE_TEST
  6360         -  Tcl_LinkVar(interp, "sqlite_query_plan",
  6361         -      (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY);
         6364  +  {
         6365  +    static const char *query_plan = "*** OBSOLETE VARIABLE ***";
         6366  +    Tcl_LinkVar(interp, "sqlite_query_plan",
         6367  +       (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY);
         6368  +  }
  6362   6369   #endif
  6363   6370   #ifdef SQLITE_DEBUG
  6364   6371     Tcl_LinkVar(interp, "sqlite_where_trace",
  6365   6372         (char*)&sqlite3WhereTrace, TCL_LINK_INT);
  6366   6373     Tcl_LinkVar(interp, "sqlite_os_trace",
  6367   6374         (char*)&sqlite3OSTrace, TCL_LINK_INT);
  6368   6375   #ifndef SQLITE_OMIT_WAL

Changes to src/test_autoext.c.

    93     93     int objc,
    94     94     Tcl_Obj *CONST objv[]
    95     95   ){
    96     96     int rc = sqlite3_auto_extension((void*)sqr_init);
    97     97     Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
    98     98     return SQLITE_OK;
    99     99   }
          100  +
          101  +/*
          102  +** tclcmd:   sqlite3_cancel_auto_extension_sqr
          103  +**
          104  +** Unregister the "sqr" extension.
          105  +*/
          106  +static int cancelAutoExtSqrObjCmd(
          107  +  void * clientData,
          108  +  Tcl_Interp *interp,
          109  +  int objc,
          110  +  Tcl_Obj *CONST objv[]
          111  +){
          112  +  int rc = sqlite3_cancel_auto_extension((void*)sqr_init);
          113  +  Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
          114  +  return SQLITE_OK;
          115  +}
   100    116   
   101    117   /*
   102    118   ** tclcmd:   sqlite3_auto_extension_cube
   103    119   **
   104    120   ** Register the "cube" extension to be loaded automatically.
   105    121   */
   106    122   static int autoExtCubeObjCmd(
................................................................................
   109    125     int objc,
   110    126     Tcl_Obj *CONST objv[]
   111    127   ){
   112    128     int rc = sqlite3_auto_extension((void*)cube_init);
   113    129     Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   114    130     return SQLITE_OK;
   115    131   }
          132  +
          133  +/*
          134  +** tclcmd:   sqlite3_cancel_auto_extension_cube
          135  +**
          136  +** Unregister the "cube" extension.
          137  +*/
          138  +static int cancelAutoExtCubeObjCmd(
          139  +  void * clientData,
          140  +  Tcl_Interp *interp,
          141  +  int objc,
          142  +  Tcl_Obj *CONST objv[]
          143  +){
          144  +  int rc = sqlite3_cancel_auto_extension((void*)cube_init);
          145  +  Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
          146  +  return SQLITE_OK;
          147  +}
   116    148   
   117    149   /*
   118    150   ** tclcmd:   sqlite3_auto_extension_broken
   119    151   **
   120    152   ** Register the broken extension to be loaded automatically.
   121    153   */
   122    154   static int autoExtBrokenObjCmd(
................................................................................
   125    157     int objc,
   126    158     Tcl_Obj *CONST objv[]
   127    159   ){
   128    160     int rc = sqlite3_auto_extension((void*)broken_init);
   129    161     Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   130    162     return SQLITE_OK;
   131    163   }
          164  +
          165  +/*
          166  +** tclcmd:   sqlite3_cancel_auto_extension_broken
          167  +**
          168  +** Unregister the broken extension.
          169  +*/
          170  +static int cancelAutoExtBrokenObjCmd(
          171  +  void * clientData,
          172  +  Tcl_Interp *interp,
          173  +  int objc,
          174  +  Tcl_Obj *CONST objv[]
          175  +){
          176  +  int rc = sqlite3_cancel_auto_extension((void*)broken_init);
          177  +  Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
          178  +  return SQLITE_OK;
          179  +}
   132    180   
   133    181   #endif /* SQLITE_OMIT_LOAD_EXTENSION */
   134    182   
   135    183   
   136    184   /*
   137    185   ** tclcmd:   sqlite3_reset_auto_extension
   138    186   **
................................................................................
   157    205     Tcl_CreateObjCommand(interp, "sqlite3_auto_extension_sqr",
   158    206             autoExtSqrObjCmd, 0, 0);
   159    207     Tcl_CreateObjCommand(interp, "sqlite3_auto_extension_cube",
   160    208             autoExtCubeObjCmd, 0, 0);
   161    209     Tcl_CreateObjCommand(interp, "sqlite3_auto_extension_broken",
   162    210             autoExtBrokenObjCmd, 0, 0);
   163    211   #endif
          212  +  Tcl_CreateObjCommand(interp, "sqlite3_cancel_auto_extension_sqr",
          213  +          cancelAutoExtSqrObjCmd, 0, 0);
          214  +  Tcl_CreateObjCommand(interp, "sqlite3_cancel_auto_extension_cube",
          215  +          cancelAutoExtCubeObjCmd, 0, 0);
          216  +  Tcl_CreateObjCommand(interp, "sqlite3_cancel_auto_extension_broken",
          217  +          cancelAutoExtBrokenObjCmd, 0, 0);
   164    218     Tcl_CreateObjCommand(interp, "sqlite3_reset_auto_extension",
   165    219             resetAutoExtObjCmd, 0, 0);
   166    220     return TCL_OK;
   167    221   }

Changes to src/test_malloc.c.

  1345   1345       { "SCHEMA_USED",         SQLITE_DBSTATUS_SCHEMA_USED         },
  1346   1346       { "STMT_USED",           SQLITE_DBSTATUS_STMT_USED           },
  1347   1347       { "LOOKASIDE_HIT",       SQLITE_DBSTATUS_LOOKASIDE_HIT       },
  1348   1348       { "LOOKASIDE_MISS_SIZE", SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE },
  1349   1349       { "LOOKASIDE_MISS_FULL", SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL },
  1350   1350       { "CACHE_HIT",           SQLITE_DBSTATUS_CACHE_HIT           },
  1351   1351       { "CACHE_MISS",          SQLITE_DBSTATUS_CACHE_MISS          },
  1352         -    { "CACHE_WRITE",         SQLITE_DBSTATUS_CACHE_WRITE         }
         1352  +    { "CACHE_WRITE",         SQLITE_DBSTATUS_CACHE_WRITE         },
         1353  +    { "DEFERRED_FKS",        SQLITE_DBSTATUS_DEFERRED_FKS        }
  1353   1354     };
  1354   1355     Tcl_Obj *pResult;
  1355   1356     if( objc!=4 ){
  1356   1357       Tcl_WrongNumArgs(interp, 1, objv, "DB PARAMETER RESETFLAG");
  1357   1358       return TCL_ERROR;
  1358   1359     }
  1359   1360     if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;

Changes to src/test_schema.c.

   340    340   }
   341    341   
   342    342   #else
   343    343   
   344    344   /*
   345    345   ** Extension load function.
   346    346   */
   347         -int sqlite3_extension_init(
          347  +#ifdef _WIN32
          348  +__declspec(dllexport)
          349  +#endif
          350  +int sqlite3_schema_init(
   348    351     sqlite3 *db, 
   349    352     char **pzErrMsg, 
   350    353     const sqlite3_api_routines *pApi
   351    354   ){
   352    355     SQLITE_EXTENSION_INIT2(pApi);
   353    356   #ifndef SQLITE_OMIT_VIRTUALTABLE
   354    357     sqlite3_create_module(db, "schema", &schemaModule, 0);
   355    358   #endif
   356    359     return 0;
   357    360   }
   358    361   
   359    362   #endif

Changes to src/test_vfs.c.

   186    186   
   187    187   static int tvfsShmOpen(sqlite3_file*);
   188    188   static int tvfsShmLock(sqlite3_file*, int , int, int);
   189    189   static int tvfsShmMap(sqlite3_file*,int,int,int, void volatile **);
   190    190   static void tvfsShmBarrier(sqlite3_file*);
   191    191   static int tvfsShmUnmap(sqlite3_file*, int);
   192    192   
          193  +static int tvfsFetch(sqlite3_file*, sqlite3_int64, int, void**);
          194  +static int tvfsUnfetch(sqlite3_file*, sqlite3_int64, void*);
          195  +
   193    196   static sqlite3_io_methods tvfs_io_methods = {
   194         -  2,                              /* iVersion */
          197  +  3,                              /* iVersion */
   195    198     tvfsClose,                      /* xClose */
   196    199     tvfsRead,                       /* xRead */
   197    200     tvfsWrite,                      /* xWrite */
   198    201     tvfsTruncate,                   /* xTruncate */
   199    202     tvfsSync,                       /* xSync */
   200    203     tvfsFileSize,                   /* xFileSize */
   201    204     tvfsLock,                       /* xLock */
................................................................................
   203    206     tvfsCheckReservedLock,          /* xCheckReservedLock */
   204    207     tvfsFileControl,                /* xFileControl */
   205    208     tvfsSectorSize,                 /* xSectorSize */
   206    209     tvfsDeviceCharacteristics,      /* xDeviceCharacteristics */
   207    210     tvfsShmMap,                     /* xShmMap */
   208    211     tvfsShmLock,                    /* xShmLock */
   209    212     tvfsShmBarrier,                 /* xShmBarrier */
   210         -  tvfsShmUnmap                    /* xShmUnmap */
          213  +  tvfsShmUnmap,                   /* xShmUnmap */
          214  +  tvfsFetch,
          215  +  tvfsUnfetch
   211    216   };
   212    217   
   213    218   static int tvfsResultCode(Testvfs *p, int *pRc){
   214    219     struct errcode {
   215    220       int eCode;
   216    221       const char *zCode;
   217    222     } aCode[] = {
................................................................................
   614    619         nByte = sizeof(sqlite3_io_methods);
   615    620       }else{
   616    621         nByte = offsetof(sqlite3_io_methods, xShmMap);
   617    622       }
   618    623   
   619    624       pMethods = (sqlite3_io_methods *)ckalloc(nByte);
   620    625       memcpy(pMethods, &tvfs_io_methods, nByte);
   621         -    pMethods->iVersion = pVfs->iVersion;
          626  +    pMethods->iVersion = pFd->pReal->pMethods->iVersion;
          627  +    if( pMethods->iVersion>pVfs->iVersion ){
          628  +      pMethods->iVersion = pVfs->iVersion;
          629  +    }
   622    630       if( pVfs->iVersion>1 && ((Testvfs *)pVfs->pAppData)->isNoshm ){
   623    631         pMethods->xShmUnmap = 0;
   624    632         pMethods->xShmLock = 0;
   625    633         pMethods->xShmBarrier = 0;
   626    634         pMethods->xShmMap = 0;
   627    635       }
   628    636       pFile->pMethods = pMethods;
................................................................................
   988    996       }
   989    997       ckfree((char *)pBuffer);
   990    998     }
   991    999     pFd->pShm = 0;
   992   1000   
   993   1001     return rc;
   994   1002   }
         1003  +
         1004  +static int tvfsFetch(
         1005  +    sqlite3_file *pFile, 
         1006  +    sqlite3_int64 iOfst, 
         1007  +    int iAmt, 
         1008  +    void **pp
         1009  +){
         1010  +  TestvfsFd *pFd = tvfsGetFd(pFile);
         1011  +  return sqlite3OsFetch(pFd->pReal, iOfst, iAmt, pp);
         1012  +}
         1013  +
         1014  +static int tvfsUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *p){
         1015  +  TestvfsFd *pFd = tvfsGetFd(pFile);
         1016  +  return sqlite3OsUnfetch(pFd->pReal, iOfst, p);
         1017  +}
   995   1018   
   996   1019   static int testvfs_obj_cmd(
   997   1020     ClientData cd,
   998   1021     Tcl_Interp *interp,
   999   1022     int objc,
  1000   1023     Tcl_Obj *CONST objv[]
  1001   1024   ){
................................................................................
  1339   1362   static int testvfs_cmd(
  1340   1363     ClientData cd,
  1341   1364     Tcl_Interp *interp,
  1342   1365     int objc,
  1343   1366     Tcl_Obj *CONST objv[]
  1344   1367   ){
  1345   1368     static sqlite3_vfs tvfs_vfs = {
  1346         -    2,                            /* iVersion */
         1369  +    3,                            /* iVersion */
  1347   1370       0,                            /* szOsFile */
  1348   1371       0,                            /* mxPathname */
  1349   1372       0,                            /* pNext */
  1350   1373       0,                            /* zName */
  1351   1374       0,                            /* pAppData */
  1352   1375       tvfsOpen,                     /* xOpen */
  1353   1376       tvfsDelete,                   /* xDelete */
................................................................................
  1365   1388       0,                            /* xDlClose */
  1366   1389   #endif /* SQLITE_OMIT_LOAD_EXTENSION */
  1367   1390       tvfsRandomness,               /* xRandomness */
  1368   1391       tvfsSleep,                    /* xSleep */
  1369   1392       tvfsCurrentTime,              /* xCurrentTime */
  1370   1393       0,                            /* xGetLastError */
  1371   1394       0,                            /* xCurrentTimeInt64 */
         1395  +    0,                            /* xSetSystemCall */
         1396  +    0,                            /* xGetSystemCall */
         1397  +    0,                            /* xNextSystemCall */
  1372   1398     };
  1373   1399   
  1374   1400     Testvfs *p;                     /* New object */
  1375   1401     sqlite3_vfs *pVfs;              /* New VFS */
  1376   1402     char *zVfs;
  1377   1403     int nByte;                      /* Bytes of space to allocate at p */
  1378   1404   
  1379   1405     int i;
  1380   1406     int isNoshm = 0;                /* True if -noshm is passed */
  1381   1407     int isFullshm = 0;              /* True if -fullshm is passed */
  1382   1408     int isDefault = 0;              /* True if -default is passed */
  1383   1409     int szOsFile = 0;               /* Value passed to -szosfile */
  1384   1410     int mxPathname = -1;            /* Value passed to -mxpathname */
  1385         -  int iVersion = 2;               /* Value passed to -iversion */
         1411  +  int iVersion = 3;               /* Value passed to -iversion */
  1386   1412   
  1387   1413     if( objc<2 || 0!=(objc%2) ) goto bad_args;
  1388   1414     for(i=2; i<objc; i += 2){
  1389   1415       int nSwitch;
  1390   1416       char *zSwitch;
  1391   1417       zSwitch = Tcl_GetStringFromObj(objv[i], &nSwitch); 
  1392   1418   

Changes to src/tokenize.c.

   392    392     int lastTokenParsed = -1;       /* type of the previous token */
   393    393     u8 enableLookaside;             /* Saved value of db->lookaside.bEnabled */
   394    394     sqlite3 *db = pParse->db;       /* The database connection */
   395    395     int mxSqlLen;                   /* Max length of an SQL string */
   396    396   
   397    397   
   398    398     mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH];
   399         -  if( db->activeVdbeCnt==0 ){
          399  +  if( db->nVdbeActive==0 ){
   400    400       db->u1.isInterrupted = 0;
   401    401     }
   402    402     pParse->rc = SQLITE_OK;
   403    403     pParse->zTail = zSql;
   404    404     i = 0;
   405    405     assert( pzErrMsg!=0 );
   406    406     pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc);

Changes to src/update.c.

   314    314     /* Begin the database scan
   315    315     */
   316    316     sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid);
   317    317     pWInfo = sqlite3WhereBegin(
   318    318         pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, 0
   319    319     );
   320    320     if( pWInfo==0 ) goto update_cleanup;
   321         -  okOnePass = pWInfo->okOnePass;
          321  +  okOnePass = sqlite3WhereOkOnePass(pWInfo);
   322    322   
   323    323     /* Remember the rowid of every item to be updated.
   324    324     */
   325    325     sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regOldRowid);
   326    326     if( !okOnePass ){
   327    327       sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid);
   328    328     }

Changes to src/vacuum.c.

   107    107     int nRes;               /* Bytes of reserved space at the end of each page */
   108    108     int nDb;                /* Number of attached databases */
   109    109   
   110    110     if( !db->autoCommit ){
   111    111       sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
   112    112       return SQLITE_ERROR;
   113    113     }
   114         -  if( db->activeVdbeCnt>1 ){
          114  +  if( db->nVdbeActive>1 ){
   115    115       sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress");
   116    116       return SQLITE_ERROR;
   117    117     }
   118    118   
   119    119     /* Save the current value of the database flags so that it can be 
   120    120     ** restored before returning. Then set the writable-schema flag, and
   121    121     ** disable CHECK and foreign key constraints.  */

Changes to src/vdbe.c.

   548    548     int pc=0;                  /* The program counter */
   549    549     Op *aOp = p->aOp;          /* Copy of p->aOp */
   550    550     Op *pOp;                   /* Current operation */
   551    551     int rc = SQLITE_OK;        /* Value to return */
   552    552     sqlite3 *db = p->db;       /* The database */
   553    553     u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */
   554    554     u8 encoding = ENC(db);     /* The database encoding */
          555  +  int iCompare = 0;          /* Result of last OP_Compare operation */
          556  +  unsigned nVmStep = 0;      /* Number of virtual machine steps */
   555    557   #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   556         -  int checkProgress;         /* True if progress callbacks are enabled */
   557         -  int nProgressOps = 0;      /* Opcodes executed since progress callback. */
          558  +  unsigned nProgressOps = 0; /* nVmStep at last progress callback. */
   558    559   #endif
   559    560     Mem *aMem = p->aMem;       /* Copy of p->aMem */
   560    561     Mem *pIn1 = 0;             /* 1st input operand */
   561    562     Mem *pIn2 = 0;             /* 2nd input operand */
   562    563     Mem *pIn3 = 0;             /* 3rd input operand */
   563    564     Mem *pOut = 0;             /* Output operand */
   564         -  int iCompare = 0;          /* Result of last OP_Compare operation */
   565    565     int *aPermute = 0;         /* Permutation of columns for OP_Compare */
   566    566     i64 lastRowid = db->lastRowid;  /* Saved value of the last insert ROWID */
   567    567   #ifdef VDBE_PROFILE
   568    568     u64 start;                 /* CPU clock count at start of opcode */
   569    569     int origPc;                /* Program counter at start of opcode */
   570    570   #endif
   571    571     /*** INSERT STACK UNION HERE ***/
................................................................................
   574    574     sqlite3VdbeEnter(p);
   575    575     if( p->rc==SQLITE_NOMEM ){
   576    576       /* This happens if a malloc() inside a call to sqlite3_column_text() or
   577    577       ** sqlite3_column_text16() failed.  */
   578    578       goto no_mem;
   579    579     }
   580    580     assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY );
          581  +  assert( p->bIsReader || p->readOnly!=0 );
   581    582     p->rc = SQLITE_OK;
   582    583     assert( p->explain==0 );
   583    584     p->pResultSet = 0;
   584    585     db->busyHandler.nBusy = 0;
   585    586     CHECK_FOR_INTERRUPT;
   586    587     sqlite3VdbeIOTraceSql(p);
   587         -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   588         -  checkProgress = db->xProgress!=0;
   589         -#endif
   590    588   #ifdef SQLITE_DEBUG
   591    589     sqlite3BeginBenignMalloc();
   592    590     if( p->pc==0  && (p->db->flags & SQLITE_VdbeListing)!=0 ){
   593    591       int i;
   594    592       printf("VDBE Program Listing:\n");
   595    593       sqlite3VdbePrintSql(p);
   596    594       for(i=0; i<p->nOp; i++){
................................................................................
   602    600     for(pc=p->pc; rc==SQLITE_OK; pc++){
   603    601       assert( pc>=0 && pc<p->nOp );
   604    602       if( db->mallocFailed ) goto no_mem;
   605    603   #ifdef VDBE_PROFILE
   606    604       origPc = pc;
   607    605       start = sqlite3Hwtime();
   608    606   #endif
          607  +    nVmStep++;
   609    608       pOp = &aOp[pc];
   610    609   
   611    610       /* Only allow tracing if SQLITE_DEBUG is defined.
   612    611       */
   613    612   #ifdef SQLITE_DEBUG
   614    613       if( p->trace ){
   615    614         if( pc==0 ){
................................................................................
   627    626   #ifdef SQLITE_TEST
   628    627       if( sqlite3_interrupt_count>0 ){
   629    628         sqlite3_interrupt_count--;
   630    629         if( sqlite3_interrupt_count==0 ){
   631    630           sqlite3_interrupt(db);
   632    631         }
   633    632       }
   634         -#endif
   635         -
   636         -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   637         -    /* Call the progress callback if it is configured and the required number
   638         -    ** of VDBE ops have been executed (either since this invocation of
   639         -    ** sqlite3VdbeExec() or since last time the progress callback was called).
   640         -    ** If the progress callback returns non-zero, exit the virtual machine with
   641         -    ** a return code SQLITE_ABORT.
   642         -    */
   643         -    if( checkProgress ){
   644         -      if( db->nProgressOps==nProgressOps ){
   645         -        int prc;
   646         -        prc = db->xProgress(db->pProgressArg);
   647         -        if( prc!=0 ){
   648         -          rc = SQLITE_INTERRUPT;
   649         -          goto vdbe_error_halt;
   650         -        }
   651         -        nProgressOps = 0;
   652         -      }
   653         -      nProgressOps++;
   654         -    }
   655    633   #endif
   656    634   
   657    635       /* On any opcode with the "out2-prerelease" tag, free any
   658    636       ** external allocations out of mem[p2] and set mem[p2] to be
   659    637       ** an undefined integer.  Opcodes will either fill in the integer
   660    638       ** value or convert mem[p2] to a different type.
   661    639       */
................................................................................
   742    720   **
   743    721   ** An unconditional jump to address P2.
   744    722   ** The next instruction executed will be 
   745    723   ** the one at index P2 from the beginning of
   746    724   ** the program.
   747    725   */
   748    726   case OP_Goto: {             /* jump */
   749         -  CHECK_FOR_INTERRUPT;
   750    727     pc = pOp->p2 - 1;
          728  +
          729  +  /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev,
          730  +  ** OP_VNext, OP_RowSetNext, or OP_SorterNext) all jump here upon
          731  +  ** completion.  Check to see if sqlite3_interrupt() has been called
          732  +  ** or if the progress callback needs to be invoked. 
          733  +  **
          734  +  ** This code uses unstructured "goto" statements and does not look clean.
          735  +  ** But that is not due to sloppy coding habits. The code is written this
          736  +  ** way for performance, to avoid having to run the interrupt and progress
          737  +  ** checks on every opcode.  This helps sqlite3_step() to run about 1.5%
          738  +  ** faster according to "valgrind --tool=cachegrind" */
          739  +check_for_interrupt:
          740  +  CHECK_FOR_INTERRUPT;
          741  +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
          742  +  /* Call the progress callback if it is configured and the required number
          743  +  ** of VDBE ops have been executed (either since this invocation of
          744  +  ** sqlite3VdbeExec() or since last time the progress callback was called).
          745  +  ** If the progress callback returns non-zero, exit the virtual machine with
          746  +  ** a return code SQLITE_ABORT.
          747  +  */
          748  +  if( db->xProgress!=0 && (nVmStep - nProgressOps)>=db->nProgressOps ){
          749  +    int prc;
          750  +    prc = db->xProgress(db->pProgressArg);
          751  +    if( prc!=0 ){
          752  +      rc = SQLITE_INTERRUPT;
          753  +      goto vdbe_error_halt;
          754  +    }
          755  +    nProgressOps = nVmStep;
          756  +  }
          757  +#endif
          758  +  
   751    759     break;
   752    760   }
   753    761   
   754    762   /* Opcode:  Gosub P1 P2 * * *
   755    763   **
   756    764   ** Write the current address onto register P1
   757    765   ** and then jump to address P2.
................................................................................
   862    870     }
   863    871     rc = sqlite3VdbeHalt(p);
   864    872     assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR );
   865    873     if( rc==SQLITE_BUSY ){
   866    874       p->rc = rc = SQLITE_BUSY;
   867    875     }else{
   868    876       assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT );
   869         -    assert( rc==SQLITE_OK || db->nDeferredCons>0 );
          877  +    assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 );
   870    878       rc = p->rc ? SQLITE_ERROR : SQLITE_DONE;
   871    879     }
   872    880     goto vdbe_return;
   873    881   }
   874    882   
   875    883   /* Opcode: Integer P1 P2 * * *
   876    884   **
................................................................................
  1408   1416       assert( memIsValid(pArg) );
  1409   1417       apVal[i] = pArg;
  1410   1418       Deephemeralize(pArg);
  1411   1419       sqlite3VdbeMemStoreType(pArg);
  1412   1420       REGISTER_TRACE(pOp->p2+i, pArg);
  1413   1421     }
  1414   1422   
  1415         -  assert( pOp->p4type==P4_FUNCDEF || pOp->p4type==P4_VDBEFUNC );
  1416         -  if( pOp->p4type==P4_FUNCDEF ){
  1417         -    ctx.pFunc = pOp->p4.pFunc;
  1418         -    ctx.pVdbeFunc = 0;
  1419         -  }else{
  1420         -    ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc;
  1421         -    ctx.pFunc = ctx.pVdbeFunc->pFunc;
  1422         -  }
  1423         -
         1423  +  assert( pOp->p4type==P4_FUNCDEF );
         1424  +  ctx.pFunc = pOp->p4.pFunc;
  1424   1425     ctx.s.flags = MEM_Null;
  1425   1426     ctx.s.db = db;
  1426   1427     ctx.s.xDel = 0;
  1427   1428     ctx.s.zMalloc = 0;
         1429  +  ctx.iOp = pc;
         1430  +  ctx.pVdbe = p;
  1428   1431   
  1429   1432     /* The output cell may already have a buffer allocated. Move
  1430   1433     ** the pointer to ctx.s so in case the user-function can use
  1431   1434     ** the already allocated buffer instead of allocating a new one.
  1432   1435     */
  1433   1436     sqlite3VdbeMemMove(&ctx.s, pOut);
  1434   1437     MemSetTypeFlag(&ctx.s, MEM_Null);
................................................................................
  1443   1446     db->lastRowid = lastRowid;
  1444   1447     (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */
  1445   1448     lastRowid = db->lastRowid;
  1446   1449   
  1447   1450     /* If any auxiliary data functions have been called by this user function,
  1448   1451     ** immediately call the destructor for any non-static values.
  1449   1452     */
  1450         -  if( ctx.pVdbeFunc ){
  1451         -    sqlite3VdbeDeleteAuxData(ctx.pVdbeFunc, pOp->p1);
  1452         -    pOp->p4.pVdbeFunc = ctx.pVdbeFunc;
  1453         -    pOp->p4type = P4_VDBEFUNC;
  1454         -  }
         1453  +  sqlite3VdbeDeleteAuxData(p, pc, pOp->p1);
  1455   1454   
  1456   1455     if( db->mallocFailed ){
  1457   1456       /* Even though a malloc() has failed, the implementation of the
  1458   1457       ** user function may have called an sqlite3_result_XXX() function
  1459   1458       ** to return a value. The following call releases any resources
  1460   1459       ** associated with such a value.
  1461   1460       */
................................................................................
  2679   2678     /* Assert that the p1 parameter is valid. Also that if there is no open
  2680   2679     ** transaction, then there cannot be any savepoints. 
  2681   2680     */
  2682   2681     assert( db->pSavepoint==0 || db->autoCommit==0 );
  2683   2682     assert( p1==SAVEPOINT_BEGIN||p1==SAVEPOINT_RELEASE||p1==SAVEPOINT_ROLLBACK );
  2684   2683     assert( db->pSavepoint || db->isTransactionSavepoint==0 );
  2685   2684     assert( checkSavepointCount(db) );
         2685  +  assert( p->bIsReader );
  2686   2686   
  2687   2687     if( p1==SAVEPOINT_BEGIN ){
  2688         -    if( db->writeVdbeCnt>0 ){
         2688  +    if( db->nVdbeWrite>0 ){
  2689   2689         /* A new savepoint cannot be created if there are active write 
  2690   2690         ** statements (i.e. open read/write incremental blob handles).
  2691   2691         */
  2692   2692         sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - "
  2693   2693           "SQL statements in progress");
  2694   2694         rc = SQLITE_BUSY;
  2695   2695       }else{
................................................................................
  2721   2721             db->nSavepoint++;
  2722   2722           }
  2723   2723       
  2724   2724           /* Link the new savepoint into the database handle's list. */
  2725   2725           pNew->pNext = db->pSavepoint;
  2726   2726           db->pSavepoint = pNew;
  2727   2727           pNew->nDeferredCons = db->nDeferredCons;
         2728  +        pNew->nDeferredImmCons = db->nDeferredImmCons;
  2728   2729         }
  2729   2730       }
  2730   2731     }else{
  2731   2732       iSavepoint = 0;
  2732   2733   
  2733   2734       /* Find the named savepoint. If there is no such savepoint, then an
  2734   2735       ** an error is returned to the user.  */
................................................................................
  2738   2739         pSavepoint = pSavepoint->pNext
  2739   2740       ){
  2740   2741         iSavepoint++;
  2741   2742       }
  2742   2743       if( !pSavepoint ){
  2743   2744         sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName);
  2744   2745         rc = SQLITE_ERROR;
  2745         -    }else if( db->writeVdbeCnt>0 && p1==SAVEPOINT_RELEASE ){
         2746  +    }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){
  2746   2747         /* It is not possible to release (commit) a savepoint if there are 
  2747   2748         ** active write statements.
  2748   2749         */
  2749   2750         sqlite3SetString(&p->zErrMsg, db, 
  2750   2751           "cannot release savepoint - SQL statements in progress"
  2751   2752         );
  2752   2753         rc = SQLITE_BUSY;
................................................................................
  2808   2809           db->pSavepoint = pSavepoint->pNext;
  2809   2810           sqlite3DbFree(db, pSavepoint);
  2810   2811           if( !isTransaction ){
  2811   2812             db->nSavepoint--;
  2812   2813           }
  2813   2814         }else{
  2814   2815           db->nDeferredCons = pSavepoint->nDeferredCons;
         2816  +        db->nDeferredImmCons = pSavepoint->nDeferredImmCons;
  2815   2817         }
  2816   2818   
  2817   2819         if( !isTransaction ){
  2818   2820           rc = sqlite3VtabSavepoint(db, p1, iSavepoint);
  2819   2821           if( rc!=SQLITE_OK ) goto abort_due_to_error;
  2820   2822         }
  2821   2823       }
................................................................................
  2839   2841     int turnOnAC;
  2840   2842   
  2841   2843     desiredAutoCommit = pOp->p1;
  2842   2844     iRollback = pOp->p2;
  2843   2845     turnOnAC = desiredAutoCommit && !db->autoCommit;
  2844   2846     assert( desiredAutoCommit==1 || desiredAutoCommit==0 );
  2845   2847     assert( desiredAutoCommit==1 || iRollback==0 );
  2846         -  assert( db->activeVdbeCnt>0 );  /* At least this one VM is active */
         2848  +  assert( db->nVdbeActive>0 );  /* At least this one VM is active */
         2849  +  assert( p->bIsReader );
  2847   2850   
  2848   2851   #if 0
  2849         -  if( turnOnAC && iRollback && db->activeVdbeCnt>1 ){
         2852  +  if( turnOnAC && iRollback && db->nVdbeActive>1 ){
  2850   2853       /* If this instruction implements a ROLLBACK and other VMs are
  2851   2854       ** still running, and a transaction is active, return an error indicating
  2852   2855       ** that the other VMs must complete first. 
  2853   2856       */
  2854   2857       sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - "
  2855   2858           "SQL statements in progress");
  2856   2859       rc = SQLITE_BUSY;
  2857   2860     }else
  2858   2861   #endif
  2859         -  if( turnOnAC && !iRollback && db->writeVdbeCnt>0 ){
         2862  +  if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){
  2860   2863       /* If this instruction implements a COMMIT and other VMs are writing
  2861   2864       ** return an error indicating that the other VMs must complete first. 
  2862   2865       */
  2863   2866       sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - "
  2864   2867           "SQL statements in progress");
  2865   2868       rc = SQLITE_BUSY;
  2866   2869     }else if( desiredAutoCommit!=db->autoCommit ){
................................................................................
  2928   2931   ** will automatically commit when the VDBE halts.
  2929   2932   **
  2930   2933   ** If P2 is zero, then a read-lock is obtained on the database file.
  2931   2934   */
  2932   2935   case OP_Transaction: {
  2933   2936     Btree *pBt;
  2934   2937   
         2938  +  assert( p->bIsReader );
         2939  +  assert( p->readOnly==0 || pOp->p2==0 );
  2935   2940     assert( pOp->p1>=0 && pOp->p1<db->nDb );
  2936   2941     assert( (p->btreeMask & (((yDbMask)1)<<pOp->p1))!=0 );
         2942  +  if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){
         2943  +    rc = SQLITE_READONLY;
         2944  +    goto abort_due_to_error;
         2945  +  }
  2937   2946     pBt = db->aDb[pOp->p1].pBt;
  2938   2947   
  2939   2948     if( pBt ){
  2940   2949       rc = sqlite3BtreeBeginTrans(pBt, pOp->p2);
  2941   2950       if( rc==SQLITE_BUSY ){
  2942   2951         p->pc = pc;
  2943   2952         p->rc = rc = SQLITE_BUSY;
................................................................................
  2944   2953         goto vdbe_return;
  2945   2954       }
  2946   2955       if( rc!=SQLITE_OK ){
  2947   2956         goto abort_due_to_error;
  2948   2957       }
  2949   2958   
  2950   2959       if( pOp->p2 && p->usesStmtJournal 
  2951         -     && (db->autoCommit==0 || db->activeVdbeCnt>1) 
         2960  +     && (db->autoCommit==0 || db->nVdbeRead>1) 
  2952   2961       ){
  2953   2962         assert( sqlite3BtreeIsInTrans(pBt) );
  2954   2963         if( p->iStatement==0 ){
  2955   2964           assert( db->nStatement>=0 && db->nSavepoint>=0 );
  2956   2965           db->nStatement++; 
  2957   2966           p->iStatement = db->nSavepoint + db->nStatement;
  2958   2967         }
................................................................................
  2962   2971           rc = sqlite3BtreeBeginStmt(pBt, p->iStatement);
  2963   2972         }
  2964   2973   
  2965   2974         /* Store the current value of the database handles deferred constraint
  2966   2975         ** counter. If the statement transaction needs to be rolled back,
  2967   2976         ** the value of this counter needs to be restored too.  */
  2968   2977         p->nStmtDefCons = db->nDeferredCons;
         2978  +      p->nStmtDefImmCons = db->nDeferredImmCons;
  2969   2979       }
  2970   2980     }
  2971   2981     break;
  2972   2982   }
  2973   2983   
  2974   2984   /* Opcode: ReadCookie P1 P2 P3 * *
  2975   2985   **
................................................................................
  2984   2994   ** executing this instruction.
  2985   2995   */
  2986   2996   case OP_ReadCookie: {               /* out2-prerelease */
  2987   2997     int iMeta;
  2988   2998     int iDb;
  2989   2999     int iCookie;
  2990   3000   
         3001  +  assert( p->bIsReader );
  2991   3002     iDb = pOp->p1;
  2992   3003     iCookie = pOp->p3;
  2993   3004     assert( pOp->p3<SQLITE_N_BTREE_META );
  2994   3005     assert( iDb>=0 && iDb<db->nDb );
  2995   3006     assert( db->aDb[iDb].pBt!=0 );
  2996   3007     assert( (p->btreeMask & (((yDbMask)1)<<iDb))!=0 );
  2997   3008   
................................................................................
  3011   3022   ** A transaction must be started before executing this opcode.
  3012   3023   */
  3013   3024   case OP_SetCookie: {       /* in3 */
  3014   3025     Db *pDb;
  3015   3026     assert( pOp->p2<SQLITE_N_BTREE_META );
  3016   3027     assert( pOp->p1>=0 && pOp->p1<db->nDb );
  3017   3028     assert( (p->btreeMask & (((yDbMask)1)<<pOp->p1))!=0 );
         3029  +  assert( p->readOnly==0 );
  3018   3030     pDb = &db->aDb[pOp->p1];
  3019   3031     assert( pDb->pBt!=0 );
  3020   3032     assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) );
  3021   3033     pIn3 = &aMem[pOp->p3];
  3022   3034     sqlite3VdbeMemIntegerify(pIn3);
  3023   3035     /* See note about index shifting on OP_ReadCookie */
  3024   3036     rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i);
................................................................................
  3061   3073     int iMeta;
  3062   3074     int iGen;
  3063   3075     Btree *pBt;
  3064   3076   
  3065   3077     assert( pOp->p1>=0 && pOp->p1<db->nDb );
  3066   3078     assert( (p->btreeMask & (((yDbMask)1)<<pOp->p1))!=0 );
  3067   3079     assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) );
         3080  +  assert( p->bIsReader );
  3068   3081     pBt = db->aDb[pOp->p1].pBt;
  3069   3082     if( pBt ){
  3070   3083       sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta);
  3071   3084       iGen = db->aDb[pOp->p1].pSchema->iGeneration;
  3072   3085     }else{
  3073   3086       iGen = iMeta = 0;
  3074   3087     }
................................................................................
  3156   3169     int wrFlag;
  3157   3170     Btree *pX;
  3158   3171     VdbeCursor *pCur;
  3159   3172     Db *pDb;
  3160   3173   
  3161   3174     assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 );
  3162   3175     assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 );
         3176  +  assert( p->bIsReader );
         3177  +  assert( pOp->opcode==OP_OpenRead || p->readOnly==0 );
  3163   3178   
  3164   3179     if( p->expired ){
  3165   3180       rc = SQLITE_ABORT;
  3166   3181       break;
  3167   3182     }
  3168   3183   
  3169   3184     nField = 0;
................................................................................
  4486   4501   */
  4487   4502   case OP_SorterNext:    /* jump */
  4488   4503   case OP_Prev:          /* jump */
  4489   4504   case OP_Next: {        /* jump */
  4490   4505     VdbeCursor *pC;
  4491   4506     int res;
  4492   4507   
  4493         -  CHECK_FOR_INTERRUPT;
  4494   4508     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  4495   4509     assert( pOp->p5<=ArraySize(p->aCounter) );
  4496   4510     pC = p->apCsr[pOp->p1];
  4497   4511     if( pC==0 ){
  4498   4512       break;  /* See ticket #2273 */
  4499   4513     }
  4500   4514     assert( pC->isSorter==(pOp->opcode==OP_SorterNext) );
................................................................................
  4515   4529       pc = pOp->p2 - 1;
  4516   4530       if( pOp->p5 ) p->aCounter[pOp->p5-1]++;
  4517   4531   #ifdef SQLITE_TEST
  4518   4532       sqlite3_search_count++;
  4519   4533   #endif
  4520   4534     }
  4521   4535     pC->rowidIsValid = 0;
  4522         -  break;
         4536  +  goto check_for_interrupt;
  4523   4537   }
  4524   4538   
  4525   4539   /* Opcode: IdxInsert P1 P2 P3 * P5
  4526   4540   **
  4527   4541   ** Register P2 holds an SQL index key made using the
  4528   4542   ** MakeRecord instructions.  This opcode writes that key
  4529   4543   ** into the index P1.  Data for the entry is nil.
................................................................................
  4726   4740   */
  4727   4741   case OP_Destroy: {     /* out2-prerelease */
  4728   4742     int iMoved;
  4729   4743     int iCnt;
  4730   4744     Vdbe *pVdbe;
  4731   4745     int iDb;
  4732   4746   
         4747  +  assert( p->readOnly==0 );
  4733   4748   #ifndef SQLITE_OMIT_VIRTUALTABLE
  4734   4749     iCnt = 0;
  4735   4750     for(pVdbe=db->pVdbe; pVdbe; pVdbe = pVdbe->pNext){
  4736         -    if( pVdbe->magic==VDBE_MAGIC_RUN && pVdbe->inVtabMethod<2 && pVdbe->pc>=0 ){
         4751  +    if( pVdbe->magic==VDBE_MAGIC_RUN && pVdbe->bIsReader 
         4752  +     && pVdbe->inVtabMethod<2 && pVdbe->pc>=0 
         4753  +    ){
  4737   4754         iCnt++;
  4738   4755       }
  4739   4756     }
  4740   4757   #else
  4741         -  iCnt = db->activeVdbeCnt;
         4758  +  iCnt = db->nVdbeRead;
  4742   4759   #endif
  4743   4760     pOut->flags = MEM_Null;
  4744   4761     if( iCnt>1 ){
  4745   4762       rc = SQLITE_LOCKED;
  4746   4763       p->errorAction = OE_Abort;
  4747   4764     }else{
  4748   4765       iDb = pOp->p3;
................................................................................
  4781   4798   **
  4782   4799   ** See also: Destroy
  4783   4800   */
  4784   4801   case OP_Clear: {
  4785   4802     int nChange;
  4786   4803    
  4787   4804     nChange = 0;
         4805  +  assert( p->readOnly==0 );
  4788   4806     assert( (p->btreeMask & (((yDbMask)1)<<pOp->p2))!=0 );
  4789   4807     rc = sqlite3BtreeClearTable(
  4790   4808         db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &nChange : 0)
  4791   4809     );
  4792   4810     if( pOp->p3 ){
  4793   4811       p->nChange += nChange;
  4794   4812       if( pOp->p3>0 ){
................................................................................
  4827   4845     int pgno;
  4828   4846     int flags;
  4829   4847     Db *pDb;
  4830   4848   
  4831   4849     pgno = 0;
  4832   4850     assert( pOp->p1>=0 && pOp->p1<db->nDb );
  4833   4851     assert( (p->btreeMask & (((yDbMask)1)<<pOp->p1))!=0 );
         4852  +  assert( p->readOnly==0 );
  4834   4853     pDb = &db->aDb[pOp->p1];
  4835   4854     assert( pDb->pBt!=0 );
  4836   4855     if( pOp->opcode==OP_CreateTable ){
  4837   4856       /* flags = BTREE_INTKEY; */
  4838   4857       flags = BTREE_INTKEY;
  4839   4858     }else{
  4840   4859       flags = BTREE_BLOBKEY;
................................................................................
  4974   4993   case OP_IntegrityCk: {
  4975   4994     int nRoot;      /* Number of tables to check.  (Number of root pages.) */
  4976   4995     int *aRoot;     /* Array of rootpage numbers for tables to be checked */
  4977   4996     int j;          /* Loop counter */
  4978   4997     int nErr;       /* Number of errors reported */
  4979   4998     char *z;        /* Text of the error report */
  4980   4999     Mem *pnErr;     /* Register keeping track of errors remaining */
  4981         -  
         5000  +
         5001  +  assert( p->bIsReader );
  4982   5002     nRoot = pOp->p2;
  4983   5003     assert( nRoot>0 );
  4984   5004     aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) );
  4985   5005     if( aRoot==0 ) goto no_mem;
  4986   5006     assert( pOp->p3>0 && pOp->p3<=p->nMem );
  4987   5007     pnErr = &aMem[pOp->p3];
  4988   5008     assert( (pnErr->flags & MEM_Int)!=0 );
................................................................................
  5035   5055   **
  5036   5056   ** Extract the smallest value from boolean index P1 and put that value into
  5037   5057   ** register P3.  Or, if boolean index P1 is initially empty, leave P3
  5038   5058   ** unchanged and jump to instruction P2.
  5039   5059   */
  5040   5060   case OP_RowSetRead: {       /* jump, in1, out3 */
  5041   5061     i64 val;
  5042         -  CHECK_FOR_INTERRUPT;
         5062  +
  5043   5063     pIn1 = &aMem[pOp->p1];
  5044   5064     if( (pIn1->flags & MEM_RowSet)==0 
  5045   5065      || sqlite3RowSetNext(pIn1->u.pRowSet, &val)==0
  5046   5066     ){
  5047   5067       /* The boolean index is empty */
  5048   5068       sqlite3VdbeMemSetNull(pIn1);
  5049   5069       pc = pOp->p2 - 1;
  5050   5070     }else{
  5051   5071       /* A value was pulled from the index */
  5052   5072       sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val);
  5053   5073     }
  5054         -  break;
         5074  +  goto check_for_interrupt;
  5055   5075   }
  5056   5076   
  5057   5077   /* Opcode: RowSetTest P1 P2 P3 P4
  5058   5078   **
  5059   5079   ** Register P3 is assumed to hold a 64-bit integer value. If register P1
  5060   5080   ** contains a RowSet object and that RowSet object contains
  5061   5081   ** the value held in P3, jump to register P2. Otherwise, insert the
................................................................................
  5261   5281   **
  5262   5282   ** Increment a "constraint counter" by P2 (P2 may be negative or positive).
  5263   5283   ** If P1 is non-zero, the database constraint counter is incremented 
  5264   5284   ** (deferred foreign key constraints). Otherwise, if P1 is zero, the 
  5265   5285   ** statement counter is incremented (immediate foreign key constraints).
  5266   5286   */
  5267   5287   case OP_FkCounter: {
  5268         -  if( pOp->p1 ){
         5288  +  if( db->flags & SQLITE_DeferFKs ){
         5289  +    db->nDeferredImmCons += pOp->p2;
         5290  +  }else if( pOp->p1 ){
  5269   5291       db->nDeferredCons += pOp->p2;
  5270   5292     }else{
  5271   5293       p->nFkConstraint += pOp->p2;
  5272   5294     }
  5273   5295     break;
  5274   5296   }
  5275   5297   
................................................................................
  5282   5304   ** If P1 is non-zero, then the jump is taken if the database constraint-counter
  5283   5305   ** is zero (the one that counts deferred constraint violations). If P1 is
  5284   5306   ** zero, the jump is taken if the statement constraint-counter is zero
  5285   5307   ** (immediate foreign key constraint violations).
  5286   5308   */
  5287   5309   case OP_FkIfZero: {         /* jump */
  5288   5310     if( pOp->p1 ){
  5289         -    if( db->nDeferredCons==0 ) pc = pOp->p2-1;
         5311  +    if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1;
  5290   5312     }else{
  5291         -    if( p->nFkConstraint==0 ) pc = pOp->p2-1;
         5313  +    if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1;
  5292   5314     }
  5293   5315     break;
  5294   5316   }
  5295   5317   #endif /* #ifndef SQLITE_OMIT_FOREIGN_KEY */
  5296   5318   
  5297   5319   #ifndef SQLITE_OMIT_AUTOINCREMENT
  5298   5320   /* Opcode: MemMax P1 P2 * * *
................................................................................
  5480   5502   ** mem[P3+2] are initialized to -1.
  5481   5503   */
  5482   5504   case OP_Checkpoint: {
  5483   5505     int i;                          /* Loop counter */
  5484   5506     int aRes[3];                    /* Results */
  5485   5507     Mem *pMem;                      /* Write results here */
  5486   5508   
         5509  +  assert( p->readOnly==0 );
  5487   5510     aRes[0] = 0;
  5488   5511     aRes[1] = aRes[2] = -1;
  5489   5512     assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE
  5490   5513          || pOp->p2==SQLITE_CHECKPOINT_FULL
  5491   5514          || pOp->p2==SQLITE_CHECKPOINT_RESTART
  5492   5515     );
  5493   5516     rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]);
................................................................................
  5529   5552          || eNew==PAGER_JOURNALMODE_PERSIST 
  5530   5553          || eNew==PAGER_JOURNALMODE_OFF
  5531   5554          || eNew==PAGER_JOURNALMODE_MEMORY
  5532   5555          || eNew==PAGER_JOURNALMODE_WAL
  5533   5556          || eNew==PAGER_JOURNALMODE_QUERY
  5534   5557     );
  5535   5558     assert( pOp->p1>=0 && pOp->p1<db->nDb );
         5559  +  assert( p->readOnly==0 );
  5536   5560   
  5537   5561     pBt = db->aDb[pOp->p1].pBt;
  5538   5562     pPager = sqlite3BtreePager(pBt);
  5539   5563     eOld = sqlite3PagerGetJournalMode(pPager);
  5540   5564     if( eNew==PAGER_JOURNALMODE_QUERY ) eNew = eOld;
  5541   5565     if( !sqlite3PagerOkToChangeJournalMode(pPager) ) eNew = eOld;
  5542   5566   
................................................................................
  5552   5576     ){
  5553   5577       eNew = eOld;
  5554   5578     }
  5555   5579   
  5556   5580     if( (eNew!=eOld)
  5557   5581      && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL)
  5558   5582     ){
  5559         -    if( !db->autoCommit || db->activeVdbeCnt>1 ){
         5583  +    if( !db->autoCommit || db->nVdbeRead>1 ){
  5560   5584         rc = SQLITE_ERROR;
  5561   5585         sqlite3SetString(&p->zErrMsg, db, 
  5562   5586             "cannot change %s wal mode from within a transaction",
  5563   5587             (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of")
  5564   5588         );
  5565   5589         break;
  5566   5590       }else{
................................................................................
  5611   5635   /* Opcode: Vacuum * * * * *
  5612   5636   **
  5613   5637   ** Vacuum the entire database.  This opcode will cause other virtual
  5614   5638   ** machines to be created and run.  It may not be called from within
  5615   5639   ** a transaction.
  5616   5640   */
  5617   5641   case OP_Vacuum: {
         5642  +  assert( p->readOnly==0 );
  5618   5643     rc = sqlite3RunVacuum(&p->zErrMsg, db);
  5619   5644     break;
  5620   5645   }
  5621   5646   #endif
  5622   5647   
  5623   5648   #if !defined(SQLITE_OMIT_AUTOVACUUM)
  5624   5649   /* Opcode: IncrVacuum P1 P2 * * *
................................................................................
  5628   5653   ** P2. Otherwise, fall through to the next instruction.
  5629   5654   */
  5630   5655   case OP_IncrVacuum: {        /* jump */
  5631   5656     Btree *pBt;
  5632   5657   
  5633   5658     assert( pOp->p1>=0 && pOp->p1<db->nDb );
  5634   5659     assert( (p->btreeMask & (((yDbMask)1)<<pOp->p1))!=0 );
         5660  +  assert( p->readOnly==0 );
  5635   5661     pBt = db->aDb[pOp->p1].pBt;
  5636   5662     rc = sqlite3BtreeIncrVacuum(pBt);
  5637   5663     if( rc==SQLITE_DONE ){
  5638   5664       pc = pOp->p2 - 1;
  5639   5665       rc = SQLITE_OK;
  5640   5666     }
  5641   5667     break;
................................................................................
  5746   5772   */
  5747   5773   case OP_VOpen: {
  5748   5774     VdbeCursor *pCur;
  5749   5775     sqlite3_vtab_cursor *pVtabCursor;
  5750   5776     sqlite3_vtab *pVtab;
  5751   5777     sqlite3_module *pModule;
  5752   5778   
         5779  +  assert( p->bIsReader );
  5753   5780     pCur = 0;
  5754   5781     pVtabCursor = 0;
  5755   5782     pVtab = pOp->p4.pVtab->pVtab;
  5756   5783     pModule = (sqlite3_module *)pVtab->pModule;
  5757   5784     assert(pVtab && pModule);
  5758   5785     rc = pModule->xOpen(pVtab, &pVtabCursor);
  5759   5786     importVtabErrMsg(p, pVtab);
................................................................................
  5943   5970       res = pModule->xEof(pCur->pVtabCursor);
  5944   5971     }
  5945   5972   
  5946   5973     if( !res ){
  5947   5974       /* If there is data, jump to P2 */
  5948   5975       pc = pOp->p2 - 1;
  5949   5976     }
  5950         -  break;
         5977  +  goto check_for_interrupt;
  5951   5978   }
  5952   5979   #endif /* SQLITE_OMIT_VIRTUALTABLE */
  5953   5980   
  5954   5981   #ifndef SQLITE_OMIT_VIRTUALTABLE
  5955   5982   /* Opcode: VRename P1 * * P4 *
  5956   5983   **
  5957   5984   ** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
................................................................................
  5962   5989     sqlite3_vtab *pVtab;
  5963   5990     Mem *pName;
  5964   5991   
  5965   5992     pVtab = pOp->p4.pVtab->pVtab;
  5966   5993     pName = &aMem[pOp->p1];
  5967   5994     assert( pVtab->pModule->xRename );
  5968   5995     assert( memIsValid(pName) );
         5996  +  assert( p->readOnly==0 );
  5969   5997     REGISTER_TRACE(pOp->p1, pName);
  5970   5998     assert( pName->flags & MEM_Str );
  5971   5999     testcase( pName->enc==SQLITE_UTF8 );
  5972   6000     testcase( pName->enc==SQLITE_UTF16BE );
  5973   6001     testcase( pName->enc==SQLITE_UTF16LE );
  5974   6002     rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
  5975   6003     if( rc==SQLITE_OK ){
................................................................................
  6013   6041     sqlite_int64 rowid;
  6014   6042     Mem **apArg;
  6015   6043     Mem *pX;
  6016   6044   
  6017   6045     assert( pOp->p2==1        || pOp->p5==OE_Fail   || pOp->p5==OE_Rollback 
  6018   6046          || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace
  6019   6047     );
         6048  +  assert( p->readOnly==0 );
  6020   6049     pVtab = pOp->p4.pVtab->pVtab;
  6021   6050     pModule = (sqlite3_module *)pVtab->pModule;
  6022   6051     nArg = pOp->p2;
  6023   6052     assert( pOp->p4type==P4_VTAB );
  6024   6053     if( ALWAYS(pModule->xUpdate) ){
  6025   6054       u8 vtabOnConflict = db->vtabOnConflict;
  6026   6055       apArg = p->apArg;
................................................................................
  6196   6225     }
  6197   6226   
  6198   6227     /* This is the only way out of this procedure.  We have to
  6199   6228     ** release the mutexes on btrees that were acquired at the
  6200   6229     ** top. */
  6201   6230   vdbe_return:
  6202   6231     db->lastRowid = lastRowid;
         6232  +  p->aCounter[SQLITE_STMTSTATUS_VM_STEP-1] += (int)nVmStep;
  6203   6233     sqlite3VdbeLeave(p);
  6204   6234     return rc;
  6205   6235   
  6206   6236     /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH
  6207   6237     ** is encountered.
  6208   6238     */
  6209   6239   too_big:

Changes to src/vdbe.h.

    26     26   */
    27     27   typedef struct Vdbe Vdbe;
    28     28   
    29     29   /*
    30     30   ** The names of the following types declared in vdbeInt.h are required
    31     31   ** for the VdbeOp definition.
    32     32   */
    33         -typedef struct VdbeFunc VdbeFunc;
    34     33   typedef struct Mem Mem;
    35     34   typedef struct SubProgram SubProgram;
    36     35   
    37     36   /*
    38     37   ** A single instruction of the virtual machine has an opcode
    39     38   ** and as many as three operands.  The instruction is recorded
    40     39   ** as an instance of the following structure:
................................................................................
    50     49     union {             /* fourth parameter */
    51     50       int i;                 /* Integer value if p4type==P4_INT32 */
    52     51       void *p;               /* Generic pointer */
    53     52       char *z;               /* Pointer to data for string (char array) types */
    54     53       i64 *pI64;             /* Used when p4type is P4_INT64 */
    55     54       double *pReal;         /* Used when p4type is P4_REAL */
    56     55       FuncDef *pFunc;        /* Used when p4type is P4_FUNCDEF */
    57         -    VdbeFunc *pVdbeFunc;   /* Used when p4type is P4_VDBEFUNC */
    58     56       CollSeq *pColl;        /* Used when p4type is P4_COLLSEQ */
    59     57       Mem *pMem;             /* Used when p4type is P4_MEM */
    60     58       VTable *pVtab;         /* Used when p4type is P4_VTAB */
    61     59       KeyInfo *pKeyInfo;     /* Used when p4type is P4_KEYINFO */
    62     60       int *ai;               /* Used when p4type is P4_INTARRAY */
    63     61       SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
    64     62       int (*xAdvance)(BtCursor *, int *);
................................................................................
   104    102   */
   105    103   #define P4_NOTUSED    0   /* The P4 parameter is not used */
   106    104   #define P4_DYNAMIC  (-1)  /* Pointer to a string obtained from sqliteMalloc() */
   107    105   #define P4_STATIC   (-2)  /* Pointer to a static string */
   108    106   #define P4_COLLSEQ  (-4)  /* P4 is a pointer to a CollSeq structure */
   109    107   #define P4_FUNCDEF  (-5)  /* P4 is a pointer to a FuncDef structure */
   110    108   #define P4_KEYINFO  (-6)  /* P4 is a pointer to a KeyInfo structure */
   111         -#define P4_VDBEFUNC (-7)  /* P4 is a pointer to a VdbeFunc structure */
   112    109   #define P4_MEM      (-8)  /* P4 is a pointer to a Mem*    structure */
   113    110   #define P4_TRANSIENT  0   /* P4 is a pointer to a transient string */
   114    111   #define P4_VTAB     (-10) /* P4 is a pointer to an sqlite3_vtab structure */
   115    112   #define P4_MPRINTF  (-11) /* P4 is a string obtained from sqlite3_mprintf() */
   116    113   #define P4_REAL     (-12) /* P4 is a 64-bit floating point value */
   117    114   #define P4_INT64    (-13) /* P4 is a 64-bit signed integer */
   118    115   #define P4_INT32    (-14) /* P4 is a 32-bit signed integer */

Changes to src/vdbeInt.h.

    40     40   
    41     41   /* Opaque type used by code in vdbesort.c */
    42     42   typedef struct VdbeSorter VdbeSorter;
    43     43   
    44     44   /* Opaque type used by the explainer */
    45     45   typedef struct Explain Explain;
    46     46   
           47  +/* Elements of the linked list at Vdbe.pAuxData */
           48  +typedef struct AuxData AuxData;
           49  +
    47     50   /*
    48     51   ** A cursor is a pointer into a single BTree within a database file.
    49     52   ** The cursor can seek to a BTree entry with a particular key, or
    50     53   ** loop over all entries of the Btree.  You can also insert new BTree
    51     54   ** entries or retrieve the key or data from the entry that the cursor
    52     55   ** is currently pointing to.
    53     56   ** 
................................................................................
   226    229   ** Return true if a memory cell is not marked as invalid.  This macro
   227    230   ** is for use inside assert() statements only.
   228    231   */
   229    232   #ifdef SQLITE_DEBUG
   230    233   #define memIsValid(M)  ((M)->flags & MEM_Invalid)==0
   231    234   #endif
   232    235   
   233         -
   234         -/* A VdbeFunc is just a FuncDef (defined in sqliteInt.h) that contains
   235         -** additional information about auxiliary information bound to arguments
   236         -** of the function.  This is used to implement the sqlite3_get_auxdata()
   237         -** and sqlite3_set_auxdata() APIs.  The "auxdata" is some auxiliary data
   238         -** that can be associated with a constant argument to a function.  This
   239         -** allows functions such as "regexp" to compile their constant regular
   240         -** expression argument once and reused the compiled code for multiple
   241         -** invocations.
          236  +/*
          237  +** Each auxilliary data pointer stored by a user defined function 
          238  +** implementation calling sqlite3_set_auxdata() is stored in an instance
          239  +** of this structure. All such structures associated with a single VM
          240  +** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed
          241  +** when the VM is halted (if not before).
   242    242   */
   243         -struct VdbeFunc {
   244         -  FuncDef *pFunc;               /* The definition of the function */
   245         -  int nAux;                     /* Number of entries allocated for apAux[] */
   246         -  struct AuxData {
   247         -    void *pAux;                   /* Aux data for the i-th argument */
   248         -    void (*xDelete)(void *);      /* Destructor for the aux data */
   249         -  } apAux[1];                   /* One slot for each function argument */
          243  +struct AuxData {
          244  +  int iOp;                        /* Instruction number of OP_Function opcode */
          245  +  int iArg;                       /* Index of function argument. */
          246  +  void *pAux;                     /* Aux data pointer */
          247  +  void (*xDelete)(void *);        /* Destructor for the aux data */
          248  +  AuxData *pNext;                 /* Next element in list */
   250    249   };
   251    250   
   252    251   /*
   253    252   ** The "context" argument for a installable function.  A pointer to an
   254    253   ** instance of this structure is the first argument to the routines used
   255    254   ** implement the SQL functions.
   256    255   **
................................................................................
   260    259   ** structure are known.
   261    260   **
   262    261   ** This structure is defined inside of vdbeInt.h because it uses substructures
   263    262   ** (Mem) which are only defined there.
   264    263   */
   265    264   struct sqlite3_context {
   266    265     FuncDef *pFunc;       /* Pointer to function information.  MUST BE FIRST */
   267         -  VdbeFunc *pVdbeFunc;  /* Auxilary data, if created. */
   268    266     Mem s;                /* The return value is stored here */
   269    267     Mem *pMem;            /* Memory cell used to store aggregate context */
   270    268     CollSeq *pColl;       /* Collating sequence */
   271    269     int isError;          /* Error code returned by the function. */
   272    270     int skipFlag;         /* Skip skip accumulator loading if true */
          271  +  int iOp;              /* Instruction number of OP_Function */
          272  +  Vdbe *pVdbe;          /* The VM that owns this context */
   273    273   };
   274    274   
   275    275   /*
   276    276   ** An Explain object accumulates indented output which is helpful
   277    277   ** in describing recursive data structures.
   278    278   */
   279    279   struct Explain {
................................................................................
   333    333     u8 minWriteFileFormat;  /* Minimum file format for writable database files */
   334    334     bft explain:2;          /* True if EXPLAIN present on SQL command */
   335    335     bft inVtabMethod:2;     /* See comments above */
   336    336     bft changeCntOn:1;      /* True to update the change-counter */
   337    337     bft expired:1;          /* True if the VM needs to be recompiled */
   338    338     bft runOnlyOnce:1;      /* Automatically expire on reset */
   339    339     bft usesStmtJournal:1;  /* True if uses a statement journal */
   340         -  bft readOnly:1;         /* True for read-only statements */
          340  +  bft readOnly:1;         /* True for statements that do not write */
          341  +  bft bIsReader:1;        /* True for statements that read */
   341    342     bft isPrepareV2:1;      /* True if prepared with prepare_v2() */
   342    343     bft doingRerun:1;       /* True if rerunning after an auto-reprepare */
   343    344     int nChange;            /* Number of db changes made since last reset */
   344    345     yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   345    346     yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   346    347     int iStatement;         /* Statement number (or 0 if has not opened stmt) */
   347         -  int aCounter[3];        /* Counters used by sqlite3_stmt_status() */
          348  +  int aCounter[4];        /* Counters used by sqlite3_stmt_status() */
   348    349   #ifndef SQLITE_OMIT_TRACE
   349    350     i64 startTime;          /* Time when query started - used for profiling */
   350    351   #endif
   351    352     i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
   352    353     i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
          354  +  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
   353    355     char *zSql;             /* Text of the SQL statement that generated this */
   354    356     void *pFree;            /* Free this when deleting the vdbe */
   355    357   #ifdef SQLITE_DEBUG
   356    358     FILE *trace;            /* Write an execution trace here, if not NULL */
   357    359   #endif
   358    360   #ifdef SQLITE_ENABLE_TREE_EXPLAIN
   359    361     Explain *pExplain;      /* The explainer */
................................................................................
   362    364     VdbeFrame *pFrame;      /* Parent frame */
   363    365     VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
   364    366     int nFrame;             /* Number of frames in pFrame list */
   365    367     u32 expmask;            /* Binding to these vars invalidates VM */
   366    368     SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */
   367    369     int nOnceFlag;          /* Size of array aOnceFlag[] */
   368    370     u8 *aOnceFlag;          /* Flags for OP_Once */
          371  +  AuxData *pAuxData;      /* Linked list of auxdata allocations */
   369    372   };
   370    373   
   371    374   /*
   372    375   ** The following are allowed values for Vdbe.magic
   373    376   */
   374    377   #define VDBE_MAGIC_INIT     0x26bceaa5    /* Building a VDBE program */
   375    378   #define VDBE_MAGIC_RUN      0xbdf20da3    /* VDBE is ready to execute */
................................................................................
   385    388   #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE)
   386    389   void sqlite3VdbePrintOp(FILE*, int, Op*);
   387    390   #endif
   388    391   u32 sqlite3VdbeSerialTypeLen(u32);
   389    392   u32 sqlite3VdbeSerialType(Mem*, int);
   390    393   u32 sqlite3VdbeSerialPut(unsigned char*, int, Mem*, int);
   391    394   u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*);
   392         -void sqlite3VdbeDeleteAuxData(VdbeFunc*, int);
          395  +void sqlite3VdbeDeleteAuxData(Vdbe*, int, int);
   393    396   
   394    397   int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *);
   395    398   int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*);
   396    399   int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *);
   397    400   int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*);
   398    401   int sqlite3VdbeExec(Vdbe*);
   399    402   int sqlite3VdbeList(Vdbe*);

Changes to src/vdbeapi.c.

   378    378       goto end_of_step;
   379    379     }
   380    380     if( p->pc<0 ){
   381    381       /* If there are no other statements currently running, then
   382    382       ** reset the interrupt flag.  This prevents a call to sqlite3_interrupt
   383    383       ** from interrupting a statement that has not yet started.
   384    384       */
   385         -    if( db->activeVdbeCnt==0 ){
          385  +    if( db->nVdbeActive==0 ){
   386    386         db->u1.isInterrupted = 0;
   387    387       }
   388    388   
   389         -    assert( db->writeVdbeCnt>0 || db->autoCommit==0 || db->nDeferredCons==0 );
          389  +    assert( db->nVdbeWrite>0 || db->autoCommit==0 
          390  +        || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
          391  +    );
   390    392   
   391    393   #ifndef SQLITE_OMIT_TRACE
   392    394       if( db->xProfile && !db->init.busy ){
   393    395         sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
   394    396       }
   395    397   #endif
   396    398   
   397         -    db->activeVdbeCnt++;
   398         -    if( p->readOnly==0 ) db->writeVdbeCnt++;
          399  +    db->nVdbeActive++;
          400  +    if( p->readOnly==0 ) db->nVdbeWrite++;
          401  +    if( p->bIsReader ) db->nVdbeRead++;
   399    402       p->pc = 0;
   400    403     }
   401    404   #ifndef SQLITE_OMIT_EXPLAIN
   402    405     if( p->explain ){
   403    406       rc = sqlite3VdbeList(p);
   404    407     }else
   405    408   #endif /* SQLITE_OMIT_EXPLAIN */
   406    409     {
   407         -    db->vdbeExecCnt++;
          410  +    db->nVdbeExec++;
   408    411       rc = sqlite3VdbeExec(p);
   409         -    db->vdbeExecCnt--;
          412  +    db->nVdbeExec--;
   410    413     }
   411    414   
   412    415   #ifndef SQLITE_OMIT_TRACE
   413    416     /* Invoke the profile callback if there is one
   414    417     */
   415    418     if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){
   416    419       sqlite3_int64 iNow;
................................................................................
   577    580   }
   578    581   
   579    582   /*
   580    583   ** Return the auxilary data pointer, if any, for the iArg'th argument to
   581    584   ** the user-function defined by pCtx.
   582    585   */
   583    586   void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){
   584         -  VdbeFunc *pVdbeFunc;
          587  +  AuxData *pAuxData;
   585    588   
   586    589     assert( sqlite3_mutex_held(pCtx->s.db->mutex) );
   587         -  pVdbeFunc = pCtx->pVdbeFunc;
   588         -  if( !pVdbeFunc || iArg>=pVdbeFunc->nAux || iArg<0 ){
   589         -    return 0;
          590  +  for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){
          591  +    if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break;
   590    592     }
   591         -  return pVdbeFunc->apAux[iArg].pAux;
          593  +
          594  +  return (pAuxData ? pAuxData->pAux : 0);
   592    595   }
   593    596   
   594    597   /*
   595    598   ** Set the auxilary data pointer and delete function, for the iArg'th
   596    599   ** argument to the user-function defined by pCtx. Any previous value is
   597    600   ** deleted by calling the delete function specified when it was set.
   598    601   */
   599    602   void sqlite3_set_auxdata(
   600    603     sqlite3_context *pCtx, 
   601    604     int iArg, 
   602    605     void *pAux, 
   603    606     void (*xDelete)(void*)
   604    607   ){
   605         -  struct AuxData *pAuxData;
   606         -  VdbeFunc *pVdbeFunc;
   607         -  if( iArg<0 ) goto failed;
          608  +  AuxData *pAuxData;
          609  +  Vdbe *pVdbe = pCtx->pVdbe;
   608    610   
   609    611     assert( sqlite3_mutex_held(pCtx->s.db->mutex) );
   610         -  pVdbeFunc = pCtx->pVdbeFunc;
   611         -  if( !pVdbeFunc || pVdbeFunc->nAux<=iArg ){
   612         -    int nAux = (pVdbeFunc ? pVdbeFunc->nAux : 0);
   613         -    int nMalloc = sizeof(VdbeFunc) + sizeof(struct AuxData)*iArg;
   614         -    pVdbeFunc = sqlite3DbRealloc(pCtx->s.db, pVdbeFunc, nMalloc);
   615         -    if( !pVdbeFunc ){
   616         -      goto failed;
   617         -    }
   618         -    pCtx->pVdbeFunc = pVdbeFunc;
   619         -    memset(&pVdbeFunc->apAux[nAux], 0, sizeof(struct AuxData)*(iArg+1-nAux));
   620         -    pVdbeFunc->nAux = iArg+1;
   621         -    pVdbeFunc->pFunc = pCtx->pFunc;
          612  +  if( iArg<0 ) goto failed;
          613  +
          614  +  for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){
          615  +    if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break;
   622    616     }
   623         -
   624         -  pAuxData = &pVdbeFunc->apAux[iArg];
   625         -  if( pAuxData->pAux && pAuxData->xDelete ){
          617  +  if( pAuxData==0 ){
          618  +    pAuxData = sqlite3DbMallocZero(pVdbe->db, sizeof(AuxData));
          619  +    if( !pAuxData ) goto failed;
          620  +    pAuxData->iOp = pCtx->iOp;
          621  +    pAuxData->iArg = iArg;
          622  +    pAuxData->pNext = pVdbe->pAuxData;
          623  +    pVdbe->pAuxData = pAuxData;
          624  +  }else if( pAuxData->xDelete ){
   626    625       pAuxData->xDelete(pAuxData->pAux);
   627    626     }
          627  +
   628    628     pAuxData->pAux = pAux;
   629    629     pAuxData->xDelete = xDelete;
   630    630     return;
   631    631   
   632    632   failed:
   633    633     if( xDelete ){
   634    634       xDelete(pAux);

Changes to src/vdbeaux.c.

   399    399   */
   400    400   static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
   401    401     int i;
   402    402     int nMaxArgs = *pMaxFuncArgs;
   403    403     Op *pOp;
   404    404     int *aLabel = p->aLabel;
   405    405     p->readOnly = 1;
          406  +  p->bIsReader = 0;
   406    407     for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
   407    408       u8 opcode = pOp->opcode;
   408    409   
   409    410       pOp->opflags = sqlite3OpcodeProperty[opcode];
   410    411       if( opcode==OP_Function || opcode==OP_AggStep ){
   411    412         if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;
   412         -    }else if( (opcode==OP_Transaction && pOp->p2!=0) || opcode==OP_Vacuum ){
          413  +    }else if( opcode==OP_Transaction ){
          414  +      if( pOp->p2!=0 ) p->readOnly = 0;
          415  +      p->bIsReader = 1;
          416  +    }else if( opcode==OP_AutoCommit || opcode==OP_Savepoint ){
          417  +      p->bIsReader = 1;
          418  +    }else if( opcode==OP_Vacuum
          419  +           || opcode==OP_JournalMode
          420  +#ifndef SQLITE_OMIT_WAL
          421  +           || opcode==OP_Checkpoint
          422  +#endif
          423  +    ){
   413    424         p->readOnly = 0;
          425  +      p->bIsReader = 1;
   414    426   #ifndef SQLITE_OMIT_VIRTUALTABLE
   415    427       }else if( opcode==OP_VUpdate ){
   416    428         if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
   417    429       }else if( opcode==OP_VFilter ){
   418    430         int n;
   419    431         assert( p->nOp - i >= 3 );
   420    432         assert( pOp[-1].opcode==OP_Integer );
................................................................................
   432    444       if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){
   433    445         assert( -1-pOp->p2<p->nLabel );
   434    446         pOp->p2 = aLabel[-1-pOp->p2];
   435    447       }
   436    448     }
   437    449     sqlite3DbFree(p->db, p->aLabel);
   438    450     p->aLabel = 0;
   439         -
   440    451     *pMaxFuncArgs = nMaxArgs;
          452  +  assert( p->bIsReader!=0 || p->btreeMask==0 );
   441    453   }
   442    454   
   443    455   /*
   444    456   ** Return the address of the next instruction to be inserted.
   445    457   */
   446    458   int sqlite3VdbeCurrentAddr(Vdbe *p){
   447    459     assert( p->magic==VDBE_MAGIC_INIT );
................................................................................
   596    608           sqlite3DbFree(db, p4);
   597    609           break;
   598    610         }
   599    611         case P4_MPRINTF: {
   600    612           if( db->pnBytesFreed==0 ) sqlite3_free(p4);
   601    613           break;
   602    614         }
   603         -      case P4_VDBEFUNC: {
   604         -        VdbeFunc *pVdbeFunc = (VdbeFunc *)p4;
   605         -        freeEphemeralFunction(db, pVdbeFunc->pFunc);
   606         -        if( db->pnBytesFreed==0 ) sqlite3VdbeDeleteAuxData(pVdbeFunc, 0);
   607         -        sqlite3DbFree(db, pVdbeFunc);
   608         -        break;
   609         -      }
   610    615         case P4_FUNCDEF: {
   611    616           freeEphemeralFunction(db, (FuncDef*)p4);
   612    617           break;
   613    618         }
   614    619         case P4_MEM: {
   615    620           if( db->pnBytesFreed==0 ){
   616    621             sqlite3ValueFree((sqlite3_value*)p4);
................................................................................
  1632   1637       releaseMemArray(&p->aMem[1], p->nMem);
  1633   1638     }
  1634   1639     while( p->pDelFrame ){
  1635   1640       VdbeFrame *pDel = p->pDelFrame;
  1636   1641       p->pDelFrame = pDel->pParent;
  1637   1642       sqlite3VdbeFrameDelete(pDel);
  1638   1643     }
         1644  +
         1645  +  /* Delete any auxdata allocations made by the VM */
         1646  +  sqlite3VdbeDeleteAuxData(p, -1, 0);
         1647  +  assert( p->pAuxData==0 );
  1639   1648   }
  1640   1649   
  1641   1650   /*
  1642   1651   ** Clean up the VM after execution.
  1643   1652   **
  1644   1653   ** This routine will automatically close any cursors, lists, and/or
  1645   1654   ** sorters that were left open.  It also deletes the values of
................................................................................
  1959   1968     }
  1960   1969   #endif
  1961   1970   
  1962   1971     return rc;
  1963   1972   }
  1964   1973   
  1965   1974   /* 
  1966         -** This routine checks that the sqlite3.activeVdbeCnt count variable
         1975  +** This routine checks that the sqlite3.nVdbeActive count variable
  1967   1976   ** matches the number of vdbe's in the list sqlite3.pVdbe that are
  1968   1977   ** currently active. An assertion fails if the two counts do not match.
  1969   1978   ** This is an internal self-check only - it is not an essential processing
  1970   1979   ** step.
  1971   1980   **
  1972   1981   ** This is a no-op if NDEBUG is defined.
  1973   1982   */
  1974   1983   #ifndef NDEBUG
  1975   1984   static void checkActiveVdbeCnt(sqlite3 *db){
  1976   1985     Vdbe *p;
  1977   1986     int cnt = 0;
  1978   1987     int nWrite = 0;
         1988  +  int nRead = 0;
  1979   1989     p = db->pVdbe;
  1980   1990     while( p ){
  1981   1991       if( p->magic==VDBE_MAGIC_RUN && p->pc>=0 ){
  1982   1992         cnt++;
  1983   1993         if( p->readOnly==0 ) nWrite++;
         1994  +      if( p->bIsReader ) nRead++;
  1984   1995       }
  1985   1996       p = p->pNext;
  1986   1997     }
  1987         -  assert( cnt==db->activeVdbeCnt );
  1988         -  assert( nWrite==db->writeVdbeCnt );
         1998  +  assert( cnt==db->nVdbeActive );
         1999  +  assert( nWrite==db->nVdbeWrite );
         2000  +  assert( nRead==db->nVdbeRead );
  1989   2001   }
  1990   2002   #else
  1991   2003   #define checkActiveVdbeCnt(x)
  1992   2004   #endif
  1993   2005   
  1994   2006   /*
  1995   2007   ** If the Vdbe passed as the first argument opened a statement-transaction,
................................................................................
  2046   2058       }
  2047   2059   
  2048   2060       /* If the statement transaction is being rolled back, also restore the 
  2049   2061       ** database handles deferred constraint counter to the value it had when 
  2050   2062       ** the statement transaction was opened.  */
  2051   2063       if( eOp==SAVEPOINT_ROLLBACK ){
  2052   2064         db->nDeferredCons = p->nStmtDefCons;
         2065  +      db->nDeferredImmCons = p->nStmtDefImmCons;
  2053   2066       }
  2054   2067     }
  2055   2068     return rc;
  2056   2069   }
  2057   2070   
  2058   2071   /*
  2059   2072   ** This function is called when a transaction opened by the database 
................................................................................
  2064   2077   ** If there are outstanding FK violations and this function returns 
  2065   2078   ** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY
  2066   2079   ** and write an error message to it. Then return SQLITE_ERROR.
  2067   2080   */
  2068   2081   #ifndef SQLITE_OMIT_FOREIGN_KEY
  2069   2082   int sqlite3VdbeCheckFk(Vdbe *p, int deferred){
  2070   2083     sqlite3 *db = p->db;
  2071         -  if( (deferred && db->nDeferredCons>0) || (!deferred && p->nFkConstraint>0) ){
         2084  +  if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) 
         2085  +   || (!deferred && p->nFkConstraint>0) 
         2086  +  ){
  2072   2087       p->rc = SQLITE_CONSTRAINT_FOREIGNKEY;
  2073   2088       p->errorAction = OE_Abort;
  2074   2089       sqlite3SetString(&p->zErrMsg, db, "foreign key constraint failed");
  2075   2090       return SQLITE_ERROR;
  2076   2091     }
  2077   2092     return SQLITE_OK;
  2078   2093   }
................................................................................
  2117   2132     if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag);
  2118   2133     closeAllCursors(p);
  2119   2134     if( p->magic!=VDBE_MAGIC_RUN ){
  2120   2135       return SQLITE_OK;
  2121   2136     }
  2122   2137     checkActiveVdbeCnt(db);
  2123   2138   
  2124         -  /* No commit or rollback needed if the program never started */
  2125         -  if( p->pc>=0 ){
         2139  +  /* No commit or rollback needed if the program never started or if the
         2140  +  ** SQL statement does not read or write a database file.  */
         2141  +  if( p->pc>=0 && p->bIsReader ){
  2126   2142       int mrc;   /* Primary error code from p->rc */
  2127   2143       int eStatementOp = 0;
  2128   2144       int isSpecialError;            /* Set to true if a 'special' error */
  2129   2145   
  2130   2146       /* Lock all btrees used by the statement */
  2131   2147       sqlite3VdbeEnter(p);
  2132   2148   
................................................................................
  2171   2187       ** VM, then we do either a commit or rollback of the current transaction. 
  2172   2188       **
  2173   2189       ** Note: This block also runs if one of the special errors handled 
  2174   2190       ** above has occurred. 
  2175   2191       */
  2176   2192       if( !sqlite3VtabInSync(db) 
  2177   2193        && db->autoCommit 
  2178         -     && db->writeVdbeCnt==(p->readOnly==0) 
         2194  +     && db->nVdbeWrite==(p->readOnly==0) 
  2179   2195       ){
  2180   2196         if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){
  2181   2197           rc = sqlite3VdbeCheckFk(p, 1);
  2182   2198           if( rc!=SQLITE_OK ){
  2183   2199             if( NEVER(p->readOnly) ){
  2184   2200               sqlite3VdbeLeave(p);
  2185   2201               return SQLITE_ERROR;
................................................................................
  2196   2212             sqlite3VdbeLeave(p);
  2197   2213             return SQLITE_BUSY;
  2198   2214           }else if( rc!=SQLITE_OK ){
  2199   2215             p->rc = rc;
  2200   2216             sqlite3RollbackAll(db, SQLITE_OK);
  2201   2217           }else{
  2202   2218             db->nDeferredCons = 0;
         2219  +          db->nDeferredImmCons = 0;
         2220  +          db->flags &= ~SQLITE_DeferFKs;
  2203   2221             sqlite3CommitInternalChanges(db);
  2204   2222           }
  2205   2223         }else{
  2206   2224           sqlite3RollbackAll(db, SQLITE_OK);
  2207   2225         }
  2208   2226         db->nStatement = 0;
  2209   2227       }else if( eStatementOp==0 ){
................................................................................
  2252   2270   
  2253   2271       /* Release the locks */
  2254   2272       sqlite3VdbeLeave(p);
  2255   2273     }
  2256   2274   
  2257   2275     /* We have successfully halted and closed the VM.  Record this fact. */
  2258   2276     if( p->pc>=0 ){
  2259         -    db->activeVdbeCnt--;
  2260         -    if( !p->readOnly ){
  2261         -      db->writeVdbeCnt--;
  2262         -    }
  2263         -    assert( db->activeVdbeCnt>=db->writeVdbeCnt );
         2277  +    db->nVdbeActive--;
         2278  +    if( !p->readOnly ) db->nVdbeWrite--;
         2279  +    if( p->bIsReader ) db->nVdbeRead--;
         2280  +    assert( db->nVdbeActive>=db->nVdbeRead );
         2281  +    assert( db->nVdbeRead>=db->nVdbeWrite );
         2282  +    assert( db->nVdbeWrite>=0 );
  2264   2283     }
  2265   2284     p->magic = VDBE_MAGIC_HALT;
  2266   2285     checkActiveVdbeCnt(db);
  2267   2286     if( p->db->mallocFailed ){
  2268   2287       p->rc = SQLITE_NOMEM;
  2269   2288     }
  2270   2289   
................................................................................
  2272   2291     ** by connection db have now been released. Call sqlite3ConnectionUnlocked() 
  2273   2292     ** to invoke any required unlock-notify callbacks.
  2274   2293     */
  2275   2294     if( db->autoCommit ){
  2276   2295       sqlite3ConnectionUnlocked(db);
  2277   2296     }
  2278   2297   
  2279         -  assert( db->activeVdbeCnt>0 || db->autoCommit==0 || db->nStatement==0 );
         2298  +  assert( db->nVdbeActive>0 || db->autoCommit==0 || db->nStatement==0 );
  2280   2299     return (p->rc==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK);
  2281   2300   }
  2282   2301   
  2283   2302   
  2284   2303   /*
  2285   2304   ** Each VDBE holds the result of the most recent sqlite3_step() call
  2286   2305   ** in p->rc.  This routine sets that result back to SQLITE_OK.
................................................................................
  2420   2439       assert( (rc & p->db->errMask)==rc );
  2421   2440     }
  2422   2441     sqlite3VdbeDelete(p);
  2423   2442     return rc;
  2424   2443   }
  2425   2444   
  2426   2445   /*
  2427         -** Call the destructor for each auxdata entry in pVdbeFunc for which
  2428         -** the corresponding bit in mask is clear.  Auxdata entries beyond 31
  2429         -** are always destroyed.  To destroy all auxdata entries, call this
  2430         -** routine with mask==0.
         2446  +** If parameter iOp is less than zero, then invoke the destructor for
         2447  +** all auxiliary data pointers currently cached by the VM passed as
         2448  +** the first argument.
         2449  +**
         2450  +** Or, if iOp is greater than or equal to zero, then the destructor is
         2451  +** only invoked for those auxiliary data pointers created by the user 
         2452  +** function invoked by the OP_Function opcode at instruction iOp of 
         2453  +** VM pVdbe, and only then if:
         2454  +**
         2455  +**    * the associated function parameter is the 32nd or later (counting
         2456  +**      from left to right), or
         2457  +**
         2458  +**    * the corresponding bit in argument mask is clear (where the first
         2459  +**      function parameter corrsponds to bit 0 etc.).
  2431   2460   */
  2432         -void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){
  2433         -  int i;
  2434         -  for(i=0; i<pVdbeFunc->nAux; i++){
  2435         -    struct AuxData *pAux = &pVdbeFunc->apAux[i];
  2436         -    if( (i>31 || !(mask&(((u32)1)<<i))) && pAux->pAux ){
         2461  +void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){
         2462  +  AuxData **pp = &pVdbe->pAuxData;
         2463  +  while( *pp ){
         2464  +    AuxData *pAux = *pp;
         2465  +    if( (iOp<0)
         2466  +     || (pAux->iOp==iOp && (pAux->iArg>31 || !(mask & ((u32)1<<pAux->iArg))))
         2467  +    ){
  2437   2468         if( pAux->xDelete ){
  2438   2469           pAux->xDelete(pAux->pAux);
  2439   2470         }
  2440         -      pAux->pAux = 0;
         2471  +      *pp = pAux->pNext;
         2472  +      sqlite3DbFree(pVdbe->db, pAux);
         2473  +    }else{
         2474  +      pp= &pAux->pNext;
  2441   2475       }
  2442   2476     }
  2443   2477   }
  2444   2478   
  2445   2479   /*
  2446   2480   ** Free all memory associated with the Vdbe passed as the second argument,
  2447   2481   ** except for object itself, which is preserved.

Changes to src/vdbetrace.c.

    43     43       zSql += n;
    44     44     }
    45     45     return nTotal;
    46     46   }
    47     47   
    48     48   /*
    49     49   ** This function returns a pointer to a nul-terminated string in memory
    50         -** obtained from sqlite3DbMalloc(). If sqlite3.vdbeExecCnt is 1, then the
           50  +** obtained from sqlite3DbMalloc(). If sqlite3.nVdbeExec is 1, then the
    51     51   ** string contains a copy of zRawSql but with host parameters expanded to 
    52         -** their current bindings. Or, if sqlite3.vdbeExecCnt is greater than 1, 
           52  +** their current bindings. Or, if sqlite3.nVdbeExec is greater than 1, 
    53     53   ** then the returned string holds a copy of zRawSql with "-- " prepended
    54     54   ** to each line of text.
    55     55   **
    56     56   ** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then
    57     57   ** then long strings and blobs are truncated to that many bytes.  This
    58     58   ** can be used to prevent unreasonably large trace strings when dealing
    59     59   ** with large (multi-megabyte) strings and blobs.
................................................................................
    83     83     StrAccum out;            /* Accumulate the output here */
    84     84     char zBase[100];         /* Initial working space */
    85     85   
    86     86     db = p->db;
    87     87     sqlite3StrAccumInit(&out, zBase, sizeof(zBase), 
    88     88                         db->aLimit[SQLITE_LIMIT_LENGTH]);
    89     89     out.db = db;
    90         -  if( db->vdbeExecCnt>1 ){
           90  +  if( db->nVdbeExec>1 ){
    91     91       while( *zRawSql ){
    92     92         const char *zStart = zRawSql;
    93     93         while( *(zRawSql++)!='\n' && *zRawSql );
    94     94         sqlite3StrAccumAppend(&out, "-- ", 3);
    95     95         sqlite3StrAccumAppend(&out, zStart, (int)(zRawSql-zStart));
    96     96       }
    97     97     }else{

Changes to src/vtab.c.

   821    821     db->aVTrans = 0;
   822    822     for(i=0; rc==SQLITE_OK && i<db->nVTrans; i++){
   823    823       int (*x)(sqlite3_vtab *);
   824    824       sqlite3_vtab *pVtab = aVTrans[i]->pVtab;
   825    825       if( pVtab && (x = pVtab->pModule->xSync)!=0 ){
   826    826         rc = x(pVtab);
   827    827         sqlite3DbFree(db, *pzErrmsg);
   828         -      *pzErrmsg = sqlite3DbStrDup(db, pVtab->zErrMsg);
   829         -      sqlite3_free(pVtab->zErrMsg);
          828  +      *pzErrmsg = pVtab->zErrMsg;
          829  +      pVtab->zErrMsg = 0;
   830    830       }
   831    831     }
   832    832     db->aVTrans = aVTrans;
   833    833     return rc;
   834    834   }
   835    835   
   836    836   /*

Changes to src/wal.c.

  2459   2459     /* If another connection has written to the database file since the
  2460   2460     ** time the read transaction on this connection was started, then
  2461   2461     ** the write is disallowed.
  2462   2462     */
  2463   2463     if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
  2464   2464       walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
  2465   2465       pWal->writeLock = 0;
  2466         -    rc = SQLITE_BUSY;
         2466  +    rc = SQLITE_BUSY_SNAPSHOT;
  2467   2467     }
  2468   2468   
  2469   2469     return rc;
  2470   2470   }
  2471   2471   
  2472   2472   /*
  2473   2473   ** End a write transaction.  The commit has already been done.  This

Changes to src/where.c.

    23     23   ** Trace output macros
    24     24   */
    25     25   #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG)
    26     26   /***/ int sqlite3WhereTrace = 0;
    27     27   #endif
    28     28   #if defined(SQLITE_DEBUG) \
    29     29       && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE))
    30         -# define WHERETRACE(X)  if(sqlite3WhereTrace) sqlite3DebugPrintf X
           30  +# define WHERETRACE(K,X)  if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X
           31  +# define WHERETRACE_ENABLED 1
    31     32   #else
    32         -# define WHERETRACE(X)
           33  +# define WHERETRACE(K,X)
    33     34   #endif
    34     35   
    35     36   /* Forward reference
    36     37   */
    37     38   typedef struct WhereClause WhereClause;
    38     39   typedef struct WhereMaskSet WhereMaskSet;
    39     40   typedef struct WhereOrInfo WhereOrInfo;
    40     41   typedef struct WhereAndInfo WhereAndInfo;
    41         -typedef struct WhereCost WhereCost;
           42  +typedef struct WhereLevel WhereLevel;
           43  +typedef struct WhereLoop WhereLoop;
           44  +typedef struct WherePath WherePath;
           45  +typedef struct WhereTerm WhereTerm;
           46  +typedef struct WhereLoopBuilder WhereLoopBuilder;
           47  +typedef struct WhereScan WhereScan;
           48  +typedef struct WhereOrCost WhereOrCost;
           49  +typedef struct WhereOrSet WhereOrSet;
           50  +
           51  +/*
           52  +** Cost X is tracked as 10*log2(X) stored in a 16-bit integer.  The
           53  +** maximum cost for ordinary tables is 64*(2**63) which becomes 6900.
           54  +** (Virtual tables can return a larger cost, but let's assume they do not.)
           55  +** So all costs can be stored in a 16-bit unsigned integer without risk
           56  +** of overflow.
           57  +**
           58  +** Costs are estimates, so don't go to the computational trouble to compute
           59  +** 10*log2(X) exactly.  Instead, a close estimate is used.  Any value of
           60  +** X<=1 is stored as 0.  X=2 is 10.  X=3 is 16.  X=1000 is 99. etc.
           61  +**
           62  +** The tool/wherecosttest.c source file implements a command-line program
           63  +** that will convert between WhereCost to integers and do addition and
           64  +** multiplication on WhereCost values.  That command-line program is a
           65  +** useful utility to have around when working with this module.
           66  +*/
           67  +typedef unsigned short int WhereCost;
           68  +
           69  +/*
           70  +** This object contains information needed to implement a single nested
           71  +** loop in WHERE clause.
           72  +**
           73  +** Contrast this object with WhereLoop.  This object describes the
           74  +** implementation of the loop.  WhereLoop describes the algorithm.
           75  +** This object contains a pointer to the WhereLoop algorithm as one of
           76  +** its elements.
           77  +**
           78  +** The WhereInfo object contains a single instance of this object for
           79  +** each term in the FROM clause (which is to say, for each of the
           80  +** nested loops as implemented).  The order of WhereLevel objects determines
           81  +** the loop nested order, with WhereInfo.a[0] being the outer loop and
           82  +** WhereInfo.a[WhereInfo.nLevel-1] being the inner loop.
           83  +*/
           84  +struct WhereLevel {
           85  +  int iLeftJoin;        /* Memory cell used to implement LEFT OUTER JOIN */
           86  +  int iTabCur;          /* The VDBE cursor used to access the table */
           87  +  int iIdxCur;          /* The VDBE cursor used to access pIdx */
           88  +  int addrBrk;          /* Jump here to break out of the loop */
           89  +  int addrNxt;          /* Jump here to start the next IN combination */
           90  +  int addrCont;         /* Jump here to continue with the next loop cycle */
           91  +  int addrFirst;        /* First instruction of interior of the loop */
           92  +  u8 iFrom;             /* Which entry in the FROM clause */
           93  +  u8 op, p5;            /* Opcode and P5 of the opcode that ends the loop */
           94  +  int p1, p2;           /* Operands of the opcode used to ends the loop */
           95  +  union {               /* Information that depends on pWLoop->wsFlags */
           96  +    struct {
           97  +      int nIn;              /* Number of entries in aInLoop[] */
           98  +      struct InLoop {
           99  +        int iCur;              /* The VDBE cursor used by this IN operator */
          100  +        int addrInTop;         /* Top of the IN loop */
          101  +        u8 eEndLoopOp;         /* IN Loop terminator. OP_Next or OP_Prev */
          102  +      } *aInLoop;           /* Information about each nested IN operator */
          103  +    } in;                 /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */
          104  +    Index *pCovidx;       /* Possible covering index for WHERE_MULTI_OR */
          105  +  } u;
          106  +  struct WhereLoop *pWLoop;  /* The selected WhereLoop object */
          107  +};
          108  +
          109  +/*
          110  +** Each instance of this object represents an algorithm for evaluating one
          111  +** term of a join.  Every term of the FROM clause will have at least
          112  +** one corresponding WhereLoop object (unless INDEXED BY constraints
          113  +** prevent a query solution - which is an error) and many terms of the
          114  +** FROM clause will have multiple WhereLoop objects, each describing a
          115  +** potential way of implementing that FROM-clause term, together with
          116  +** dependencies and cost estimates for using the chosen algorithm.
          117  +**
          118  +** Query planning consists of building up a collection of these WhereLoop
          119  +** objects, then computing a particular sequence of WhereLoop objects, with
          120  +** one WhereLoop object per FROM clause term, that satisfy all dependencies
          121  +** and that minimize the overall cost.
          122  +*/
          123  +struct WhereLoop {
          124  +  Bitmask prereq;       /* Bitmask of other loops that must run first */
          125  +  Bitmask maskSelf;     /* Bitmask identifying table iTab */
          126  +#ifdef SQLITE_DEBUG
          127  +  char cId;             /* Symbolic ID of this loop for debugging use */
          128  +#endif
          129  +  u8 iTab;              /* Position in FROM clause of table for this loop */
          130  +  u8 iSortIdx;          /* Sorting index number.  0==None */
          131  +  WhereCost rSetup;     /* One-time setup cost (ex: create transient index) */
          132  +  WhereCost rRun;       /* Cost of running each loop */
          133  +  WhereCost nOut;       /* Estimated number of output rows */
          134  +  union {
          135  +    struct {               /* Information for internal btree tables */
          136  +      int nEq;               /* Number of equality constraints */
          137  +      Index *pIndex;         /* Index used, or NULL */
          138  +    } btree;
          139  +    struct {               /* Information for virtual tables */
          140  +      int idxNum;            /* Index number */
          141  +      u8 needFree;           /* True if sqlite3_free(idxStr) is needed */
          142  +      u8 isOrdered;          /* True if satisfies ORDER BY */
          143  +      u16 omitMask;          /* Terms that may be omitted */
          144  +      char *idxStr;          /* Index identifier string */
          145  +    } vtab;
          146  +  } u;
          147  +  u32 wsFlags;          /* WHERE_* flags describing the plan */
          148  +  u16 nLTerm;           /* Number of entries in aLTerm[] */
          149  +  /**** whereLoopXfer() copies fields above ***********************/
          150  +# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot)
          151  +  u16 nLSlot;           /* Number of slots allocated for aLTerm[] */
          152  +  WhereTerm **aLTerm;   /* WhereTerms used */
          153  +  WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */
          154  +  WhereTerm *aLTermSpace[4];  /* Initial aLTerm[] space */
          155  +};
          156  +
          157  +/* This object holds the prerequisites and the cost of running a
          158  +** subquery on one operand of an OR operator in the WHERE clause.
          159  +** See WhereOrSet for additional information 
          160  +*/
          161  +struct WhereOrCost {
          162  +  Bitmask prereq;     /* Prerequisites */
          163  +  WhereCost rRun;     /* Cost of running this subquery */
          164  +  WhereCost nOut;     /* Number of outputs for this subquery */
          165  +};
          166  +
          167  +/* The WhereOrSet object holds a set of possible WhereOrCosts that
          168  +** correspond to the subquery(s) of OR-clause processing.  At most
          169  +** favorable N_OR_COST elements are retained.
          170  +*/
          171  +#define N_OR_COST 3
          172  +struct WhereOrSet {
          173  +  u16 n;                      /* Number of valid a[] entries */
          174  +  WhereOrCost a[N_OR_COST];   /* Set of best costs */
          175  +};
          176  +
          177  +
          178  +/* Forward declaration of methods */
          179  +static int whereLoopResize(sqlite3*, WhereLoop*, int);
          180  +
          181  +/*
          182  +** Each instance of this object holds a sequence of WhereLoop objects
          183  +** that implement some or all of a query plan.
          184  +**
          185  +** Think of each WhereLoop object as a node in a graph with arcs
          186  +** showing dependences and costs for travelling between nodes.  (That is
          187  +** not a completely accurate description because WhereLoop costs are a
          188  +** vector, not a scalar, and because dependences are many-to-one, not
          189  +** one-to-one as are graph nodes.  But it is a useful visualization aid.)
          190  +** Then a WherePath object is a path through the graph that visits some
          191  +** or all of the WhereLoop objects once.
          192  +**
          193  +** The "solver" works by creating the N best WherePath objects of length
          194  +** 1.  Then using those as a basis to compute the N best WherePath objects
          195  +** of length 2.  And so forth until the length of WherePaths equals the
          196  +** number of nodes in the FROM clause.  The best (lowest cost) WherePath
          197  +** at the end is the choosen query plan.
          198  +*/
          199  +struct WherePath {
          200  +  Bitmask maskLoop;     /* Bitmask of all WhereLoop objects in this path */
          201  +  Bitmask revLoop;      /* aLoop[]s that should be reversed for ORDER BY */
          202  +  WhereCost nRow;       /* Estimated number of rows generated by this path */
          203  +  WhereCost rCost;      /* Total cost of this path */
          204  +  u8 isOrdered;         /* True if this path satisfies ORDER BY */
          205  +  u8 isOrderedValid;    /* True if the isOrdered field is valid */
          206  +  WhereLoop **aLoop;    /* Array of WhereLoop objects implementing this path */
          207  +};
    42    208   
    43    209   /*
    44    210   ** The query generator uses an array of instances of this structure to
    45    211   ** help it analyze the subexpressions of the WHERE clause.  Each WHERE
    46    212   ** clause subexpression is separated from the others by AND operators,
    47    213   ** usually, or sometimes subexpressions separated by OR.
    48    214   **
................................................................................
    87    253   ** bits in the Bitmask.  So, in the example above, the cursor numbers
    88    254   ** would be mapped into integers 0 through 7.
    89    255   **
    90    256   ** The number of terms in a join is limited by the number of bits
    91    257   ** in prereqRight and prereqAll.  The default is 64 bits, hence SQLite
    92    258   ** is only able to process joins with 64 or fewer tables.
    93    259   */
    94         -typedef struct WhereTerm WhereTerm;
    95    260   struct WhereTerm {
    96    261     Expr *pExpr;            /* Pointer to the subexpression that is this term */
    97    262     int iParent;            /* Disable pWC->a[iParent] when this term disabled */
    98    263     int leftCursor;         /* Cursor number of X in "X <op> <expr>" */
    99    264     union {
   100    265       int leftColumn;         /* Column number of X in "X <op> <expr>" */
   101    266       WhereOrInfo *pOrInfo;   /* Extra information if (eOperator & WO_OR)!=0 */
................................................................................
   121    286   #define TERM_OR_OK      0x40   /* Used during OR-clause processing */
   122    287   #ifdef SQLITE_ENABLE_STAT3
   123    288   #  define TERM_VNULL    0x80   /* Manufactured x>NULL or x<=NULL term */
   124    289   #else
   125    290   #  define TERM_VNULL    0x00   /* Disabled if not using stat3 */
   126    291   #endif
   127    292   
          293  +/*
          294  +** An instance of the WhereScan object is used as an iterator for locating
          295  +** terms in the WHERE clause that are useful to the query planner.
          296  +*/
          297  +struct WhereScan {
          298  +  WhereClause *pOrigWC;      /* Original, innermost WhereClause */
          299  +  WhereClause *pWC;          /* WhereClause currently being scanned */
          300  +  char *zCollName;           /* Required collating sequence, if not NULL */
          301  +  char idxaff;               /* Must match this affinity, if zCollName!=NULL */
          302  +  unsigned char nEquiv;      /* Number of entries in aEquiv[] */
          303  +  unsigned char iEquiv;      /* Next unused slot in aEquiv[] */
          304  +  u32 opMask;                /* Acceptable operators */
          305  +  int k;                     /* Resume scanning at this->pWC->a[this->k] */
          306  +  int aEquiv[22];            /* Cursor,Column pairs for equivalence classes */
          307  +};
          308  +
   128    309   /*
   129    310   ** An instance of the following structure holds all information about a
   130    311   ** WHERE clause.  Mostly this is a container for one or more WhereTerms.
   131    312   **
   132    313   ** Explanation of pOuter:  For a WHERE clause of the form
   133    314   **
   134    315   **           a AND ((b AND c) OR (d AND e)) AND f
   135    316   **
   136    317   ** There are separate WhereClause objects for the whole clause and for
   137    318   ** the subclauses "(b AND c)" and "(d AND e)".  The pOuter field of the
   138    319   ** subclauses points to the WhereClause object for the whole clause.
   139    320   */
   140    321   struct WhereClause {
   141         -  Parse *pParse;           /* The parser context */
   142         -  WhereMaskSet *pMaskSet;  /* Mapping of table cursor numbers to bitmasks */
          322  +  WhereInfo *pWInfo;       /* WHERE clause processing context */
   143    323     WhereClause *pOuter;     /* Outer conjunction */
   144    324     u8 op;                   /* Split operator.  TK_AND or TK_OR */
   145         -  u16 wctrlFlags;          /* Might include WHERE_AND_ONLY */
   146    325     int nTerm;               /* Number of terms */
   147    326     int nSlot;               /* Number of entries in a[] */
   148    327     WhereTerm *a;            /* Each a[] describes a term of the WHERE cluase */
   149    328   #if defined(SQLITE_SMALL_STACK)
   150    329     WhereTerm aStatic[1];    /* Initial static space for a[] */
   151    330   #else
   152    331     WhereTerm aStatic[8];    /* Initial static space for a[] */
................................................................................
   198    377   */
   199    378   struct WhereMaskSet {
   200    379     int n;                        /* Number of assigned cursor values */
   201    380     int ix[BMS];                  /* Cursor assigned to each bit */
   202    381   };
   203    382   
   204    383   /*
   205         -** A WhereCost object records a lookup strategy and the estimated
   206         -** cost of pursuing that strategy.
          384  +** This object is a convenience wrapper holding all information needed
          385  +** to construct WhereLoop objects for a particular query.
   207    386   */
   208         -struct WhereCost {
   209         -  WherePlan plan;    /* The lookup strategy */
   210         -  double rCost;      /* Overall cost of pursuing this search strategy */
   211         -  Bitmask used;      /* Bitmask of cursors used by this plan */
          387  +struct WhereLoopBuilder {
          388  +  WhereInfo *pWInfo;        /* Information about this WHERE */
          389  +  WhereClause *pWC;         /* WHERE clause terms */
          390  +  ExprList *pOrderBy;       /* ORDER BY clause */
          391  +  WhereLoop *pNew;          /* Template WhereLoop */
          392  +  WhereOrSet *pOrSet;       /* Record best loops here, if not NULL */
          393  +};
          394  +
          395  +/*
          396  +** The WHERE clause processing routine has two halves.  The
          397  +** first part does the start of the WHERE loop and the second
          398  +** half does the tail of the WHERE loop.  An instance of
          399  +** this structure is returned by the first half and passed
          400  +** into the second half to give some continuity.
          401  +**
          402  +** An instance of this object holds the complete state of the query
          403  +** planner.
          404  +*/
          405  +struct WhereInfo {
          406  +  Parse *pParse;            /* Parsing and code generating context */
          407  +  SrcList *pTabList;        /* List of tables in the join */
          408  +  ExprList *pOrderBy;       /* The ORDER BY clause or NULL */
          409  +  ExprList *pResultSet;     /* Result set. DISTINCT operates on these */
          410  +  WhereLoop *pLoops;        /* List of all WhereLoop objects */
          411  +  Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
          412  +  WhereCost nRowOut;        /* Estimated number of output rows */
          413  +  u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
          414  +  u8 bOBSat;                /* ORDER BY satisfied by indices */
          415  +  u8 okOnePass;             /* Ok to use one-pass algorithm for UPDATE/DELETE */
          416  +  u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
          417  +  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
          418  +  u8 nLevel;                /* Number of nested loop */
          419  +  int iTop;                 /* The very beginning of the WHERE loop */
          420  +  int iContinue;            /* Jump here to continue with next record */
          421  +  int iBreak;               /* Jump here to break out of the loop */
          422  +  int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
          423  +  WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
          424  +  WhereClause sWC;          /* Decomposition of the WHERE clause */
          425  +  WhereLevel a[1];          /* Information about each nest loop in WHERE */
   212    426   };
   213    427   
   214    428   /*
   215         -** Bitmasks for the operators that indices are able to exploit.  An
          429  +** Bitmasks for the operators on WhereTerm objects.  These are all
          430  +** operators that are of interest to the query planner.  An
   216    431   ** OR-ed combination of these values can be used when searching for
   217         -** terms in the where clause.
          432  +** particular WhereTerms within a WhereClause.
   218    433   */
   219    434   #define WO_IN     0x001
   220    435   #define WO_EQ     0x002
   221    436   #define WO_LT     (WO_EQ<<(TK_LT-TK_EQ))
   222    437   #define WO_LE     (WO_EQ<<(TK_LE-TK_EQ))
   223    438   #define WO_GT     (WO_EQ<<(TK_GT-TK_EQ))
   224    439   #define WO_GE     (WO_EQ<<(TK_GE-TK_EQ))
................................................................................
   229    444   #define WO_EQUIV  0x400       /* Of the form A==B, both columns */
   230    445   #define WO_NOOP   0x800       /* This term does not restrict search space */
   231    446   
   232    447   #define WO_ALL    0xfff       /* Mask of all possible WO_* values */
   233    448   #define WO_SINGLE 0x0ff       /* Mask of all non-compound WO_* values */
   234    449   
   235    450   /*
   236         -** Value for wsFlags returned by bestIndex() and stored in
   237         -** WhereLevel.wsFlags.  These flags determine which search
   238         -** strategies are appropriate.
   239         -**
   240         -** The least significant 12 bits is reserved as a mask for WO_ values above.
   241         -** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL.
   242         -** But if the table is the right table of a left join, WhereLevel.wsFlags
   243         -** is set to WO_IN|WO_EQ.  The WhereLevel.wsFlags field can then be used as
   244         -** the "op" parameter to findTerm when we are resolving equality constraints.
   245         -** ISNULL constraints will then not be used on the right table of a left
   246         -** join.  Tickets #2177 and #2189.
   247         -*/
   248         -#define WHERE_ROWID_EQ     0x00001000  /* rowid=EXPR or rowid IN (...) */
   249         -#define WHERE_ROWID_RANGE  0x00002000  /* rowid<EXPR and/or rowid>EXPR */
   250         -#define WHERE_COLUMN_EQ    0x00010000  /* x=EXPR or x IN (...) or x IS NULL */
   251         -#define WHERE_COLUMN_RANGE 0x00020000  /* x<EXPR and/or x>EXPR */
   252         -#define WHERE_COLUMN_IN    0x00040000  /* x IN (...) */
   253         -#define WHERE_COLUMN_NULL  0x00080000  /* x IS NULL */
   254         -#define WHERE_INDEXED      0x000f0000  /* Anything that uses an index */
   255         -#define WHERE_NOT_FULLSCAN 0x100f3000  /* Does not do a full table scan */
   256         -#define WHERE_IN_ABLE      0x080f1000  /* Able to support an IN operator */
   257         -#define WHERE_TOP_LIMIT    0x00100000  /* x<EXPR or x<=EXPR constraint */
   258         -#define WHERE_BTM_LIMIT    0x00200000  /* x>EXPR or x>=EXPR constraint */
   259         -#define WHERE_BOTH_LIMIT   0x00300000  /* Both x>EXPR and x<EXPR */
   260         -#define WHERE_IDX_ONLY     0x00400000  /* Use index only - omit table */
   261         -#define WHERE_ORDERED      0x00800000  /* Output will appear in correct order */
   262         -#define WHERE_REVERSE      0x01000000  /* Scan in reverse order */
   263         -#define WHERE_UNIQUE       0x02000000  /* Selects no more than one row */
   264         -#define WHERE_ALL_UNIQUE   0x04000000  /* This and all prior have one row */
   265         -#define WHERE_OB_UNIQUE    0x00004000  /* Values in ORDER BY columns are 
   266         -                                       ** different for every output row */
   267         -#define WHERE_VIRTUALTABLE 0x08000000  /* Use virtual-table processing */
   268         -#define WHERE_MULTI_OR     0x10000000  /* OR using multiple indices */
   269         -#define WHERE_TEMP_INDEX   0x20000000  /* Uses an ephemeral index */
   270         -#define WHERE_DISTINCT     0x40000000  /* Correct order for DISTINCT */
   271         -#define WHERE_COVER_SCAN   0x80000000  /* Full scan of a covering index */
   272         -
   273         -/*
   274         -** This module contains many separate subroutines that work together to
   275         -** find the best indices to use for accessing a particular table in a query.
   276         -** An instance of the following structure holds context information about the
   277         -** index search so that it can be more easily passed between the various
   278         -** routines.
   279         -*/
   280         -typedef struct WhereBestIdx WhereBestIdx;
   281         -struct WhereBestIdx {
   282         -  Parse *pParse;                  /* Parser context */
   283         -  WhereClause *pWC;               /* The WHERE clause */
   284         -  struct SrcList_item *pSrc;      /* The FROM clause term to search */
   285         -  Bitmask notReady;               /* Mask of cursors not available */
   286         -  Bitmask notValid;               /* Cursors not available for any purpose */
   287         -  ExprList *pOrderBy;             /* The ORDER BY clause */
   288         -  ExprList *pDistinct;            /* The select-list if query is DISTINCT */
   289         -  sqlite3_index_info **ppIdxInfo; /* Index information passed to xBestIndex */
   290         -  int i, n;                       /* Which loop is being coded; # of loops */
   291         -  WhereLevel *aLevel;             /* Info about outer loops */
   292         -  WhereCost cost;                 /* Lowest cost query plan */
   293         -};
   294         -
   295         -/*
   296         -** Return TRUE if the probe cost is less than the baseline cost
   297         -*/
   298         -static int compareCost(const WhereCost *pProbe, const WhereCost *pBaseline){
   299         -  if( pProbe->rCost<pBaseline->rCost ) return 1;
   300         -  if( pProbe->rCost>pBaseline->rCost ) return 0;
   301         -  if( pProbe->plan.nOBSat>pBaseline->plan.nOBSat ) return 1;
   302         -  if( pProbe->plan.nRow<pBaseline->plan.nRow ) return 1;
   303         -  return 0;
          451  +** These are definitions of bits in the WhereLoop.wsFlags field.
          452  +** The particular combination of bits in each WhereLoop help to
          453  +** determine the algorithm that WhereLoop represents.
          454  +*/
          455  +#define WHERE_COLUMN_EQ    0x00000001  /* x=EXPR */
          456  +#define WHERE_COLUMN_RANGE 0x00000002  /* x<EXPR and/or x>EXPR */
          457  +#define WHERE_COLUMN_IN    0x00000004  /* x IN (...) */
          458  +#define WHERE_COLUMN_NULL  0x00000008  /* x IS NULL */
          459  +#define WHERE_CONSTRAINT   0x0000000f  /* Any of the WHERE_COLUMN_xxx values */
          460  +#define WHERE_TOP_LIMIT    0x00000010  /* x<EXPR or x<=EXPR constraint */
          461  +#define WHERE_BTM_LIMIT    0x00000020  /* x>EXPR or x>=EXPR constraint */
          462  +#define WHERE_BOTH_LIMIT   0x00000030  /* Both x>EXPR and x<EXPR */
          463  +#define WHERE_IDX_ONLY     0x00000040  /* Use index only - omit table */
          464  +#define WHERE_IPK          0x00000100  /* x is the INTEGER PRIMARY KEY */
          465  +#define WHERE_INDEXED      0x00000200  /* WhereLoop.u.btree.pIndex is valid */
          466  +#define WHERE_VIRTUALTABLE 0x00000400  /* WhereLoop.u.vtab is valid */
          467  +#define WHERE_IN_ABLE      0x00000800  /* Able to support an IN operator */
          468  +#define WHERE_ONEROW       0x00001000  /* Selects no more than one row */
          469  +#define WHERE_MULTI_OR     0x00002000  /* OR using multiple indices */
          470  +#define WHERE_AUTO_INDEX   0x00004000  /* Uses an ephemeral index */
          471  +
          472  +
          473  +/* Convert a WhereCost value (10 times log2(X)) into its integer value X.
          474  +** A rough approximation is used.  The value returned is not exact.
          475  +*/
          476  +static u64 whereCostToInt(WhereCost x){
          477  +  u64 n;
          478  +  if( x<10 ) return 1;
          479  +  n = x%10;
          480  +  x /= 10;
          481  +  if( n>=5 ) n -= 2;
          482  +  else if( n>=1 ) n -= 1;
          483  +  if( x>=3 ) return (n+8)<<(x-3);
          484  +  return (n+8)>>(3-x);
          485  +}
          486  +
          487  +/*
          488  +** Return the estimated number of output rows from a WHERE clause
          489  +*/
          490  +u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
          491  +  return whereCostToInt(pWInfo->nRowOut);
          492  +}
          493  +
          494  +/*
          495  +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
          496  +** WHERE clause returns outputs for DISTINCT processing.
          497  +*/
          498  +int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
          499  +  return pWInfo->eDistinct;
          500  +}
          501  +
          502  +/*
          503  +** Return TRUE if the WHERE clause returns rows in ORDER BY order.
          504  +** Return FALSE if the output needs to be sorted.
          505  +*/
          506  +int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
          507  +  return pWInfo->bOBSat!=0;
          508  +}
          509  +
          510  +/*
          511  +** Return the VDBE address or label to jump to in order to continue
          512  +** immediately with the next row of a WHERE clause.
          513  +*/
          514  +int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
          515  +  return pWInfo->iContinue;
          516  +}
          517  +
          518  +/*
          519  +** Return the VDBE address or label to jump to in order to break
          520  +** out of a WHERE loop.
          521  +*/
          522  +int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
          523  +  return pWInfo->iBreak;
          524  +}
          525  +
          526  +/*
          527  +** Return TRUE if an UPDATE or DELETE statement can operate directly on
          528  +** the rowids returned by a WHERE clause.  Return FALSE if doing an
          529  +** UPDATE or DELETE might change subsequent WHERE clause results.
          530  +*/
          531  +int sqlite3WhereOkOnePass(WhereInfo *pWInfo){
          532  +  return pWInfo->okOnePass;
          533  +}
          534  +
          535  +/*
          536  +** Move the content of pSrc into pDest
          537  +*/
          538  +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
          539  +  pDest->n = pSrc->n;
          540  +  memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
          541  +}
          542  +
          543  +/*
          544  +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
          545  +**
          546  +** The new entry might overwrite an existing entry, or it might be
          547  +** appended, or it might be discarded.  Do whatever is the right thing
          548  +** so that pSet keeps the N_OR_COST best entries seen so far.
          549  +*/
          550  +static int whereOrInsert(
          551  +  WhereOrSet *pSet,      /* The WhereOrSet to be updated */
          552  +  Bitmask prereq,        /* Prerequisites of the new entry */
          553  +  WhereCost rRun,        /* Run-cost of the new entry */
          554  +  WhereCost nOut         /* Number of outputs for the new entry */
          555  +){
          556  +  u16 i;
          557  +  WhereOrCost *p;
          558  +  for(i=pSet->n, p=pSet->a; i>0; i--, p++){
          559  +    if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
          560  +      goto whereOrInsert_done;
          561  +    }
          562  +    if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
          563  +      return 0;
          564  +    }
          565  +  }
          566  +  if( pSet->n<N_OR_COST ){
          567  +    p = &pSet->a[pSet->n++];
          568  +    p->nOut = nOut;
          569  +  }else{
          570  +    p = pSet->a;
          571  +    for(i=1; i<pSet->n; i++){
          572  +      if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
          573  +    }
          574  +    if( p->rRun<=rRun ) return 0;
          575  +  }
          576  +whereOrInsert_done:
          577  +  p->prereq = prereq;
          578  +  p->rRun = rRun;
          579  +  if( p->nOut>nOut ) p->nOut = nOut;
          580  +  return 1;
   304    581   }
   305    582   
   306    583   /*
   307    584   ** Initialize a preallocated WhereClause structure.
   308    585   */
   309    586   static void whereClauseInit(
   310    587     WhereClause *pWC,        /* The WhereClause to be initialized */
   311         -  Parse *pParse,           /* The parsing context */
   312         -  WhereMaskSet *pMaskSet,  /* Mapping from table cursor numbers to bitmasks */
   313         -  u16 wctrlFlags           /* Might include WHERE_AND_ONLY */
          588  +  WhereInfo *pWInfo        /* The WHERE processing context */
   314    589   ){
   315         -  pWC->pParse = pParse;
   316         -  pWC->pMaskSet = pMaskSet;
          590  +  pWC->pWInfo = pWInfo;
   317    591     pWC->pOuter = 0;
   318    592     pWC->nTerm = 0;
   319    593     pWC->nSlot = ArraySize(pWC->aStatic);
   320    594     pWC->a = pWC->aStatic;
   321         -  pWC->wctrlFlags = wctrlFlags;
   322    595   }
   323    596   
   324    597   /* Forward reference */
   325    598   static void whereClauseClear(WhereClause*);
   326    599   
   327    600   /*
   328    601   ** Deallocate all memory associated with a WhereOrInfo object.
................................................................................
   343    616   /*
   344    617   ** Deallocate a WhereClause structure.  The WhereClause structure
   345    618   ** itself is not freed.  This routine is the inverse of whereClauseInit().
   346    619   */
   347    620   static void whereClauseClear(WhereClause *pWC){
   348    621     int i;
   349    622     WhereTerm *a;
   350         -  sqlite3 *db = pWC->pParse->db;
          623  +  sqlite3 *db = pWC->pWInfo->pParse->db;
   351    624     for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){
   352    625       if( a->wtFlags & TERM_DYNAMIC ){
   353    626         sqlite3ExprDelete(db, a->pExpr);
   354    627       }
   355    628       if( a->wtFlags & TERM_ORINFO ){
   356    629         whereOrInfoDelete(db, a->u.pOrInfo);
   357    630       }else if( a->wtFlags & TERM_ANDINFO ){
................................................................................
   384    657   */
   385    658   static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){
   386    659     WhereTerm *pTerm;
   387    660     int idx;
   388    661     testcase( wtFlags & TERM_VIRTUAL );  /* EV: R-00211-15100 */
   389    662     if( pWC->nTerm>=pWC->nSlot ){
   390    663       WhereTerm *pOld = pWC->a;
   391         -    sqlite3 *db = pWC->pParse->db;
          664  +    sqlite3 *db = pWC->pWInfo->pParse->db;
   392    665       pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 );
   393    666       if( pWC->a==0 ){
   394    667         if( wtFlags & TERM_DYNAMIC ){
   395    668           sqlite3ExprDelete(db, p);
   396    669         }
   397    670         pWC->a = pOld;
   398    671         return 0;
................................................................................
   424    697   ** The original WHERE clause in pExpr is unaltered.  All this routine
   425    698   ** does is make slot[] entries point to substructure within pExpr.
   426    699   **
   427    700   ** In the previous sentence and in the diagram, "slot[]" refers to
   428    701   ** the WhereClause.a[] array.  The slot[] array grows as needed to contain
   429    702   ** all terms of the WHERE clause.
   430    703   */
   431         -static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){
   432         -  pWC->op = (u8)op;
          704  +static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){
          705  +  pWC->op = op;
   433    706     if( pExpr==0 ) return;
   434    707     if( pExpr->op!=op ){
   435    708       whereClauseInsert(pWC, pExpr, 0);
   436    709     }else{
   437    710       whereSplit(pWC, pExpr->pLeft, op);
   438    711       whereSplit(pWC, pExpr->pRight, op);
   439    712     }
   440    713   }
   441    714   
   442    715   /*
   443         -** Initialize an expression mask set (a WhereMaskSet object)
          716  +** Initialize a WhereMaskSet object
   444    717   */
   445         -#define initMaskSet(P)  memset(P, 0, sizeof(*P))
          718  +#define initMaskSet(P)  (P)->n=0
   446    719   
   447    720   /*
   448    721   ** Return the bitmask for the given cursor number.  Return 0 if
   449    722   ** iCursor is not in the set.
   450    723   */
   451    724   static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){
   452    725     int i;
   453    726     assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
   454    727     for(i=0; i<pMaskSet->n; i++){
   455    728       if( pMaskSet->ix[i]==iCursor ){
   456         -      return ((Bitmask)1)<<i;
          729  +      return MASKBIT(i);
   457    730       }
   458    731     }
   459    732     return 0;
   460    733   }
   461    734   
   462    735   /*
   463    736   ** Create a new mask for cursor iCursor.
................................................................................
   469    742   */
   470    743   static void createMask(WhereMaskSet *pMaskSet, int iCursor){
   471    744     assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
   472    745     pMaskSet->ix[pMaskSet->n++] = iCursor;
   473    746   }
   474    747   
   475    748   /*
   476         -** This routine walks (recursively) an expression tree and generates
          749  +** These routine walk (recursively) an expression tree and generates
   477    750   ** a bitmask indicating which tables are used in that expression
   478    751   ** tree.
   479         -**
   480         -** In order for this routine to work, the calling function must have
   481         -** previously invoked sqlite3ResolveExprNames() on the expression.  See
   482         -** the header comment on that routine for additional information.
   483         -** The sqlite3ResolveExprNames() routines looks for column names and
   484         -** sets their opcodes to TK_COLUMN and their Expr.iTable fields to
   485         -** the VDBE cursor number of the table.  This routine just has to
   486         -** translate the cursor numbers into bitmask values and OR all
   487         -** the bitmasks together.
   488    752   */
   489    753   static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*);
   490    754   static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*);
   491    755   static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){
   492    756     Bitmask mask = 0;
   493    757     if( p==0 ) return 0;
   494    758     if( p->op==TK_COLUMN ){
................................................................................
   534    798     }
   535    799     return mask;
   536    800   }
   537    801   
   538    802   /*
   539    803   ** Return TRUE if the given operator is one of the operators that is
   540    804   ** allowed for an indexable WHERE clause term.  The allowed operators are
   541         -** "=", "<", ">", "<=", ">=", and "IN".
          805  +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL"
   542    806   **
   543    807   ** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be
   544    808   ** of one of the following forms: column = expression column > expression
   545    809   ** column >= expression column < expression column <= expression
   546    810   ** expression = column expression > column expression >= column
   547    811   ** expression < column expression <= column column IN
   548    812   ** (expression-list) column IN (subquery) column IS NULL
................................................................................
   561    825   #define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}
   562    826   
   563    827   /*
   564    828   ** Commute a comparison operator.  Expressions of the form "X op Y"
   565    829   ** are converted into "Y op X".
   566    830   **
   567    831   ** If left/right precedence rules come into play when determining the
   568         -** collating
   569         -** side of the comparison, it remains associated with the same side after
   570         -** the commutation. So "Y collate NOCASE op X" becomes 
   571         -** "X op Y". This is because any collation sequence on
          832  +** collating sequence, then COLLATE operators are adjusted to ensure
          833  +** that the collating sequence does not change.  For example:
          834  +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on
   572    835   ** the left hand side of a comparison overrides any collation sequence 
   573    836   ** attached to the right. For the same reason the EP_Collate flag
   574    837   ** is not commuted.
   575    838   */
   576    839   static void exprCommute(Parse *pParse, Expr *pExpr){
   577    840     u16 expRight = (pExpr->pRight->flags & EP_Collate);
   578    841     u16 expLeft = (pExpr->pLeft->flags & EP_Collate);
................................................................................
   620    883     assert( op!=TK_EQ || c==WO_EQ );
   621    884     assert( op!=TK_LT || c==WO_LT );
   622    885     assert( op!=TK_LE || c==WO_LE );
   623    886     assert( op!=TK_GT || c==WO_GT );
   624    887     assert( op!=TK_GE || c==WO_GE );
   625    888     return c;
   626    889   }
          890  +
          891  +/*
          892  +** Advance to the next WhereTerm that matches according to the criteria
          893  +** established when the pScan object was initialized by whereScanInit().
          894  +** Return NULL if there are no more matching WhereTerms.
          895  +*/
          896  +static WhereTerm *whereScanNext(WhereScan *pScan){
          897  +  int iCur;            /* The cursor on the LHS of the term */
          898  +  int iColumn;         /* The column on the LHS of the term.  -1 for IPK */
          899  +  Expr *pX;            /* An expression being tested */
          900  +  WhereClause *pWC;    /* Shorthand for pScan->pWC */
          901  +  WhereTerm *pTerm;    /* The term being tested */
          902  +  int k = pScan->k;    /* Where to start scanning */
          903  +
          904  +  while( pScan->iEquiv<=pScan->nEquiv ){
          905  +    iCur = pScan->aEquiv[pScan->iEquiv-2];
          906  +    iColumn = pScan->aEquiv[pScan->iEquiv-1];
          907  +    while( (pWC = pScan->pWC)!=0 ){
          908  +      for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
          909  +        if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn ){
          910  +          if( (pTerm->eOperator & WO_EQUIV)!=0
          911  +           && pScan->nEquiv<ArraySize(pScan->aEquiv)
          912  +          ){
          913  +            int j;
          914  +            pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight);
          915  +            assert( pX->op==TK_COLUMN );
          916  +            for(j=0; j<pScan->nEquiv; j+=2){
          917  +              if( pScan->aEquiv[j]==pX->iTable
          918  +               && pScan->aEquiv[j+1]==pX->iColumn ){
          919  +                  break;
          920  +              }
          921  +            }
          922  +            if( j==pScan->nEquiv ){
          923  +              pScan->aEquiv[j] = pX->iTable;
          924  +              pScan->aEquiv[j+1] = pX->iColumn;
          925  +              pScan->nEquiv += 2;
          926  +            }
          927  +          }
          928  +          if( (pTerm->eOperator & pScan->opMask)!=0 ){
          929  +            /* Verify the affinity and collating sequence match */
          930  +            if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
          931  +              CollSeq *pColl;
          932  +              Parse *pParse = pWC->pWInfo->pParse;
          933  +              pX = pTerm->pExpr;
          934  +              if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
          935  +                continue;
          936  +              }
          937  +              assert(pX->pLeft);
          938  +              pColl = sqlite3BinaryCompareCollSeq(pParse,
          939  +                                                  pX->pLeft, pX->pRight);
          940  +              if( pColl==0 ) pColl = pParse->db->pDfltColl;
          941  +              if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){
          942  +                continue;
          943  +              }
          944  +            }
          945  +            if( (pTerm->eOperator & WO_EQ)!=0
          946  +             && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN
          947  +             && pX->iTable==pScan->aEquiv[0]
          948  +             && pX->iColumn==pScan->aEquiv[1]
          949  +            ){
          950  +              continue;
          951  +            }
          952  +            pScan->k = k+1;
          953  +            return pTerm;
          954  +          }
          955  +        }
          956  +      }
          957  +      pScan->pWC = pScan->pWC->pOuter;
          958  +      k = 0;
          959  +    }
          960  +    pScan->pWC = pScan->pOrigWC;
          961  +    k = 0;
          962  +    pScan->iEquiv += 2;
          963  +  }
          964  +  return 0;
          965  +}
          966  +
          967  +/*
          968  +** Initialize a WHERE clause scanner object.  Return a pointer to the
          969  +** first match.  Return NULL if there are no matches.
          970  +**
          971  +** The scanner will be searching the WHERE clause pWC.  It will look
          972  +** for terms of the form "X <op> <expr>" where X is column iColumn of table
          973  +** iCur.  The <op> must be one of the operators described by opMask.
          974  +**
          975  +** If the search is for X and the WHERE clause contains terms of the
          976  +** form X=Y then this routine might also return terms of the form
          977  +** "Y <op> <expr>".  The number of levels of transitivity is limited,
          978  +** but is enough to handle most commonly occurring SQL statements.
          979  +**
          980  +** If X is not the INTEGER PRIMARY KEY then X must be compatible with
          981  +** index pIdx.
          982  +*/
          983  +static WhereTerm *whereScanInit(
          984  +  WhereScan *pScan,       /* The WhereScan object being initialized */
          985  +  WhereClause *pWC,       /* The WHERE clause to be scanned */
          986  +  int iCur,               /* Cursor to scan for */
          987  +  int iColumn,            /* Column to scan for */
          988  +  u32 opMask,             /* Operator(s) to scan for */
          989  +  Index *pIdx             /* Must be compatible with this index */
          990  +){
          991  +  int j;
          992  +
          993  +  /* memset(pScan, 0, sizeof(*pScan)); */
          994  +  pScan->pOrigWC = pWC;
          995  +  pScan->pWC = pWC;
          996  +  if( pIdx && iColumn>=0 ){
          997  +    pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
          998  +    for(j=0; pIdx->aiColumn[j]!=iColumn; j++){
          999  +      if( NEVER(j>=pIdx->nColumn) ) return 0;
         1000  +    }
         1001  +    pScan->zCollName = pIdx->azColl[j];
         1002  +  }else{
         1003  +    pScan->idxaff = 0;
         1004  +    pScan->zCollName = 0;
         1005  +  }
         1006  +  pScan->opMask = opMask;
         1007  +  pScan->k = 0;
         1008  +  pScan->aEquiv[0] = iCur;
         1009  +  pScan->aEquiv[1] = iColumn;
         1010  +  pScan->nEquiv = 2;
         1011  +  pScan->iEquiv = 2;
         1012  +  return whereScanNext(pScan);
         1013  +}
   627   1014   
   628   1015   /*
   629   1016   ** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
   630   1017   ** where X is a reference to the iColumn of table iCur and <op> is one of
   631   1018   ** the WO_xx operator codes specified by the op parameter.
   632   1019   ** Return a pointer to the term.  Return 0 if not found.
   633   1020   **
................................................................................
   652   1039     WhereClause *pWC,     /* The WHERE clause to be searched */
   653   1040     int iCur,             /* Cursor number of LHS */
   654   1041     int iColumn,          /* Column number of LHS */
   655   1042     Bitmask notReady,     /* RHS must not overlap with this mask */
   656   1043     u32 op,               /* Mask of WO_xx values describing operator */
   657   1044     Index *pIdx           /* Must be compatible with this index, if not NULL */
   658   1045   ){
   659         -  WhereTerm *pTerm;            /* Term being examined as possible result */
   660         -  WhereTerm *pResult = 0;      /* The answer to return */
   661         -  WhereClause *pWCOrig = pWC;  /* Original pWC value */
   662         -  int j, k;                    /* Loop counters */
   663         -  Expr *pX;                /* Pointer to an expression */
   664         -  Parse *pParse;           /* Parsing context */
   665         -  int iOrigCol = iColumn;  /* Original value of iColumn */
   666         -  int nEquiv = 2;          /* Number of entires in aEquiv[] */
   667         -  int iEquiv = 2;          /* Number of entries of aEquiv[] processed so far */
   668         -  int aEquiv[22];          /* iCur,iColumn and up to 10 other equivalents */
   669         -
   670         -  assert( iCur>=0 );
   671         -  aEquiv[0] = iCur;
   672         -  aEquiv[1] = iColumn;
   673         -  for(;;){
   674         -    for(pWC=pWCOrig; pWC; pWC=pWC->pOuter){
   675         -      for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){
   676         -        if( pTerm->leftCursor==iCur
   677         -          && pTerm->u.leftColumn==iColumn
   678         -        ){
   679         -          if( (pTerm->prereqRight & notReady)==0
   680         -           && (pTerm->eOperator & op & WO_ALL)!=0
   681         -          ){
   682         -            if( iOrigCol>=0 && pIdx && (pTerm->eOperator & WO_ISNULL)==0 ){
   683         -              CollSeq *pColl;
   684         -              char idxaff;
   685         -      
   686         -              pX = pTerm->pExpr;
   687         -              pParse = pWC->pParse;
   688         -              idxaff = pIdx->pTable->aCol[iOrigCol].affinity;
   689         -              if( !sqlite3IndexAffinityOk(pX, idxaff) ){
   690         -                continue;
   691         -              }
   692         -      
   693         -              /* Figure out the collation sequence required from an index for
   694         -              ** it to be useful for optimising expression pX. Store this
   695         -              ** value in variable pColl.
   696         -              */
   697         -              assert(pX->pLeft);
   698         -              pColl = sqlite3BinaryCompareCollSeq(pParse,pX->pLeft,pX->pRight);
   699         -              if( pColl==0 ) pColl = pParse->db->pDfltColl;
   700         -      
   701         -              for(j=0; pIdx->aiColumn[j]!=iOrigCol; j++){
   702         -                if( NEVER(j>=pIdx->nColumn) ) return 0;
   703         -              }
   704         -              if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ){
   705         -                continue;
   706         -              }
   707         -            }
   708         -            if( pTerm->prereqRight==0 && (pTerm->eOperator&WO_EQ)!=0 ){
   709         -              pResult = pTerm;
   710         -              goto findTerm_success;
   711         -            }else if( pResult==0 ){
   712         -              pResult = pTerm;
   713         -            }
   714         -          }
   715         -          if( (pTerm->eOperator & WO_EQUIV)!=0
   716         -           && nEquiv<ArraySize(aEquiv)
   717         -          ){
   718         -            pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight);
   719         -            assert( pX->op==TK_COLUMN );
   720         -            for(j=0; j<nEquiv; j+=2){
   721         -              if( aEquiv[j]==pX->iTable && aEquiv[j+1]==pX->iColumn ) break;
   722         -            }
   723         -            if( j==nEquiv ){
   724         -              aEquiv[j] = pX->iTable;
   725         -              aEquiv[j+1] = pX->iColumn;
   726         -              nEquiv += 2;
   727         -            }
   728         -          }
   729         -        }
   730         -      }
   731         -    }
   732         -    if( iEquiv>=nEquiv ) break;
   733         -    iCur = aEquiv[iEquiv++];
   734         -    iColumn = aEquiv[iEquiv++];
   735         -  }
   736         -findTerm_success:
         1046  +  WhereTerm *pResult = 0;
         1047  +  WhereTerm *p;
         1048  +  WhereScan scan;
         1049  +
         1050  +  p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
         1051  +  while( p ){
         1052  +    if( (p->prereqRight & notReady)==0 ){
         1053  +      if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){
         1054  +        return p;
         1055  +      }
         1056  +      if( pResult==0 ) pResult = p;
         1057  +    }
         1058  +    p = whereScanNext(&scan);
         1059  +  }
   737   1060     return pResult;
   738   1061   }
   739   1062   
   740   1063   /* Forward reference */
   741   1064   static void exprAnalyze(SrcList*, WhereClause*, int);
   742   1065   
   743   1066   /*
   744   1067   ** Call exprAnalyze on all terms in a WHERE clause.  
   745         -**
   746         -**
   747   1068   */
   748   1069   static void exprAnalyzeAll(
   749   1070     SrcList *pTabList,       /* the FROM clause */
   750   1071     WhereClause *pWC         /* the WHERE clause to be analyzed */
   751   1072   ){
   752   1073     int i;
   753   1074     for(i=pWC->nTerm-1; i>=0; i--){
................................................................................
   971   1292   ** zero.  This term is not useful for search.
   972   1293   */
   973   1294   static void exprAnalyzeOrTerm(
   974   1295     SrcList *pSrc,            /* the FROM clause */
   975   1296     WhereClause *pWC,         /* the complete WHERE clause */
   976   1297     int idxTerm               /* Index of the OR-term to be analyzed */
   977   1298   ){
   978         -  Parse *pParse = pWC->pParse;            /* Parser context */
         1299  +  WhereInfo *pWInfo = pWC->pWInfo;        /* WHERE clause processing context */
         1300  +  Parse *pParse = pWInfo->pParse;         /* Parser context */
   979   1301     sqlite3 *db = pParse->db;               /* Database connection */
   980   1302     WhereTerm *pTerm = &pWC->a[idxTerm];    /* The term to be analyzed */
   981   1303     Expr *pExpr = pTerm->pExpr;             /* The expression of the term */
   982         -  WhereMaskSet *pMaskSet = pWC->pMaskSet; /* Table use masks */
   983   1304     int i;                                  /* Loop counters */
   984   1305     WhereClause *pOrWc;       /* Breakup of pTerm into subterms */
   985   1306     WhereTerm *pOrTerm;       /* A Sub-term within the pOrWc */
   986   1307     WhereOrInfo *pOrInfo;     /* Additional information associated with pTerm */
   987   1308     Bitmask chngToIN;         /* Tables that might satisfy case 1 */
   988   1309     Bitmask indexable;        /* Tables that are indexable, satisfying case 2 */
   989   1310   
................................................................................
   994   1315     */
   995   1316     assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 );
   996   1317     assert( pExpr->op==TK_OR );
   997   1318     pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo));
   998   1319     if( pOrInfo==0 ) return;
   999   1320     pTerm->wtFlags |= TERM_ORINFO;
  1000   1321     pOrWc = &pOrInfo->wc;
  1001         -  whereClauseInit(pOrWc, pWC->pParse, pMaskSet, pWC->wctrlFlags);
         1322  +  whereClauseInit(pOrWc, pWInfo);
  1002   1323     whereSplit(pOrWc, pExpr, TK_OR);
  1003   1324     exprAnalyzeAll(pSrc, pOrWc);
  1004   1325     if( db->mallocFailed ) return;
  1005   1326     assert( pOrWc->nTerm>=2 );
  1006   1327   
  1007   1328     /*
  1008   1329     ** Compute the set of tables that might satisfy cases 1 or 2.
................................................................................
  1020   1341           WhereTerm *pAndTerm;
  1021   1342           int j;
  1022   1343           Bitmask b = 0;
  1023   1344           pOrTerm->u.pAndInfo = pAndInfo;
  1024   1345           pOrTerm->wtFlags |= TERM_ANDINFO;
  1025   1346           pOrTerm->eOperator = WO_AND;
  1026   1347           pAndWC = &pAndInfo->wc;
  1027         -        whereClauseInit(pAndWC, pWC->pParse, pMaskSet, pWC->wctrlFlags);
         1348  +        whereClauseInit(pAndWC, pWC->pWInfo);
  1028   1349           whereSplit(pAndWC, pOrTerm->pExpr, TK_AND);
  1029   1350           exprAnalyzeAll(pSrc, pAndWC);
  1030   1351           pAndWC->pOuter = pWC;
  1031   1352           testcase( db->mallocFailed );
  1032   1353           if( !db->mallocFailed ){
  1033   1354             for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){
  1034   1355               assert( pAndTerm->pExpr );
  1035   1356               if( allowedOp(pAndTerm->pExpr->op) ){
  1036         -              b |= getMask(pMaskSet, pAndTerm->leftCursor);
         1357  +              b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
  1037   1358               }
  1038   1359             }
  1039   1360           }
  1040   1361           indexable &= b;
  1041   1362         }
  1042   1363       }else if( pOrTerm->wtFlags & TERM_COPIED ){
  1043   1364         /* Skip this term for now.  We revisit it when we process the
  1044   1365         ** corresponding TERM_VIRTUAL term */
  1045   1366       }else{
  1046   1367         Bitmask b;
  1047         -      b = getMask(pMaskSet, pOrTerm->leftCursor);
         1368  +      b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor);
  1048   1369         if( pOrTerm->wtFlags & TERM_VIRTUAL ){
  1049   1370           WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent];
  1050         -        b |= getMask(pMaskSet, pOther->leftCursor);
         1371  +        b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor);
  1051   1372         }
  1052   1373         indexable &= b;
  1053   1374         if( (pOrTerm->eOperator & WO_EQ)==0 ){
  1054   1375           chngToIN = 0;
  1055   1376         }else{
  1056   1377           chngToIN &= b;
  1057   1378         }
................................................................................
  1105   1426           pOrTerm->wtFlags &= ~TERM_OR_OK;
  1106   1427           if( pOrTerm->leftCursor==iCursor ){
  1107   1428             /* This is the 2-bit case and we are on the second iteration and
  1108   1429             ** current term is from the first iteration.  So skip this term. */
  1109   1430             assert( j==1 );
  1110   1431             continue;
  1111   1432           }
  1112         -        if( (chngToIN & getMask(pMaskSet, pOrTerm->leftCursor))==0 ){
         1433  +        if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){
  1113   1434             /* This term must be of the form t1.a==t2.b where t2 is in the
  1114   1435             ** chngToIN set but t1 is not.  This term will be either preceeded
  1115   1436             ** or follwed by an inverted copy (t2.b==t1.a).  Skip this term 
  1116   1437             ** and use its inversion. */
  1117   1438             testcase( pOrTerm->wtFlags & TERM_COPIED );
  1118   1439             testcase( pOrTerm->wtFlags & TERM_VIRTUAL );
  1119   1440             assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) );
................................................................................
  1124   1445           break;
  1125   1446         }
  1126   1447         if( i<0 ){
  1127   1448           /* No candidate table+column was found.  This can only occur
  1128   1449           ** on the second iteration */
  1129   1450           assert( j==1 );
  1130   1451           assert( IsPowerOfTwo(chngToIN) );
  1131         -        assert( chngToIN==getMask(pMaskSet, iCursor) );
         1452  +        assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) );
  1132   1453           break;
  1133   1454         }
  1134   1455         testcase( j==1 );
  1135   1456   
  1136   1457         /* We have found a candidate table and column.  Check to see if that
  1137   1458         ** table and column is common to every term in the OR clause */
  1138   1459         okToChngToIN = 1;
................................................................................
  1173   1494   
  1174   1495         for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){
  1175   1496           if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue;
  1176   1497           assert( pOrTerm->eOperator & WO_EQ );
  1177   1498           assert( pOrTerm->leftCursor==iCursor );
  1178   1499           assert( pOrTerm->u.leftColumn==iColumn );
  1179   1500           pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0);
  1180         -        pList = sqlite3ExprListAppend(pWC->pParse, pList, pDup);
         1501  +        pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup);
  1181   1502           pLeft = pOrTerm->pExpr->pLeft;
  1182   1503         }
  1183   1504         assert( pLeft!=0 );
  1184   1505         pDup = sqlite3ExprDup(db, pLeft, 0);
  1185   1506         pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0);
  1186   1507         if( pNew ){
  1187   1508           int idxNew;
................................................................................
  1222   1543   ** and the copy has idxParent set to the index of the original term.
  1223   1544   */
  1224   1545   static void exprAnalyze(
  1225   1546     SrcList *pSrc,            /* the FROM clause */
  1226   1547     WhereClause *pWC,         /* the WHERE clause */
  1227   1548     int idxTerm               /* Index of the term to be analyzed */
  1228   1549   ){
         1550  +  WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */
  1229   1551     WhereTerm *pTerm;                /* The term to be analyzed */
  1230   1552     WhereMaskSet *pMaskSet;          /* Set of table index masks */
  1231   1553     Expr *pExpr;                     /* The expression to be analyzed */
  1232   1554     Bitmask prereqLeft;              /* Prerequesites of the pExpr->pLeft */
  1233   1555     Bitmask prereqAll;               /* Prerequesites of pExpr */
  1234   1556     Bitmask extraRight = 0;          /* Extra dependencies on LEFT JOIN */
  1235   1557     Expr *pStr1 = 0;                 /* RHS of LIKE/GLOB operator */
  1236   1558     int isComplete = 0;              /* RHS of LIKE/GLOB ends with wildcard */
  1237   1559     int noCase = 0;                  /* LIKE/GLOB distinguishes case */
  1238   1560     int op;                          /* Top-level operator.  pExpr->op */
  1239         -  Parse *pParse = pWC->pParse;     /* Parsing context */
         1561  +  Parse *pParse = pWInfo->pParse;  /* Parsing context */
  1240   1562     sqlite3 *db = pParse->db;        /* Database connection */
  1241   1563   
  1242   1564     if( db->mallocFailed ){
  1243   1565       return;
  1244   1566     }
  1245   1567     pTerm = &pWC->a[idxTerm];
  1246         -  pMaskSet = pWC->pMaskSet;
         1568  +  pMaskSet = &pWInfo->sMaskSet;
  1247   1569     pExpr = pTerm->pExpr;
  1248   1570     assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE );
  1249   1571     prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft);
  1250   1572     op = pExpr->op;
  1251   1573     if( op==TK_IN ){
  1252   1574       assert( pExpr->pRight==0 );
  1253   1575       if( ExprHasProperty(pExpr, EP_xIsSelect) ){
................................................................................
  1483   1805     ** TERM_VNULL tag will suppress the not-null check at the beginning
  1484   1806     ** of the loop.  Without the TERM_VNULL flag, the not-null check at
  1485   1807     ** the start of the loop will prevent any results from being returned.
  1486   1808     */
  1487   1809     if( pExpr->op==TK_NOTNULL
  1488   1810      && pExpr->pLeft->op==TK_COLUMN
  1489   1811      && pExpr->pLeft->iColumn>=0
         1812  +   && OptimizationEnabled(db, SQLITE_Stat3)
  1490   1813     ){
  1491   1814       Expr *pNewExpr;
  1492   1815       Expr *pLeft = pExpr->pLeft;
  1493   1816       int idxNew;
  1494   1817       WhereTerm *pNewTerm;
  1495   1818   
  1496   1819       pNewExpr = sqlite3PExpr(pParse, TK_GT,
................................................................................
  1517   1840     /* Prevent ON clause terms of a LEFT JOIN from being used to drive
  1518   1841     ** an index for tables to the left of the join.
  1519   1842     */
  1520   1843     pTerm->prereqRight |= extraRight;
  1521   1844   }
  1522   1845   
  1523   1846   /*
  1524         -** This function searches the expression list passed as the second argument
  1525         -** for an expression of type TK_COLUMN that refers to the same column and
  1526         -** uses the same collation sequence as the iCol'th column of index pIdx.
  1527         -** Argument iBase is the cursor number used for the table that pIdx refers
  1528         -** to.
         1847  +** This function searches pList for a entry that matches the iCol-th column
         1848  +** of index pIdx.
  1529   1849   **
  1530   1850   ** If such an expression is found, its index in pList->a[] is returned. If
  1531   1851   ** no expression is found, -1 is returned.
  1532   1852   */
  1533   1853   static int findIndexCol(
  1534   1854     Parse *pParse,                  /* Parse context */
  1535   1855     ExprList *pList,                /* Expression list to search */
................................................................................
  1551   1871           return i;
  1552   1872         }
  1553   1873       }
  1554   1874     }
  1555   1875   
  1556   1876     return -1;
  1557   1877   }
  1558         -
  1559         -/*
  1560         -** This routine determines if pIdx can be used to assist in processing a
  1561         -** DISTINCT qualifier. In other words, it tests whether or not using this
  1562         -** index for the outer loop guarantees that rows with equal values for
  1563         -** all expressions in the pDistinct list are delivered grouped together.
  1564         -**
  1565         -** For example, the query 
  1566         -**
  1567         -**   SELECT DISTINCT a, b, c FROM tbl WHERE a = ?
  1568         -**
  1569         -** can benefit from any index on columns "b" and "c".
  1570         -*/
  1571         -static int isDistinctIndex(
  1572         -  Parse *pParse,                  /* Parsing context */
  1573         -  WhereClause *pWC,               /* The WHERE clause */
  1574         -  Index *pIdx,                    /* The index being considered */
  1575         -  int base,                       /* Cursor number for the table pIdx is on */
  1576         -  ExprList *pDistinct,            /* The DISTINCT expressions */
  1577         -  int nEqCol                      /* Number of index columns with == */
  1578         -){
  1579         -  Bitmask mask = 0;               /* Mask of unaccounted for pDistinct exprs */
  1580         -  int i;                          /* Iterator variable */
  1581         -
  1582         -  assert( pDistinct!=0 );
  1583         -  if( pIdx->zName==0 || pDistinct->nExpr>=BMS ) return 0;
  1584         -  testcase( pDistinct->nExpr==BMS-1 );
  1585         -
  1586         -  /* Loop through all the expressions in the distinct list. If any of them
  1587         -  ** are not simple column references, return early. Otherwise, test if the
  1588         -  ** WHERE clause contains a "col=X" clause. If it does, the expression
  1589         -  ** can be ignored. If it does not, and the column does not belong to the
  1590         -  ** same table as index pIdx, return early. Finally, if there is no
  1591         -  ** matching "col=X" expression and the column is on the same table as pIdx,
  1592         -  ** set the corresponding bit in variable mask.
  1593         -  */
  1594         -  for(i=0; i<pDistinct->nExpr; i++){
  1595         -    WhereTerm *pTerm;
  1596         -    Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr);
  1597         -    if( p->op!=TK_COLUMN ) return 0;
  1598         -    pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0);
  1599         -    if( pTerm ){
  1600         -      Expr *pX = pTerm->pExpr;
  1601         -      CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
  1602         -      CollSeq *p2 = sqlite3ExprCollSeq(pParse, p);
  1603         -      if( p1==p2 ) continue;
  1604         -    }
  1605         -    if( p->iTable!=base ) return 0;
  1606         -    mask |= (((Bitmask)1) << i);
  1607         -  }
  1608         -
  1609         -  for(i=nEqCol; mask && i<pIdx->nColumn; i++){
  1610         -    int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i);
  1611         -    if( iExpr<0 ) break;
  1612         -    mask &= ~(((Bitmask)1) << iExpr);
  1613         -  }
  1614         -
  1615         -  return (mask==0);
  1616         -}
  1617         -
  1618   1878   
  1619   1879   /*
  1620   1880   ** Return true if the DISTINCT expression-list passed as the third argument
  1621         -** is redundant. A DISTINCT list is redundant if the database contains a
  1622         -** UNIQUE index that guarantees that the result of the query will be distinct
  1623         -** anyway.
         1881  +** is redundant.
         1882  +**
         1883  +** A DISTINCT list is redundant if the database contains some subset of
         1884  +** columns that are unique and non-null.
  1624   1885   */
  1625   1886   static int isDistinctRedundant(
  1626         -  Parse *pParse,
  1627         -  SrcList *pTabList,
  1628         -  WhereClause *pWC,
  1629         -  ExprList *pDistinct
         1887  +  Parse *pParse,            /* Parsing context */
         1888  +  SrcList *pTabList,        /* The FROM clause */
         1889  +  WhereClause *pWC,         /* The WHERE clause */
         1890  +  ExprList *pDistinct       /* The result set that needs to be DISTINCT */
  1630   1891   ){
  1631   1892     Table *pTab;
  1632   1893     Index *pIdx;
  1633   1894     int i;                          
  1634   1895     int iBase;
  1635   1896   
  1636   1897     /* If there is more than one table or sub-select in the FROM clause of
................................................................................
  1678   1939         return 1;
  1679   1940       }
  1680   1941     }
  1681   1942   
  1682   1943     return 0;
  1683   1944   }
  1684   1945   
  1685         -/*
  1686         -** Prepare a crude estimate of the logarithm of the input value.
  1687         -** The results need not be exact.  This is only used for estimating
  1688         -** the total cost of performing operations with O(logN) or O(NlogN)
  1689         -** complexity.  Because N is just a guess, it is no great tragedy if
  1690         -** logN is a little off.
  1691         -*/
  1692         -static double estLog(double N){
  1693         -  double logN = 1;
  1694         -  double x = 10;
  1695         -  while( N>x ){
  1696         -    logN += 1;
  1697         -    x *= 10;
  1698         -  }
  1699         -  return logN;
         1946  +/* 
         1947  +** The (an approximate) sum of two WhereCosts.  This computation is
         1948  +** not a simple "+" operator because WhereCost is stored as a logarithmic
         1949  +** value.
         1950  +** 
         1951  +*/
         1952  +static WhereCost whereCostAdd(WhereCost a, WhereCost b){
         1953  +  static const unsigned char x[] = {
         1954  +     10, 10,                         /* 0,1 */
         1955  +      9, 9,                          /* 2,3 */
         1956  +      8, 8,                          /* 4,5 */
         1957  +      7, 7, 7,                       /* 6,7,8 */
         1958  +      6, 6, 6,                       /* 9,10,11 */
         1959  +      5, 5, 5,                       /* 12-14 */
         1960  +      4, 4, 4, 4,                    /* 15-18 */
         1961  +      3, 3, 3, 3, 3, 3,              /* 19-24 */
         1962  +      2, 2, 2, 2, 2, 2, 2,           /* 25-31 */
         1963  +  };
         1964  +  if( a>=b ){
         1965  +    if( a>b+49 ) return a;
         1966  +    if( a>b+31 ) return a+1;
         1967  +    return a+x[a-b];
         1968  +  }else{
         1969  +    if( b>a+49 ) return b;
         1970  +    if( b>a+31 ) return b+1;
         1971  +    return b+x[b-a];
         1972  +  }
         1973  +}
         1974  +
         1975  +/*
         1976  +** Convert an integer into a WhereCost.  In other words, compute a
         1977  +** good approximatation for 10*log2(x).
         1978  +*/
         1979  +static WhereCost whereCost(tRowcnt x){
         1980  +  static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 };
         1981  +  WhereCost y = 40;
         1982  +  if( x<8 ){
         1983  +    if( x<2 ) return 0;
         1984  +    while( x<8 ){  y -= 10; x <<= 1; }
         1985  +  }else{
         1986  +    while( x>255 ){ y += 40; x >>= 4; }
         1987  +    while( x>15 ){  y += 10; x >>= 1; }
         1988  +  }
         1989  +  return a[x&7] + y - 10;
         1990  +}
         1991  +
         1992  +#ifndef SQLITE_OMIT_VIRTUALTABLE
         1993  +/*
         1994  +** Convert a double (as received from xBestIndex of a virtual table)
         1995  +** into a WhereCost.  In other words, compute an approximation for
         1996  +** 10*log2(x).
         1997  +*/
         1998  +static WhereCost whereCostFromDouble(double x){
         1999  +  u64 a;
         2000  +  WhereCost e;
         2001  +  assert( sizeof(x)==8 && sizeof(a)==8 );
         2002  +  if( x<=1 ) return 0;
         2003  +  if( x<=2000000000 ) return whereCost((tRowcnt)x);
         2004  +  memcpy(&a, &x, 8);
         2005  +  e = (a>>52) - 1022;
         2006  +  return e*10;
         2007  +}
         2008  +#endif /* SQLITE_OMIT_VIRTUALTABLE */
         2009  +
         2010  +/*
         2011  +** Estimate the logarithm of the input value to base 2.
         2012  +*/
         2013  +static WhereCost estLog(WhereCost N){
         2014  +  WhereCost x = whereCost(N);
         2015  +  return x>33 ? x - 33 : 0;
  1700   2016   }
  1701   2017   
  1702   2018   /*
  1703   2019   ** Two routines for printing the content of an sqlite3_index_info
  1704   2020   ** structure.  Used for testing and debugging only.  If neither
  1705   2021   ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
  1706   2022   ** are no-ops.
  1707   2023   */
  1708         -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_DEBUG)
         2024  +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
  1709   2025   static void TRACE_IDX_INPUTS(sqlite3_index_info *p){
  1710   2026     int i;
  1711   2027     if( !sqlite3WhereTrace ) return;
  1712   2028     for(i=0; i<p->nConstraint; i++){
  1713   2029       sqlite3DebugPrintf("  constraint[%d]: col=%d termid=%d op=%d usabled=%d\n",
  1714   2030          i,
  1715   2031          p->aConstraint[i].iColumn,
................................................................................
  1739   2055     sqlite3DebugPrintf("  estimatedCost=%g\n", p->estimatedCost);
  1740   2056   }
  1741   2057   #else
  1742   2058   #define TRACE_IDX_INPUTS(A)
  1743   2059   #define TRACE_IDX_OUTPUTS(A)
  1744   2060   #endif
  1745   2061   
  1746         -/* 
  1747         -** Required because bestIndex() is called by bestOrClauseIndex() 
  1748         -*/
  1749         -static void bestIndex(WhereBestIdx*);
  1750         -
  1751         -/*
  1752         -** This routine attempts to find an scanning strategy that can be used 
  1753         -** to optimize an 'OR' expression that is part of a WHERE clause. 
  1754         -**
  1755         -** The table associated with FROM clause term pSrc may be either a
  1756         -** regular B-Tree table or a virtual table.
  1757         -*/
  1758         -static void bestOrClauseIndex(WhereBestIdx *p){
  1759         -#ifndef SQLITE_OMIT_OR_OPTIMIZATION
  1760         -  WhereClause *pWC = p->pWC;           /* The WHERE clause */
  1761         -  struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */
  1762         -  const int iCur = pSrc->iCursor;      /* The cursor of the table  */
  1763         -  const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur);  /* Bitmask for pSrc */
  1764         -  WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm];        /* End of pWC->a[] */
  1765         -  WhereTerm *pTerm;                    /* A single term of the WHERE clause */
  1766         -
  1767         -  /* The OR-clause optimization is disallowed if the INDEXED BY or
  1768         -  ** NOT INDEXED clauses are used or if the WHERE_AND_ONLY bit is set. */
  1769         -  if( pSrc->notIndexed || pSrc->pIndex!=0 ){
  1770         -    return;
  1771         -  }
  1772         -  if( pWC->wctrlFlags & WHERE_AND_ONLY ){
  1773         -    return;
  1774         -  }
  1775         -
  1776         -  /* Search the WHERE clause terms for a usable WO_OR term. */
  1777         -  for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
  1778         -    if( (pTerm->eOperator & WO_OR)!=0
  1779         -     && ((pTerm->prereqAll & ~maskSrc) & p->notReady)==0
  1780         -     && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 
  1781         -    ){
  1782         -      WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
  1783         -      WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
  1784         -      WhereTerm *pOrTerm;
  1785         -      int flags = WHERE_MULTI_OR;
  1786         -      double rTotal = 0;
  1787         -      double nRow = 0;
  1788         -      Bitmask used = 0;
  1789         -      WhereBestIdx sBOI;
  1790         -
  1791         -      sBOI = *p;
  1792         -      sBOI.pOrderBy = 0;
  1793         -      sBOI.pDistinct = 0;
  1794         -      sBOI.ppIdxInfo = 0;
  1795         -      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
  1796         -        WHERETRACE(("... Multi-index OR testing for term %d of %d....\n", 
  1797         -          (pOrTerm - pOrWC->a), (pTerm - pWC->a)
  1798         -        ));
  1799         -        if( (pOrTerm->eOperator& WO_AND)!=0 ){
  1800         -          sBOI.pWC = &pOrTerm->u.pAndInfo->wc;
  1801         -          bestIndex(&sBOI);
  1802         -        }else if( pOrTerm->leftCursor==iCur ){
  1803         -          WhereClause tempWC;
  1804         -          tempWC.pParse = pWC->pParse;
  1805         -          tempWC.pMaskSet = pWC->pMaskSet;
  1806         -          tempWC.pOuter = pWC;
  1807         -          tempWC.op = TK_AND;
  1808         -          tempWC.a = pOrTerm;
  1809         -          tempWC.wctrlFlags = 0;
  1810         -          tempWC.nTerm = 1;
  1811         -          sBOI.pWC = &tempWC;
  1812         -          bestIndex(&sBOI);
  1813         -        }else{
  1814         -          continue;
  1815         -        }
  1816         -        rTotal += sBOI.cost.rCost;
  1817         -        nRow += sBOI.cost.plan.nRow;
  1818         -        used |= sBOI.cost.used;
  1819         -        if( rTotal>=p->cost.rCost ) break;
  1820         -      }
  1821         -
  1822         -      /* If there is an ORDER BY clause, increase the scan cost to account 
  1823         -      ** for the cost of the sort. */
  1824         -      if( p->pOrderBy!=0 ){
  1825         -        WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n",
  1826         -                    rTotal, rTotal+nRow*estLog(nRow)));
  1827         -        rTotal += nRow*estLog(nRow);
  1828         -      }
  1829         -
  1830         -      /* If the cost of scanning using this OR term for optimization is
  1831         -      ** less than the current cost stored in pCost, replace the contents
  1832         -      ** of pCost. */
  1833         -      WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow));
  1834         -      if( rTotal<p->cost.rCost ){
  1835         -        p->cost.rCost = rTotal;
  1836         -        p->cost.used = used;
  1837         -        p->cost.plan.nRow = nRow;
  1838         -        p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0;
  1839         -        p->cost.plan.wsFlags = flags;
  1840         -        p->cost.plan.u.pTerm = pTerm;
  1841         -      }
  1842         -    }
  1843         -  }
  1844         -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */
  1845         -}
  1846         -
  1847   2062   #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
  1848   2063   /*
  1849   2064   ** Return TRUE if the WHERE clause term pTerm is of a form where it
  1850   2065   ** could be used with an index to access pSrc, assuming an appropriate
  1851   2066   ** index existed.
  1852   2067   */
  1853   2068   static int termCanDriveIndex(
................................................................................
  1855   2070     struct SrcList_item *pSrc,     /* Table we are trying to access */
  1856   2071     Bitmask notReady               /* Tables in outer loops of the join */
  1857   2072   ){
  1858   2073     char aff;
  1859   2074     if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
  1860   2075     if( (pTerm->eOperator & WO_EQ)==0 ) return 0;
  1861   2076     if( (pTerm->prereqRight & notReady)!=0 ) return 0;
         2077  +  if( pTerm->u.leftColumn<0 ) return 0;
  1862   2078     aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity;
  1863   2079     if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
  1864   2080     return 1;
  1865   2081   }
  1866   2082   #endif
  1867   2083   
  1868         -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
  1869         -/*
  1870         -** If the query plan for pSrc specified in pCost is a full table scan
  1871         -** and indexing is allows (if there is no NOT INDEXED clause) and it
  1872         -** possible to construct a transient index that would perform better
  1873         -** than a full table scan even when the cost of constructing the index
  1874         -** is taken into account, then alter the query plan to use the
  1875         -** transient index.
  1876         -*/
  1877         -static void bestAutomaticIndex(WhereBestIdx *p){
  1878         -  Parse *pParse = p->pParse;            /* The parsing context */
  1879         -  WhereClause *pWC = p->pWC;            /* The WHERE clause */
  1880         -  struct SrcList_item *pSrc = p->pSrc;  /* The FROM clause term to search */
  1881         -  double nTableRow;                     /* Rows in the input table */
  1882         -  double logN;                          /* log(nTableRow) */
  1883         -  double costTempIdx;         /* per-query cost of the transient index */
  1884         -  WhereTerm *pTerm;           /* A single term of the WHERE clause */
  1885         -  WhereTerm *pWCEnd;          /* End of pWC->a[] */
  1886         -  Table *pTable;              /* Table tht might be indexed */
  1887         -
  1888         -  if( pParse->nQueryLoop<=(double)1 ){
  1889         -    /* There is no point in building an automatic index for a single scan */
  1890         -    return;
  1891         -  }
  1892         -  if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){
  1893         -    /* Automatic indices are disabled at run-time */
  1894         -    return;
  1895         -  }
  1896         -  if( (p->cost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0
  1897         -   && (p->cost.plan.wsFlags & WHERE_COVER_SCAN)==0
  1898         -  ){
  1899         -    /* We already have some kind of index in use for this query. */
  1900         -    return;
  1901         -  }
  1902         -  if( pSrc->viaCoroutine ){
  1903         -    /* Cannot index a co-routine */
  1904         -    return;
  1905         -  }
  1906         -  if( pSrc->notIndexed ){
  1907         -    /* The NOT INDEXED clause appears in the SQL. */
  1908         -    return;
  1909         -  }
  1910         -  if( pSrc->isCorrelated ){
  1911         -    /* The source is a correlated sub-query. No point in indexing it. */
  1912         -    return;
  1913         -  }
  1914         -
  1915         -  assert( pParse->nQueryLoop >= (double)1 );
  1916         -  pTable = pSrc->pTab;
  1917         -  nTableRow = pTable->nRowEst;
  1918         -  logN = estLog(nTableRow);
  1919         -  costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1);
  1920         -  if( costTempIdx>=p->cost.rCost ){
  1921         -    /* The cost of creating the transient table would be greater than
  1922         -    ** doing the full table scan */
  1923         -    return;
  1924         -  }
  1925         -
  1926         -  /* Search for any equality comparison term */
  1927         -  pWCEnd = &pWC->a[pWC->nTerm];
  1928         -  for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
  1929         -    if( termCanDriveIndex(pTerm, pSrc, p->notReady) ){
  1930         -      WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n",
  1931         -                    p->cost.rCost, costTempIdx));
  1932         -      p->cost.rCost = costTempIdx;
  1933         -      p->cost.plan.nRow = logN + 1;
  1934         -      p->cost.plan.wsFlags = WHERE_TEMP_INDEX;
  1935         -      p->cost.used = pTerm->prereqRight;
  1936         -      break;
  1937         -    }
  1938         -  }
  1939         -}
  1940         -#else
  1941         -# define bestAutomaticIndex(A)  /* no-op */
  1942         -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
  1943         -
  1944   2084   
  1945   2085   #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
  1946   2086   /*
  1947   2087   ** Generate code to construct the Index object for an automatic index
  1948   2088   ** and to set up the WhereLevel object pLevel so that the code generator
  1949   2089   ** makes use of the automatic index.
  1950   2090   */
................................................................................
  1966   2106     KeyInfo *pKeyinfo;          /* Key information for the index */   
  1967   2107     int addrTop;                /* Top of the index fill loop */
  1968   2108     int regRecord;              /* Register holding an index record */
  1969   2109     int n;                      /* Column counter */
  1970   2110     int i;                      /* Loop counter */
  1971   2111     int mxBitCol;               /* Maximum column in pSrc->colUsed */
  1972   2112     CollSeq *pColl;             /* Collating sequence to on a column */
         2113  +  WhereLoop *pLoop;           /* The Loop object */
  1973   2114     Bitmask idxCols;            /* Bitmap of columns used for indexing */
  1974   2115     Bitmask extraCols;          /* Bitmap of additional columns */
         2116  +  u8 sentWarning = 0;         /* True if a warnning has been issued */
  1975   2117   
  1976   2118     /* Generate code to skip over the creation and initialization of the
  1977   2119     ** transient index on 2nd and subsequent iterations of the loop. */
  1978   2120     v = pParse->pVdbe;
  1979   2121     assert( v!=0 );
  1980   2122     addrInit = sqlite3CodeOnce(pParse);
  1981   2123   
  1982   2124     /* Count the number of columns that will be added to the index
  1983   2125     ** and used to match WHERE clause constraints */
  1984   2126     nColumn = 0;
  1985   2127     pTable = pSrc->pTab;
  1986   2128     pWCEnd = &pWC->a[pWC->nTerm];
         2129  +  pLoop = pLevel->pWLoop;
  1987   2130     idxCols = 0;
  1988   2131     for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
  1989   2132       if( termCanDriveIndex(pTerm, pSrc, notReady) ){
  1990   2133         int iCol = pTerm->u.leftColumn;
  1991         -      Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<<iCol;
         2134  +      Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
  1992   2135         testcase( iCol==BMS );
  1993   2136         testcase( iCol==BMS-1 );
         2137  +      if( !sentWarning ){
         2138  +        sqlite3_log(SQLITE_WARNING_AUTOINDEX,
         2139  +            "automatic index on %s(%s)", pTable->zName,
         2140  +            pTable->aCol[iCol].zName);
         2141  +        sentWarning = 1;
         2142  +      }
  1994   2143         if( (idxCols & cMask)==0 ){
  1995         -        nColumn++;
         2144  +        if( whereLoopResize(pParse->db, pLoop, nColumn+1) ) return;
         2145  +        pLoop->aLTerm[nColumn++] = pTerm;
  1996   2146           idxCols |= cMask;
  1997   2147         }
  1998   2148       }
  1999   2149     }
  2000   2150     assert( nColumn>0 );
  2001         -  pLevel->plan.nEq = nColumn;
         2151  +  pLoop->u.btree.nEq = pLoop->nLTerm = nColumn;
         2152  +  pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
         2153  +                     | WHERE_AUTO_INDEX;
  2002   2154   
  2003   2155     /* Count the number of additional columns needed to create a
  2004   2156     ** covering index.  A "covering index" is an index that contains all
  2005   2157     ** columns that are needed by the query.  With a covering index, the
  2006   2158     ** original table never needs to be accessed.  Automatic indices must
  2007   2159     ** be a covering index because the index will not be updated if the
  2008   2160     ** original table changes and the index and table cannot both be used
  2009   2161     ** if they go out of sync.
  2010   2162     */
  2011         -  extraCols = pSrc->colUsed & (~idxCols | (((Bitmask)1)<<(BMS-1)));
         2163  +  extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
  2012   2164     mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol;
  2013   2165     testcase( pTable->nCol==BMS-1 );
  2014   2166     testcase( pTable->nCol==BMS-2 );
  2015   2167     for(i=0; i<mxBitCol; i++){
  2016         -    if( extraCols & (((Bitmask)1)<<i) ) nColumn++;
         2168  +    if( extraCols & MASKBIT(i) ) nColumn++;
  2017   2169     }
  2018         -  if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){
         2170  +  if( pSrc->colUsed & MASKBIT(BMS-1) ){
  2019   2171       nColumn += pTable->nCol - BMS + 1;
  2020   2172     }
  2021         -  pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WO_EQ;
         2173  +  pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY;
  2022   2174   
  2023   2175     /* Construct the Index object to describe this index */
  2024   2176     nByte = sizeof(Index);
  2025   2177     nByte += nColumn*sizeof(int);     /* Index.aiColumn */
  2026   2178     nByte += nColumn*sizeof(char*);   /* Index.azColl */
  2027   2179     nByte += nColumn;                 /* Index.aSortOrder */
  2028   2180     pIdx = sqlite3DbMallocZero(pParse->db, nByte);
  2029   2181     if( pIdx==0 ) return;
  2030         -  pLevel->plan.u.pIdx = pIdx;
         2182  +  pLoop->u.btree.pIndex = pIdx;
  2031   2183     pIdx->azColl = (char**)&pIdx[1];
  2032   2184     pIdx->aiColumn = (int*)&pIdx->azColl[nColumn];
  2033   2185     pIdx->aSortOrder = (u8*)&pIdx->aiColumn[nColumn];
  2034   2186     pIdx->zName = "auto-index";
  2035   2187     pIdx->nColumn = nColumn;
  2036   2188     pIdx->pTable = pTable;
  2037   2189     n = 0;
  2038   2190     idxCols = 0;
  2039   2191     for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
  2040   2192       if( termCanDriveIndex(pTerm, pSrc, notReady) ){
  2041   2193         int iCol = pTerm->u.leftColumn;
  2042         -      Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<<iCol;
         2194  +      Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
         2195  +      testcase( iCol==BMS-1 );
         2196  +      testcase( iCol==BMS );
  2043   2197         if( (idxCols & cMask)==0 ){
  2044   2198           Expr *pX = pTerm->pExpr;
  2045   2199           idxCols |= cMask;
  2046   2200           pIdx->aiColumn[n] = pTerm->u.leftColumn;
  2047   2201           pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
  2048   2202           pIdx->azColl[n] = ALWAYS(pColl) ? pColl->zName : "BINARY";
  2049   2203           n++;
  2050   2204         }
  2051   2205       }
  2052   2206     }
  2053         -  assert( (u32)n==pLevel->plan.nEq );
         2207  +  assert( (u32)n==pLoop->u.btree.nEq );
  2054   2208   
  2055   2209     /* Add additional columns needed to make the automatic index into
  2056   2210     ** a covering index */
  2057   2211     for(i=0; i<mxBitCol; i++){
  2058         -    if( extraCols & (((Bitmask)1)<<i) ){
         2212  +    if( extraCols & MASKBIT(i) ){
  2059   2213         pIdx->aiColumn[n] = i;
  2060   2214         pIdx->azColl[n] = "BINARY";
  2061   2215         n++;
  2062   2216       }
  2063   2217     }
  2064         -  if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){
         2218  +  if( pSrc->colUsed & MASKBIT(BMS-1) ){
  2065   2219       for(i=BMS-1; i<pTable->nCol; i++){
  2066   2220         pIdx->aiColumn[n] = i;
  2067   2221         pIdx->azColl[n] = "BINARY";
  2068   2222         n++;
  2069   2223       }
  2070   2224     }
  2071   2225     assert( n==nColumn );
  2072   2226   
  2073   2227     /* Create the automatic index */
  2074   2228     pKeyinfo = sqlite3IndexKeyinfo(pParse, pIdx);
  2075   2229     assert( pLevel->iIdxCur>=0 );
         2230  +  pLevel->iIdxCur = pParse->nTab++;
  2076   2231     sqlite3VdbeAddOp4(v, OP_OpenAutoindex, pLevel->iIdxCur, nColumn+1, 0,
  2077   2232                       (char*)pKeyinfo, P4_KEYINFO_HANDOFF);
  2078   2233     VdbeComment((v, "for %s", pTable->zName));
  2079   2234   
  2080   2235     /* Fill the automatic index with content */
  2081   2236     addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur);
  2082   2237     regRecord = sqlite3GetTempReg(pParse);
................................................................................
  2095   2250   
  2096   2251   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2097   2252   /*
  2098   2253   ** Allocate and populate an sqlite3_index_info structure. It is the 
  2099   2254   ** responsibility of the caller to eventually release the structure
  2100   2255   ** by passing the pointer returned by this function to sqlite3_free().
  2101   2256   */
  2102         -static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){
  2103         -  Parse *pParse = p->pParse; 
  2104         -  WhereClause *pWC = p->pWC;
  2105         -  struct SrcList_item *pSrc = p->pSrc;
  2106         -  ExprList *pOrderBy = p->pOrderBy;
         2257  +static sqlite3_index_info *allocateIndexInfo(
         2258  +  Parse *pParse,
         2259  +  WhereClause *pWC,
         2260  +  struct SrcList_item *pSrc,
         2261  +  ExprList *pOrderBy
         2262  +){
  2107   2263     int i, j;
  2108   2264     int nTerm;
  2109   2265     struct sqlite3_index_constraint *pIdxCons;
  2110   2266     struct sqlite3_index_orderby *pIdxOrderBy;
  2111   2267     struct sqlite3_index_constraint_usage *pUsage;
  2112   2268     WhereTerm *pTerm;
  2113   2269     int nOrderBy;
  2114   2270     sqlite3_index_info *pIdxInfo;
  2115   2271   
  2116         -  WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName));
  2117         -
  2118   2272     /* Count the number of possible WHERE clause constraints referring
  2119   2273     ** to this virtual table */
  2120   2274     for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
  2121   2275       if( pTerm->leftCursor != pSrc->iCursor ) continue;
  2122   2276       assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
  2123   2277       testcase( pTerm->eOperator & WO_IN );
  2124   2278       testcase( pTerm->eOperator & WO_ISNULL );
................................................................................
  2146   2300     /* Allocate the sqlite3_index_info structure
  2147   2301     */
  2148   2302     pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
  2149   2303                              + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
  2150   2304                              + sizeof(*pIdxOrderBy)*nOrderBy );
  2151   2305     if( pIdxInfo==0 ){
  2152   2306       sqlite3ErrorMsg(pParse, "out of memory");
  2153         -    /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
  2154   2307       return 0;
  2155   2308     }
  2156   2309   
  2157   2310     /* Initialize the structure.  The sqlite3_index_info structure contains
  2158   2311     ** many fields that are declared "const" to prevent xBestIndex from
  2159   2312     ** changing them.  We have to do some funky casting in order to
  2160   2313     ** initialize those fields.
................................................................................
  2202   2355   
  2203   2356     return pIdxInfo;
  2204   2357   }
  2205   2358   
  2206   2359   /*
  2207   2360   ** The table object reference passed as the second argument to this function
  2208   2361   ** must represent a virtual table. This function invokes the xBestIndex()
  2209         -** method of the virtual table with the sqlite3_index_info pointer passed
  2210         -** as the argument.
         2362  +** method of the virtual table with the sqlite3_index_info object that
         2363  +** comes in as the 3rd argument to this function.
  2211   2364   **
  2212   2365   ** If an error occurs, pParse is populated with an error message and a
  2213   2366   ** non-zero value is returned. Otherwise, 0 is returned and the output
  2214   2367   ** part of the sqlite3_index_info structure is left populated.
  2215   2368   **
  2216   2369   ** Whether or not an error is returned, it is the responsibility of the
  2217   2370   ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
................................................................................
  2218   2371   ** that this is required.
  2219   2372   */
  2220   2373   static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
  2221   2374     sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
  2222   2375     int i;
  2223   2376     int rc;
  2224   2377   
  2225         -  WHERETRACE(("xBestIndex for %s\n", pTab->zName));
  2226   2378     TRACE_IDX_INPUTS(p);
  2227   2379     rc = pVtab->pModule->xBestIndex(pVtab, p);
  2228   2380     TRACE_IDX_OUTPUTS(p);
  2229   2381   
  2230   2382     if( rc!=SQLITE_OK ){
  2231   2383       if( rc==SQLITE_NOMEM ){
  2232   2384         pParse->db->mallocFailed = 1;
................................................................................
  2244   2396         sqlite3ErrorMsg(pParse, 
  2245   2397             "table %s: xBestIndex returned an invalid plan", pTab->zName);
  2246   2398       }
  2247   2399     }
  2248   2400   
  2249   2401     return pParse->nErr;
  2250   2402   }
  2251         -
  2252         -
  2253         -/*
  2254         -** Compute the best index for a virtual table.
  2255         -**
  2256         -** The best index is computed by the xBestIndex method of the virtual
  2257         -** table module.  This routine is really just a wrapper that sets up
  2258         -** the sqlite3_index_info structure that is used to communicate with
  2259         -** xBestIndex.
  2260         -**
  2261         -** In a join, this routine might be called multiple times for the
  2262         -** same virtual table.  The sqlite3_index_info structure is created
  2263         -** and initialized on the first invocation and reused on all subsequent
  2264         -** invocations.  The sqlite3_index_info structure is also used when
  2265         -** code is generated to access the virtual table.  The whereInfoDelete() 
  2266         -** routine takes care of freeing the sqlite3_index_info structure after
  2267         -** everybody has finished with it.
  2268         -*/
  2269         -static void bestVirtualIndex(WhereBestIdx *p){
  2270         -  Parse *pParse = p->pParse;      /* The parsing context */
  2271         -  WhereClause *pWC = p->pWC;      /* The WHERE clause */
  2272         -  struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */
  2273         -  Table *pTab = pSrc->pTab;
  2274         -  sqlite3_index_info *pIdxInfo;
  2275         -  struct sqlite3_index_constraint *pIdxCons;
  2276         -  struct sqlite3_index_constraint_usage *pUsage;
  2277         -  WhereTerm *pTerm;
  2278         -  int i, j;
  2279         -  int nOrderBy;
  2280         -  int bAllowIN;                   /* Allow IN optimizations */
  2281         -  double rCost;
  2282         -
  2283         -  /* Make sure wsFlags is initialized to some sane value. Otherwise, if the 
  2284         -  ** malloc in allocateIndexInfo() fails and this function returns leaving
  2285         -  ** wsFlags in an uninitialized state, the caller may behave unpredictably.
  2286         -  */
  2287         -  memset(&p->cost, 0, sizeof(p->cost));
  2288         -  p->cost.plan.wsFlags = WHERE_VIRTUALTABLE;
  2289         -
  2290         -  /* If the sqlite3_index_info structure has not been previously
  2291         -  ** allocated and initialized, then allocate and initialize it now.
  2292         -  */
  2293         -  pIdxInfo = *p->ppIdxInfo;
  2294         -  if( pIdxInfo==0 ){
  2295         -    *p->ppIdxInfo = pIdxInfo = allocateIndexInfo(p);
  2296         -  }
  2297         -  if( pIdxInfo==0 ){
  2298         -    return;
  2299         -  }
  2300         -
  2301         -  /* At this point, the sqlite3_index_info structure that pIdxInfo points
  2302         -  ** to will have been initialized, either during the current invocation or
  2303         -  ** during some prior invocation.  Now we just have to customize the
  2304         -  ** details of pIdxInfo for the current invocation and pass it to
  2305         -  ** xBestIndex.
  2306         -  */
  2307         -
  2308         -  /* The module name must be defined. Also, by this point there must
  2309         -  ** be a pointer to an sqlite3_vtab structure. Otherwise
  2310         -  ** sqlite3ViewGetColumnNames() would have picked up the error. 
  2311         -  */
  2312         -  assert( pTab->azModuleArg && pTab->azModuleArg[0] );
  2313         -  assert( sqlite3GetVTable(pParse->db, pTab) );
  2314         -
  2315         -  /* Try once or twice.  On the first attempt, allow IN optimizations.
  2316         -  ** If an IN optimization is accepted by the virtual table xBestIndex
  2317         -  ** method, but the  pInfo->aConstrainUsage.omit flag is not set, then
  2318         -  ** the query will not work because it might allow duplicate rows in
  2319         -  ** output.  In that case, run the xBestIndex method a second time
  2320         -  ** without the IN constraints.  Usually this loop only runs once.
  2321         -  ** The loop will exit using a "break" statement.
  2322         -  */
  2323         -  for(bAllowIN=1; 1; bAllowIN--){
  2324         -    assert( bAllowIN==0 || bAllowIN==1 );
  2325         -
  2326         -    /* Set the aConstraint[].usable fields and initialize all 
  2327         -    ** output variables to zero.
  2328         -    **
  2329         -    ** aConstraint[].usable is true for constraints where the right-hand
  2330         -    ** side contains only references to tables to the left of the current
  2331         -    ** table.  In other words, if the constraint is of the form:
  2332         -    **
  2333         -    **           column = expr
  2334         -    **
  2335         -    ** and we are evaluating a join, then the constraint on column is 
  2336         -    ** only valid if all tables referenced in expr occur to the left
  2337         -    ** of the table containing column.
  2338         -    **
  2339         -    ** The aConstraints[] array contains entries for all constraints
  2340         -    ** on the current table.  That way we only have to compute it once
  2341         -    ** even though we might try to pick the best index multiple times.
  2342         -    ** For each attempt at picking an index, the order of tables in the
  2343         -    ** join might be different so we have to recompute the usable flag
  2344         -    ** each time.
  2345         -    */
  2346         -    pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
  2347         -    pUsage = pIdxInfo->aConstraintUsage;
  2348         -    for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){
  2349         -      j = pIdxCons->iTermOffset;
  2350         -      pTerm = &pWC->a[j];
  2351         -      if( (pTerm->prereqRight&p->notReady)==0
  2352         -       && (bAllowIN || (pTerm->eOperator & WO_IN)==0)
  2353         -      ){
  2354         -        pIdxCons->usable = 1;
  2355         -      }else{
  2356         -        pIdxCons->usable = 0;
  2357         -      }
  2358         -    }
  2359         -    memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint);
  2360         -    if( pIdxInfo->needToFreeIdxStr ){
  2361         -      sqlite3_free(pIdxInfo->idxStr);
  2362         -    }
  2363         -    pIdxInfo->idxStr = 0;
  2364         -    pIdxInfo->idxNum = 0;
  2365         -    pIdxInfo->needToFreeIdxStr = 0;
  2366         -    pIdxInfo->orderByConsumed = 0;
  2367         -    /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */
  2368         -    pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2);
  2369         -    nOrderBy = pIdxInfo->nOrderBy;
  2370         -    if( !p->pOrderBy ){
  2371         -      pIdxInfo->nOrderBy = 0;
  2372         -    }
  2373         -  
  2374         -    if( vtabBestIndex(pParse, pTab, pIdxInfo) ){
  2375         -      return;
  2376         -    }
  2377         -  
  2378         -    pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
  2379         -    for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){
  2380         -      if( pUsage[i].argvIndex>0 ){
  2381         -        j = pIdxCons->iTermOffset;
  2382         -        pTerm = &pWC->a[j];
  2383         -        p->cost.used |= pTerm->prereqRight;
  2384         -        if( (pTerm->eOperator & WO_IN)!=0 ){
  2385         -          if( pUsage[i].omit==0 ){
  2386         -            /* Do not attempt to use an IN constraint if the virtual table
  2387         -            ** says that the equivalent EQ constraint cannot be safely omitted.
  2388         -            ** If we do attempt to use such a constraint, some rows might be
  2389         -            ** repeated in the output. */
  2390         -            break;
  2391         -          }
  2392         -          /* A virtual table that is constrained by an IN clause may not
  2393         -          ** consume the ORDER BY clause because (1) the order of IN terms
  2394         -          ** is not necessarily related to the order of output terms and
  2395         -          ** (2) Multiple outputs from a single IN value will not merge
  2396         -          ** together.  */
  2397         -          pIdxInfo->orderByConsumed = 0;
  2398         -        }
  2399         -      }
  2400         -    }
  2401         -    if( i>=pIdxInfo->nConstraint ) break;
  2402         -  }
  2403         -
  2404         -  /* The orderByConsumed signal is only valid if all outer loops collectively
  2405         -  ** generate just a single row of output.
  2406         -  */
  2407         -  if( pIdxInfo->orderByConsumed ){
  2408         -    for(i=0; i<p->i; i++){
  2409         -      if( (p->aLevel[i].plan.wsFlags & WHERE_UNIQUE)==0 ){
  2410         -        pIdxInfo->orderByConsumed = 0;
  2411         -      }
  2412         -    }
  2413         -  }
  2414         -  
  2415         -  /* If there is an ORDER BY clause, and the selected virtual table index
  2416         -  ** does not satisfy it, increase the cost of the scan accordingly. This
  2417         -  ** matches the processing for non-virtual tables in bestBtreeIndex().
  2418         -  */
  2419         -  rCost = pIdxInfo->estimatedCost;
  2420         -  if( p->pOrderBy && pIdxInfo->orderByConsumed==0 ){
  2421         -    rCost += estLog(rCost)*rCost;
  2422         -  }
  2423         -
  2424         -  /* The cost is not allowed to be larger than SQLITE_BIG_DBL (the
  2425         -  ** inital value of lowestCost in this loop. If it is, then the
  2426         -  ** (cost<lowestCost) test below will never be true.
  2427         -  ** 
  2428         -  ** Use "(double)2" instead of "2.0" in case OMIT_FLOATING_POINT 
  2429         -  ** is defined.
  2430         -  */
  2431         -  if( (SQLITE_BIG_DBL/((double)2))<rCost ){
  2432         -    p->cost.rCost = (SQLITE_BIG_DBL/((double)2));
  2433         -  }else{
  2434         -    p->cost.rCost = rCost;
  2435         -  }
  2436         -  p->cost.plan.u.pVtabIdx = pIdxInfo;
  2437         -  if( pIdxInfo->orderByConsumed ){
  2438         -    p->cost.plan.wsFlags |= WHERE_ORDERED;
  2439         -    p->cost.plan.nOBSat = nOrderBy;
  2440         -  }else{
  2441         -    p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0;
  2442         -  }
  2443         -  p->cost.plan.nEq = 0;
  2444         -  pIdxInfo->nOrderBy = nOrderBy;
  2445         -
  2446         -  /* Try to find a more efficient access pattern by using multiple indexes
  2447         -  ** to optimize an OR expression within the WHERE clause. 
  2448         -  */
  2449         -  bestOrClauseIndex(p);
  2450         -}
  2451         -#endif /* SQLITE_OMIT_VIRTUALTABLE */
         2403  +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
         2404  +
  2452   2405   
  2453   2406   #ifdef SQLITE_ENABLE_STAT3
  2454   2407   /*
  2455   2408   ** Estimate the location of a particular key among all keys in an
  2456   2409   ** index.  Store the results in aStat as follows:
  2457   2410   **
  2458   2411   **    aStat[0]      Est. number of rows less than pVal
................................................................................
  2531   2484         const u8 *z;
  2532   2485         if( eType==SQLITE_BLOB ){
  2533   2486           z = (const u8 *)sqlite3_value_blob(pVal);
  2534   2487           pColl = db->pDfltColl;
  2535   2488           assert( pColl->enc==SQLITE_UTF8 );
  2536   2489         }else{
  2537   2490           pColl = sqlite3GetCollSeq(pParse, SQLITE_UTF8, 0, *pIdx->azColl);
  2538         -        if( pColl==0 ){
  2539         -          return SQLITE_ERROR;
  2540         -        }
         2491  +        /* If the collating sequence was unavailable, we should have failed
         2492  +        ** long ago and never reached this point.  But we'll check just to
         2493  +        ** be doubly sure. */
         2494  +        if( NEVER(pColl==0) ) return SQLITE_ERROR;
  2541   2495           z = (const u8 *)sqlite3ValueText(pVal, pColl->enc);
  2542   2496           if( !z ){
  2543   2497             return SQLITE_NOMEM;
  2544   2498           }
  2545   2499           assert( z && pColl && pColl->xCmp );
  2546   2500         }
  2547   2501         n = sqlite3ValueBytes(pVal, pColl->enc);
................................................................................
  2686   2640   */
  2687   2641   static int whereRangeScanEst(
  2688   2642     Parse *pParse,       /* Parsing & code generating context */
  2689   2643     Index *p,            /* The index containing the range-compared column; "x" */
  2690   2644     int nEq,             /* index into p->aCol[] of the range-compared column */
  2691   2645     WhereTerm *pLower,   /* Lower bound on the range. ex: "x>123" Might be NULL */
  2692   2646     WhereTerm *pUpper,   /* Upper bound on the range. ex: "x<455" Might be NULL */
  2693         -  double *pRangeDiv   /* OUT: Reduce search space by this divisor */
         2647  +  WhereCost *pRangeDiv /* OUT: Reduce search space by this divisor */
  2694   2648   ){
  2695   2649     int rc = SQLITE_OK;
  2696   2650   
  2697   2651   #ifdef SQLITE_ENABLE_STAT3
  2698   2652   
  2699         -  if( nEq==0 && p->nSample ){
         2653  +  if( nEq==0 && p->nSample && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){
  2700   2654       sqlite3_value *pRangeVal;
  2701   2655       tRowcnt iLower = 0;
  2702   2656       tRowcnt iUpper = p->aiRowEst[0];
  2703   2657       tRowcnt a[2];
  2704   2658       u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity;
  2705   2659   
  2706   2660       if( pLower ){
................................................................................
  2724   2678         ){
  2725   2679           iUpper = a[0];
  2726   2680           if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1];
  2727   2681         }
  2728   2682         sqlite3ValueFree(pRangeVal);
  2729   2683       }
  2730   2684       if( rc==SQLITE_OK ){
  2731         -      if( iUpper<=iLower ){
  2732         -        *pRangeDiv = (double)p->aiRowEst[0];
  2733         -      }else{
  2734         -        *pRangeDiv = (double)p->aiRowEst[0]/(double)(iUpper - iLower);
         2685  +      WhereCost iBase = whereCost(p->aiRowEst[0]);
         2686  +      if( iUpper>iLower ){
         2687  +        iBase -= whereCost(iUpper - iLower);
  2735   2688         }
  2736         -      WHERETRACE(("range scan regions: %u..%u  div=%g\n",
  2737         -                  (u32)iLower, (u32)iUpper, *pRangeDiv));
         2689  +      *pRangeDiv = iBase;
         2690  +      WHERETRACE(0x100, ("range scan regions: %u..%u  div=%d\n",
         2691  +                         (u32)iLower, (u32)iUpper, *pRangeDiv));
  2738   2692         return SQLITE_OK;
  2739   2693       }
  2740   2694     }
  2741   2695   #else
  2742   2696     UNUSED_PARAMETER(pParse);
  2743   2697     UNUSED_PARAMETER(p);
  2744   2698     UNUSED_PARAMETER(nEq);
  2745   2699   #endif
  2746   2700     assert( pLower || pUpper );
  2747         -  *pRangeDiv = (double)1;
  2748         -  if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *pRangeDiv *= (double)4;
  2749         -  if( pUpper ) *pRangeDiv *= (double)4;
         2701  +  *pRangeDiv = 0;
         2702  +  /* TUNING:  Each inequality constraint reduces the search space 4-fold.
         2703  +  ** A BETWEEN operator, therefore, reduces the search space 16-fold */
         2704  +  if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){
         2705  +    *pRangeDiv += 20;  assert( 20==whereCost(4) );
         2706  +  }
         2707  +  if( pUpper ){
         2708  +    *pRangeDiv += 20;  assert( 20==whereCost(4) );
         2709  +  }
  2750   2710     return rc;
  2751   2711   }
  2752   2712   
  2753   2713   #ifdef SQLITE_ENABLE_STAT3
  2754   2714   /*
  2755   2715   ** Estimate the number of rows that will be returned based on
  2756   2716   ** an equality constraint x=VALUE and where that VALUE occurs in
................................................................................
  2768   2728   ** for a UTF conversion required for comparison.  The error is stored
  2769   2729   ** in the pParse structure.
  2770   2730   */
  2771   2731   static int whereEqualScanEst(
  2772   2732     Parse *pParse,       /* Parsing & code generating context */
  2773   2733     Index *p,            /* The index whose left-most column is pTerm */
  2774   2734     Expr *pExpr,         /* Expression for VALUE in the x=VALUE constraint */
  2775         -  double *pnRow        /* Write the revised row estimate here */
         2735  +  tRowcnt *pnRow       /* Write the revised row estimate here */
  2776   2736   ){
  2777   2737     sqlite3_value *pRhs = 0;  /* VALUE on right-hand side of pTerm */
  2778   2738     u8 aff;                   /* Column affinity */
  2779   2739     int rc;                   /* Subfunction return code */
  2780   2740     tRowcnt a[2];             /* Statistics */
  2781   2741   
  2782   2742     assert( p->aSample!=0 );
................................................................................
  2787   2747       if( rc ) goto whereEqualScanEst_cancel;
  2788   2748     }else{
  2789   2749       pRhs = sqlite3ValueNew(pParse->db);
  2790   2750     }
  2791   2751     if( pRhs==0 ) return SQLITE_NOTFOUND;
  2792   2752     rc = whereKeyStats(pParse, p, pRhs, 0, a);
  2793   2753     if( rc==SQLITE_OK ){
  2794         -    WHERETRACE(("equality scan regions: %d\n", (int)a[1]));
         2754  +    WHERETRACE(0x100,("equality scan regions: %d\n", (int)a[1]));
  2795   2755       *pnRow = a[1];
  2796   2756     }
  2797   2757   whereEqualScanEst_cancel:
  2798   2758     sqlite3ValueFree(pRhs);
  2799   2759     return rc;
  2800   2760   }
  2801   2761   #endif /* defined(SQLITE_ENABLE_STAT3) */
................................................................................
  2817   2777   ** for a UTF conversion required for comparison.  The error is stored
  2818   2778   ** in the pParse structure.
  2819   2779   */
  2820   2780   static int whereInScanEst(
  2821   2781     Parse *pParse,       /* Parsing & code generating context */
  2822   2782     Index *p,            /* The index whose left-most column is pTerm */
  2823   2783     ExprList *pList,     /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
  2824         -  double *pnRow        /* Write the revised row estimate here */
         2784  +  tRowcnt *pnRow       /* Write the revised row estimate here */
  2825   2785   ){
  2826         -  int rc = SQLITE_OK;         /* Subfunction return code */
  2827         -  double nEst;                /* Number of rows for a single term */
  2828         -  double nRowEst = (double)0; /* New estimate of the number of rows */
  2829         -  int i;                      /* Loop counter */
         2786  +  int rc = SQLITE_OK;     /* Subfunction return code */
         2787  +  tRowcnt nEst;           /* Number of rows for a single term */
         2788  +  tRowcnt nRowEst = 0;    /* New estimate of the number of rows */
         2789  +  int i;                  /* Loop counter */
  2830   2790   
  2831   2791     assert( p->aSample!=0 );
  2832   2792     for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
  2833   2793       nEst = p->aiRowEst[0];
  2834   2794       rc = whereEqualScanEst(pParse, p, pList->a[i].pExpr, &nEst);
  2835   2795       nRowEst += nEst;
  2836   2796     }
  2837   2797     if( rc==SQLITE_OK ){
  2838   2798       if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0];
  2839   2799       *pnRow = nRowEst;
  2840         -    WHERETRACE(("IN row estimate: est=%g\n", nRowEst));
         2800  +    WHERETRACE(0x100,("IN row estimate: est=%g\n", nRowEst));
  2841   2801     }
  2842   2802     return rc;
  2843   2803   }
  2844   2804   #endif /* defined(SQLITE_ENABLE_STAT3) */
  2845   2805   
  2846         -/*
  2847         -** Check to see if column iCol of the table with cursor iTab will appear
  2848         -** in sorted order according to the current query plan.
  2849         -**
  2850         -** Return values:
  2851         -**
  2852         -**    0   iCol is not ordered
  2853         -**    1   iCol has only a single value
  2854         -**    2   iCol is in ASC order
  2855         -**    3   iCol is in DESC order
  2856         -*/
  2857         -static int isOrderedColumn(
  2858         -  WhereBestIdx *p,
  2859         -  int iTab,
  2860         -  int iCol
  2861         -){
  2862         -  int i, j;
  2863         -  WhereLevel *pLevel = &p->aLevel[p->i-1];
  2864         -  Index *pIdx;
  2865         -  u8 sortOrder;
  2866         -  for(i=p->i-1; i>=0; i--, pLevel--){
  2867         -    if( pLevel->iTabCur!=iTab ) continue;
  2868         -    if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){
  2869         -      return 1;
  2870         -    }
  2871         -    assert( (pLevel->plan.wsFlags & WHERE_ORDERED)!=0 );
  2872         -    if( (pIdx = pLevel->plan.u.pIdx)!=0 ){
  2873         -      if( iCol<0 ){
  2874         -        sortOrder = 0;
  2875         -        testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 );
  2876         -      }else{
  2877         -        int n = pIdx->nColumn;
  2878         -        for(j=0; j<n; j++){
  2879         -          if( iCol==pIdx->aiColumn[j] ) break;
  2880         -        }
  2881         -        if( j>=n ) return 0;
  2882         -        sortOrder = pIdx->aSortOrder[j];
  2883         -        testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 );
  2884         -      }
  2885         -    }else{
  2886         -      if( iCol!=(-1) ) return 0;
  2887         -      sortOrder = 0;
  2888         -      testcase( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 );
  2889         -    }
  2890         -    if( (pLevel->plan.wsFlags & WHERE_REVERSE)!=0 ){
  2891         -      assert( sortOrder==0 || sortOrder==1 );
  2892         -      testcase( sortOrder==1 );
  2893         -      sortOrder = 1 - sortOrder;
  2894         -    }
  2895         -    return sortOrder+2;
  2896         -  }
  2897         -  return 0;
  2898         -}
  2899         -
  2900         -/*
  2901         -** This routine decides if pIdx can be used to satisfy the ORDER BY
  2902         -** clause, either in whole or in part.  The return value is the 
  2903         -** cumulative number of terms in the ORDER BY clause that are satisfied
  2904         -** by the index pIdx and other indices in outer loops.
  2905         -**
  2906         -** The table being queried has a cursor number of "base".  pIdx is the
  2907         -** index that is postulated for use to access the table.
  2908         -**
  2909         -** The *pbRev value is set to 0 order 1 depending on whether or not
  2910         -** pIdx should be run in the forward order or in reverse order.
  2911         -*/
  2912         -static int isSortingIndex(
  2913         -  WhereBestIdx *p,    /* Best index search context */
  2914         -  Index *pIdx,        /* The index we are testing */
  2915         -  int base,           /* Cursor number for the table to be sorted */
  2916         -  int *pbRev,         /* Set to 1 for reverse-order scan of pIdx */
  2917         -  int *pbObUnique     /* ORDER BY column values will different in every row */
  2918         -){
  2919         -  int i;                        /* Number of pIdx terms used */
  2920         -  int j;                        /* Number of ORDER BY terms satisfied */
  2921         -  int sortOrder = 2;            /* 0: forward.  1: backward.  2: unknown */
  2922         -  int nTerm;                    /* Number of ORDER BY terms */
  2923         -  struct ExprList_item *pOBItem;/* A term of the ORDER BY clause */
  2924         -  Table *pTab = pIdx->pTable;   /* Table that owns index pIdx */
  2925         -  ExprList *pOrderBy;           /* The ORDER BY clause */
  2926         -  Parse *pParse = p->pParse;    /* Parser context */
  2927         -  sqlite3 *db = pParse->db;     /* Database connection */
  2928         -  int nPriorSat;                /* ORDER BY terms satisfied by outer loops */
  2929         -  int seenRowid = 0;            /* True if an ORDER BY rowid term is seen */
  2930         -  int uniqueNotNull;            /* pIdx is UNIQUE with all terms are NOT NULL */
  2931         -  int outerObUnique;            /* Outer loops generate different values in
  2932         -                                ** every row for the ORDER BY columns */
  2933         -
  2934         -  if( p->i==0 ){
  2935         -    nPriorSat = 0;
  2936         -    outerObUnique = 1;
  2937         -  }else{
  2938         -    u32 wsFlags = p->aLevel[p->i-1].plan.wsFlags;
  2939         -    nPriorSat = p->aLevel[p->i-1].plan.nOBSat;
  2940         -    if( (wsFlags & WHERE_ORDERED)==0 ){
  2941         -      /* This loop cannot be ordered unless the next outer loop is
  2942         -      ** also ordered */
  2943         -      return nPriorSat;
  2944         -    }
  2945         -    if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ){
  2946         -      /* Only look at the outer-most loop if the OrderByIdxJoin
  2947         -      ** optimization is disabled */
  2948         -      return nPriorSat;
  2949         -    }
  2950         -    testcase( wsFlags & WHERE_OB_UNIQUE );
  2951         -    testcase( wsFlags & WHERE_ALL_UNIQUE );
  2952         -    outerObUnique = (wsFlags & (WHERE_OB_UNIQUE|WHERE_ALL_UNIQUE))!=0;
  2953         -  }
  2954         -  pOrderBy = p->pOrderBy;
  2955         -  assert( pOrderBy!=0 );
  2956         -  if( pIdx->bUnordered ){
  2957         -    /* Hash indices (indicated by the "unordered" tag on sqlite_stat1) cannot
  2958         -    ** be used for sorting */
  2959         -    return nPriorSat;
  2960         -  }
  2961         -  nTerm = pOrderBy->nExpr;
  2962         -  uniqueNotNull = pIdx->onError!=OE_None;
  2963         -  assert( nTerm>0 );
  2964         -
  2965         -  /* Argument pIdx must either point to a 'real' named index structure, 
  2966         -  ** or an index structure allocated on the stack by bestBtreeIndex() to
  2967         -  ** represent the rowid index that is part of every table.  */
  2968         -  assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) );
  2969         -
  2970         -  /* Match terms of the ORDER BY clause against columns of
  2971         -  ** the index.
  2972         -  **
  2973         -  ** Note that indices have pIdx->nColumn regular columns plus
  2974         -  ** one additional column containing the rowid.  The rowid column
  2975         -  ** of the index is also allowed to match against the ORDER BY
  2976         -  ** clause.
  2977         -  */
  2978         -  j = nPriorSat;
  2979         -  for(i=0,pOBItem=&pOrderBy->a[j]; j<nTerm && i<=pIdx->nColumn; i++){
  2980         -    Expr *pOBExpr;          /* The expression of the ORDER BY pOBItem */
  2981         -    CollSeq *pColl;         /* The collating sequence of pOBExpr */
  2982         -    int termSortOrder;      /* Sort order for this term */
  2983         -    int iColumn;            /* The i-th column of the index.  -1 for rowid */
  2984         -    int iSortOrder;         /* 1 for DESC, 0 for ASC on the i-th index term */
  2985         -    int isEq;               /* Subject to an == or IS NULL constraint */
  2986         -    int isMatch;            /* ORDER BY term matches the index term */
  2987         -    const char *zColl;      /* Name of collating sequence for i-th index term */
  2988         -    WhereTerm *pConstraint; /* A constraint in the WHERE clause */
  2989         -
  2990         -    /* If the next term of the ORDER BY clause refers to anything other than
  2991         -    ** a column in the "base" table, then this index will not be of any
  2992         -    ** further use in handling the ORDER BY. */
  2993         -    pOBExpr = sqlite3ExprSkipCollate(pOBItem->pExpr);
  2994         -    if( pOBExpr->op!=TK_COLUMN || pOBExpr->iTable!=base ){
  2995         -      break;
  2996         -    }
  2997         -
  2998         -    /* Find column number and collating sequence for the next entry
  2999         -    ** in the index */
  3000         -    if( pIdx->zName && i<pIdx->nColumn ){
  3001         -      iColumn = pIdx->aiColumn[i];
  3002         -      if( iColumn==pIdx->pTable->iPKey ){
  3003         -        iColumn = -1;
  3004         -      }
  3005         -      iSortOrder = pIdx->aSortOrder[i];
  3006         -      zColl = pIdx->azColl[i];
  3007         -      assert( zColl!=0 );
  3008         -    }else{
  3009         -      iColumn = -1;
  3010         -      iSortOrder = 0;
  3011         -      zColl = 0;
  3012         -    }
  3013         -
  3014         -    /* Check to see if the column number and collating sequence of the
  3015         -    ** index match the column number and collating sequence of the ORDER BY
  3016         -    ** clause entry.  Set isMatch to 1 if they both match. */
  3017         -    if( pOBExpr->iColumn==iColumn ){
  3018         -      if( zColl ){
  3019         -        pColl = sqlite3ExprCollSeq(pParse, pOBItem->pExpr);
  3020         -        if( !pColl ) pColl = db->pDfltColl;
  3021         -        isMatch = sqlite3StrICmp(pColl->zName, zColl)==0;
  3022         -      }else{
  3023         -        isMatch = 1;
  3024         -      }
  3025         -    }else{
  3026         -      isMatch = 0;
  3027         -    }
  3028         -
  3029         -    /* termSortOrder is 0 or 1 for whether or not the access loop should
  3030         -    ** run forward or backwards (respectively) in order to satisfy this 
  3031         -    ** term of the ORDER BY clause. */
  3032         -    assert( pOBItem->sortOrder==0 || pOBItem->sortOrder==1 );
  3033         -    assert( iSortOrder==0 || iSortOrder==1 );
  3034         -    termSortOrder = iSortOrder ^ pOBItem->sortOrder;
  3035         -
  3036         -    /* If X is the column in the index and ORDER BY clause, check to see
  3037         -    ** if there are any X= or X IS NULL constraints in the WHERE clause. */
  3038         -    pConstraint = findTerm(p->pWC, base, iColumn, p->notReady,
  3039         -                           WO_EQ|WO_ISNULL|WO_IN, pIdx);
  3040         -    if( pConstraint==0 ){
  3041         -      isEq = 0;
  3042         -    }else if( (pConstraint->eOperator & WO_IN)!=0 ){
  3043         -      isEq = 0;
  3044         -    }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){
  3045         -      uniqueNotNull = 0;
  3046         -      isEq = 1;  /* "X IS NULL" means X has only a single value */
  3047         -    }else if( pConstraint->prereqRight==0 ){
  3048         -      isEq = 1;  /* Constraint "X=constant" means X has only a single value */
  3049         -    }else{
  3050         -      Expr *pRight = pConstraint->pExpr->pRight;
  3051         -      if( pRight->op==TK_COLUMN ){
  3052         -        WHERETRACE(("       .. isOrderedColumn(tab=%d,col=%d)",
  3053         -                    pRight->iTable, pRight->iColumn));
  3054         -        isEq = isOrderedColumn(p, pRight->iTable, pRight->iColumn);
  3055         -        WHERETRACE((" -> isEq=%d\n", isEq));
  3056         -
  3057         -        /* If the constraint is of the form X=Y where Y is an ordered value
  3058         -        ** in an outer loop, then make sure the sort order of Y matches the
  3059         -        ** sort order required for X. */
  3060         -        if( isMatch && isEq>=2 && isEq!=pOBItem->sortOrder+2 ){
  3061         -          testcase( isEq==2 );
  3062         -          testcase( isEq==3 );
  3063         -          break;
  3064         -        }
  3065         -      }else{
  3066         -        isEq = 0;  /* "X=expr" places no ordering constraints on X */
  3067         -      }
  3068         -    }
  3069         -    if( !isMatch ){
  3070         -      if( isEq==0 ){
  3071         -        break;
  3072         -      }else{
  3073         -        continue;
  3074         -      }
  3075         -    }else if( isEq!=1 ){
  3076         -      if( sortOrder==2 ){
  3077         -        sortOrder = termSortOrder;
  3078         -      }else if( termSortOrder!=sortOrder ){
  3079         -        break;
  3080         -      }
  3081         -    }
  3082         -    j++;
  3083         -    pOBItem++;
  3084         -    if( iColumn<0 ){
  3085         -      seenRowid = 1;
  3086         -      break;
  3087         -    }else if( pTab->aCol[iColumn].notNull==0 && isEq!=1 ){
  3088         -      testcase( isEq==0 );
  3089         -      testcase( isEq==2 );
  3090         -      testcase( isEq==3 );
  3091         -      uniqueNotNull = 0;
  3092         -    }
  3093         -  }
  3094         -  if( seenRowid ){
  3095         -    uniqueNotNull = 1;
  3096         -  }else if( uniqueNotNull==0 || i<pIdx->nColumn ){
  3097         -    uniqueNotNull = 0;
  3098         -  }
  3099         -
  3100         -  /* If we have not found at least one ORDER BY term that matches the
  3101         -  ** index, then show no progress. */
  3102         -  if( pOBItem==&pOrderBy->a[nPriorSat] ) return nPriorSat;
  3103         -
  3104         -  /* Either the outer queries must generate rows where there are no two
  3105         -  ** rows with the same values in all ORDER BY columns, or else this
  3106         -  ** loop must generate just a single row of output.  Example:  Suppose
  3107         -  ** the outer loops generate A=1 and A=1, and this loop generates B=3
  3108         -  ** and B=4.  Then without the following test, ORDER BY A,B would 
  3109         -  ** generate the wrong order output: 1,3 1,4 1,3 1,4
  3110         -  */
  3111         -  if( outerObUnique==0 && uniqueNotNull==0 ) return nPriorSat;
  3112         -  *pbObUnique = uniqueNotNull;
  3113         -
  3114         -  /* Return the necessary scan order back to the caller */
  3115         -  *pbRev = sortOrder & 1;
  3116         -
  3117         -  /* If there was an "ORDER BY rowid" term that matched, or it is only
  3118         -  ** possible for a single row from this table to match, then skip over
  3119         -  ** any additional ORDER BY terms dealing with this table.
  3120         -  */
  3121         -  if( uniqueNotNull ){
  3122         -    /* Advance j over additional ORDER BY terms associated with base */
  3123         -    WhereMaskSet *pMS = p->pWC->pMaskSet;
  3124         -    Bitmask m = ~getMask(pMS, base);
  3125         -    while( j<nTerm && (exprTableUsage(pMS, pOrderBy->a[j].pExpr)&m)==0 ){
  3126         -      j++;
  3127         -    }
  3128         -  }
  3129         -  return j;
  3130         -}
  3131         -
  3132         -/*
  3133         -** Find the best query plan for accessing a particular table.  Write the
  3134         -** best query plan and its cost into the p->cost.
  3135         -**
  3136         -** The lowest cost plan wins.  The cost is an estimate of the amount of
  3137         -** CPU and disk I/O needed to process the requested result.
  3138         -** Factors that influence cost include:
  3139         -**
  3140         -**    *  The estimated number of rows that will be retrieved.  (The
  3141         -**       fewer the better.)
  3142         -**
  3143         -**    *  Whether or not sorting must occur.
  3144         -**
  3145         -**    *  Whether or not there must be separate lookups in the
  3146         -**       index and in the main table.
  3147         -**
  3148         -** If there was an INDEXED BY clause (pSrc->pIndex) attached to the table in
  3149         -** the SQL statement, then this function only considers plans using the 
  3150         -** named index. If no such plan is found, then the returned cost is
  3151         -** SQLITE_BIG_DBL. If a plan is found that uses the named index, 
  3152         -** then the cost is calculated in the usual way.
  3153         -**
  3154         -** If a NOT INDEXED clause was attached to the table 
  3155         -** in the SELECT statement, then no indexes are considered. However, the 
  3156         -** selected plan may still take advantage of the built-in rowid primary key
  3157         -** index.
  3158         -*/
  3159         -static void bestBtreeIndex(WhereBestIdx *p){
  3160         -  Parse *pParse = p->pParse;  /* The parsing context */
  3161         -  WhereClause *pWC = p->pWC;  /* The WHERE clause */
  3162         -  struct SrcList_item *pSrc = p->pSrc; /* The FROM clause term to search */
  3163         -  int iCur = pSrc->iCursor;   /* The cursor of the table to be accessed */
  3164         -  Index *pProbe;              /* An index we are evaluating */
  3165         -  Index *pIdx;                /* Copy of pProbe, or zero for IPK index */
  3166         -  int eqTermMask;             /* Current mask of valid equality operators */
  3167         -  int idxEqTermMask;          /* Index mask of valid equality operators */
  3168         -  Index sPk;                  /* A fake index object for the primary key */
  3169         -  tRowcnt aiRowEstPk[2];      /* The aiRowEst[] value for the sPk index */
  3170         -  int aiColumnPk = -1;        /* The aColumn[] value for the sPk index */
  3171         -  int wsFlagMask;             /* Allowed flags in p->cost.plan.wsFlag */
  3172         -  int nPriorSat;              /* ORDER BY terms satisfied by outer loops */
  3173         -  int nOrderBy;               /* Number of ORDER BY terms */
  3174         -  char bSortInit;             /* Initializer for bSort in inner loop */
  3175         -  char bDistInit;             /* Initializer for bDist in inner loop */
  3176         -
  3177         -
  3178         -  /* Initialize the cost to a worst-case value */
  3179         -  memset(&p->cost, 0, sizeof(p->cost));
  3180         -  p->cost.rCost = SQLITE_BIG_DBL;
  3181         -
  3182         -  /* If the pSrc table is the right table of a LEFT JOIN then we may not
  3183         -  ** use an index to satisfy IS NULL constraints on that table.  This is
  3184         -  ** because columns might end up being NULL if the table does not match -
  3185         -  ** a circumstance which the index cannot help us discover.  Ticket #2177.
  3186         -  */
  3187         -  if( pSrc->jointype & JT_LEFT ){
  3188         -    idxEqTermMask = WO_EQ|WO_IN;
  3189         -  }else{
  3190         -    idxEqTermMask = WO_EQ|WO_IN|WO_ISNULL;
  3191         -  }
  3192         -
  3193         -  if( pSrc->pIndex ){
  3194         -    /* An INDEXED BY clause specifies a particular index to use */
  3195         -    pIdx = pProbe = pSrc->pIndex;
  3196         -    wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE);
  3197         -    eqTermMask = idxEqTermMask;
  3198         -  }else{
  3199         -    /* There is no INDEXED BY clause.  Create a fake Index object in local
  3200         -    ** variable sPk to represent the rowid primary key index.  Make this
  3201         -    ** fake index the first in a chain of Index objects with all of the real
  3202         -    ** indices to follow */
  3203         -    Index *pFirst;                  /* First of real indices on the table */
  3204         -    memset(&sPk, 0, sizeof(Index));
  3205         -    sPk.nColumn = 1;
  3206         -    sPk.aiColumn = &aiColumnPk;
  3207         -    sPk.aiRowEst = aiRowEstPk;
  3208         -    sPk.onError = OE_Replace;
  3209         -    sPk.pTable = pSrc->pTab;
  3210         -    aiRowEstPk[0] = pSrc->pTab->nRowEst;
  3211         -    aiRowEstPk[1] = 1;
  3212         -    pFirst = pSrc->pTab->pIndex;
  3213         -    if( pSrc->notIndexed==0 ){
  3214         -      /* The real indices of the table are only considered if the
  3215         -      ** NOT INDEXED qualifier is omitted from the FROM clause */
  3216         -      sPk.pNext = pFirst;
  3217         -    }
  3218         -    pProbe = &sPk;
  3219         -    wsFlagMask = ~(
  3220         -        WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE
  3221         -    );
  3222         -    eqTermMask = WO_EQ|WO_IN;
  3223         -    pIdx = 0;
  3224         -  }
  3225         -
  3226         -  nOrderBy = p->pOrderBy ? p->pOrderBy->nExpr : 0;
  3227         -  if( p->i ){
  3228         -    nPriorSat = p->aLevel[p->i-1].plan.nOBSat;
  3229         -    bSortInit = nPriorSat<nOrderBy;
  3230         -    bDistInit = 0;
  3231         -  }else{
  3232         -    nPriorSat = 0;
  3233         -    bSortInit = nOrderBy>0;
  3234         -    bDistInit = p->pDistinct!=0;
  3235         -  }
  3236         -
  3237         -  /* Loop over all indices looking for the best one to use
  3238         -  */
  3239         -  for(; pProbe; pIdx=pProbe=pProbe->pNext){
  3240         -    const tRowcnt * const aiRowEst = pProbe->aiRowEst;
  3241         -    WhereCost pc;               /* Cost of using pProbe */
  3242         -    double log10N = (double)1;  /* base-10 logarithm of nRow (inexact) */
  3243         -
  3244         -    /* The following variables are populated based on the properties of
  3245         -    ** index being evaluated. They are then used to determine the expected
  3246         -    ** cost and number of rows returned.
  3247         -    **
  3248         -    **  pc.plan.nEq: 
  3249         -    **    Number of equality terms that can be implemented using the index.
  3250         -    **    In other words, the number of initial fields in the index that
  3251         -    **    are used in == or IN or NOT NULL constraints of the WHERE clause.
  3252         -    **
  3253         -    **  nInMul:  
  3254         -    **    The "in-multiplier". This is an estimate of how many seek operations 
  3255         -    **    SQLite must perform on the index in question. For example, if the 
  3256         -    **    WHERE clause is:
  3257         -    **
  3258         -    **      WHERE a IN (1, 2, 3) AND b IN (4, 5, 6)
  3259         -    **
  3260         -    **    SQLite must perform 9 lookups on an index on (a, b), so nInMul is 
  3261         -    **    set to 9. Given the same schema and either of the following WHERE 
  3262         -    **    clauses:
  3263         -    **
  3264         -    **      WHERE a =  1
  3265         -    **      WHERE a >= 2
  3266         -    **
  3267         -    **    nInMul is set to 1.
  3268         -    **
  3269         -    **    If there exists a WHERE term of the form "x IN (SELECT ...)", then 
  3270         -    **    the sub-select is assumed to return 25 rows for the purposes of 
  3271         -    **    determining nInMul.
  3272         -    **
  3273         -    **  bInEst:  
  3274         -    **    Set to true if there was at least one "x IN (SELECT ...)" term used 
  3275         -    **    in determining the value of nInMul.  Note that the RHS of the
  3276         -    **    IN operator must be a SELECT, not a value list, for this variable
  3277         -    **    to be true.
  3278         -    **
  3279         -    **  rangeDiv:
  3280         -    **    An estimate of a divisor by which to reduce the search space due
  3281         -    **    to inequality constraints.  In the absence of sqlite_stat3 ANALYZE
  3282         -    **    data, a single inequality reduces the search space to 1/4rd its
  3283         -    **    original size (rangeDiv==4).  Two inequalities reduce the search
  3284         -    **    space to 1/16th of its original size (rangeDiv==16).
  3285         -    **
  3286         -    **  bSort:   
  3287         -    **    Boolean. True if there is an ORDER BY clause that will require an 
  3288         -    **    external sort (i.e. scanning the index being evaluated will not 
  3289         -    **    correctly order records).
  3290         -    **
  3291         -    **  bDist:
  3292         -    **    Boolean. True if there is a DISTINCT clause that will require an 
  3293         -    **    external btree.
  3294         -    **
  3295         -    **  bLookup: 
  3296         -    **    Boolean. True if a table lookup is required for each index entry
  3297         -    **    visited.  In other words, true if this is not a covering index.
  3298         -    **    This is always false for the rowid primary key index of a table.
  3299         -    **    For other indexes, it is true unless all the columns of the table
  3300         -    **    used by the SELECT statement are present in the index (such an
  3301         -    **    index is sometimes described as a covering index).
  3302         -    **    For example, given the index on (a, b), the second of the following 
  3303         -    **    two queries requires table b-tree lookups in order to find the value
  3304         -    **    of column c, but the first does not because columns a and b are
  3305         -    **    both available in the index.
  3306         -    **
  3307         -    **             SELECT a, b    FROM tbl WHERE a = 1;
  3308         -    **             SELECT a, b, c FROM tbl WHERE a = 1;
  3309         -    */
  3310         -    int bInEst = 0;               /* True if "x IN (SELECT...)" seen */
  3311         -    int nInMul = 1;               /* Number of distinct equalities to lookup */
  3312         -    double rangeDiv = (double)1;  /* Estimated reduction in search space */
  3313         -    int nBound = 0;               /* Number of range constraints seen */
  3314         -    char bSort = bSortInit;       /* True if external sort required */
  3315         -    char bDist = bDistInit;       /* True if index cannot help with DISTINCT */
  3316         -    char bLookup = 0;             /* True if not a covering index */
  3317         -    WhereTerm *pTerm;             /* A single term of the WHERE clause */
  3318         -#ifdef SQLITE_ENABLE_STAT3
  3319         -    WhereTerm *pFirstTerm = 0;    /* First term matching the index */
  3320         -#endif
  3321         -
  3322         -    WHERETRACE((
  3323         -      "   %s(%s):\n",
  3324         -      pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk")
  3325         -    ));
  3326         -    memset(&pc, 0, sizeof(pc));
  3327         -    pc.plan.nOBSat = nPriorSat;
  3328         -
  3329         -    /* Determine the values of pc.plan.nEq and nInMul */
  3330         -    for(pc.plan.nEq=0; pc.plan.nEq<pProbe->nColumn; pc.plan.nEq++){
  3331         -      int j = pProbe->aiColumn[pc.plan.nEq];
  3332         -      pTerm = findTerm(pWC, iCur, j, p->notReady, eqTermMask, pIdx);
  3333         -      if( pTerm==0 ) break;
  3334         -      pc.plan.wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ);
  3335         -      testcase( pTerm->pWC!=pWC );
  3336         -      if( pTerm->eOperator & WO_IN ){
  3337         -        Expr *pExpr = pTerm->pExpr;
  3338         -        pc.plan.wsFlags |= WHERE_COLUMN_IN;
  3339         -        if( ExprHasProperty(pExpr, EP_xIsSelect) ){
  3340         -          /* "x IN (SELECT ...)":  Assume the SELECT returns 25 rows */
  3341         -          nInMul *= 25;
  3342         -          bInEst = 1;
  3343         -        }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
  3344         -          /* "x IN (value, value, ...)" */
  3345         -          nInMul *= pExpr->x.pList->nExpr;
  3346         -        }
  3347         -      }else if( pTerm->eOperator & WO_ISNULL ){
  3348         -        pc.plan.wsFlags |= WHERE_COLUMN_NULL;
  3349         -      }
  3350         -#ifdef SQLITE_ENABLE_STAT3
  3351         -      if( pc.plan.nEq==0 && pProbe->aSample ) pFirstTerm = pTerm;
  3352         -#endif
  3353         -      pc.used |= pTerm->prereqRight;
  3354         -    }
  3355         - 
  3356         -    /* If the index being considered is UNIQUE, and there is an equality 
  3357         -    ** constraint for all columns in the index, then this search will find
  3358         -    ** at most a single row. In this case set the WHERE_UNIQUE flag to 
  3359         -    ** indicate this to the caller.
  3360         -    **
  3361         -    ** Otherwise, if the search may find more than one row, test to see if
  3362         -    ** there is a range constraint on indexed column (pc.plan.nEq+1) that
  3363         -    ** can be optimized using the index. 
  3364         -    */
  3365         -    if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){
  3366         -      testcase( pc.plan.wsFlags & WHERE_COLUMN_IN );
  3367         -      testcase( pc.plan.wsFlags & WHERE_COLUMN_NULL );
  3368         -      if( (pc.plan.wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){
  3369         -        pc.plan.wsFlags |= WHERE_UNIQUE;
  3370         -        if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){
  3371         -          pc.plan.wsFlags |= WHERE_ALL_UNIQUE;
  3372         -        }
  3373         -      }
  3374         -    }else if( pProbe->bUnordered==0 ){
  3375         -      int j;
  3376         -      j = (pc.plan.nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[pc.plan.nEq]);
  3377         -      if( findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){
  3378         -        WhereTerm *pTop, *pBtm;
  3379         -        pTop = findTerm(pWC, iCur, j, p->notReady, WO_LT|WO_LE, pIdx);
  3380         -        pBtm = findTerm(pWC, iCur, j, p->notReady, WO_GT|WO_GE, pIdx);
  3381         -        whereRangeScanEst(pParse, pProbe, pc.plan.nEq, pBtm, pTop, &rangeDiv);
  3382         -        if( pTop ){
  3383         -          nBound = 1;
  3384         -          pc.plan.wsFlags |= WHERE_TOP_LIMIT;
  3385         -          pc.used |= pTop->prereqRight;
  3386         -          testcase( pTop->pWC!=pWC );
  3387         -        }
  3388         -        if( pBtm ){
  3389         -          nBound++;
  3390         -          pc.plan.wsFlags |= WHERE_BTM_LIMIT;
  3391         -          pc.used |= pBtm->prereqRight;
  3392         -          testcase( pBtm->pWC!=pWC );
  3393         -        }
  3394         -        pc.plan.wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE);
  3395         -      }
  3396         -    }
  3397         -
  3398         -    /* If there is an ORDER BY clause and the index being considered will
  3399         -    ** naturally scan rows in the required order, set the appropriate flags
  3400         -    ** in pc.plan.wsFlags. Otherwise, if there is an ORDER BY clause but
  3401         -    ** the index will scan rows in a different order, set the bSort
  3402         -    ** variable.  */
  3403         -    if( bSort && (pSrc->jointype & JT_LEFT)==0 ){
  3404         -      int bRev = 2;
  3405         -      int bObUnique = 0;
  3406         -      WHERETRACE(("      --> before isSortIndex: nPriorSat=%d\n",nPriorSat));
  3407         -      pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev, &bObUnique);
  3408         -      WHERETRACE(("      --> after  isSortIndex: bRev=%d bObU=%d nOBSat=%d\n",
  3409         -                  bRev, bObUnique, pc.plan.nOBSat));
  3410         -      if( nPriorSat<pc.plan.nOBSat || (pc.plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){
  3411         -        pc.plan.wsFlags |= WHERE_ORDERED;
  3412         -        if( bObUnique ) pc.plan.wsFlags |= WHERE_OB_UNIQUE;
  3413         -      }
  3414         -      if( nOrderBy==pc.plan.nOBSat ){
  3415         -        bSort = 0;
  3416         -        pc.plan.wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE;
  3417         -      }
  3418         -      if( bRev & 1 ) pc.plan.wsFlags |= WHERE_REVERSE;
  3419         -    }
  3420         -
  3421         -    /* If there is a DISTINCT qualifier and this index will scan rows in
  3422         -    ** order of the DISTINCT expressions, clear bDist and set the appropriate
  3423         -    ** flags in pc.plan.wsFlags. */
  3424         -    if( bDist
  3425         -     && isDistinctIndex(pParse, pWC, pProbe, iCur, p->pDistinct, pc.plan.nEq)
  3426         -     && (pc.plan.wsFlags & WHERE_COLUMN_IN)==0
  3427         -    ){
  3428         -      bDist = 0;
  3429         -      pc.plan.wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT;
  3430         -    }
  3431         -
  3432         -    /* If currently calculating the cost of using an index (not the IPK
  3433         -    ** index), determine if all required column data may be obtained without 
  3434         -    ** using the main table (i.e. if the index is a covering
  3435         -    ** index for this query). If it is, set the WHERE_IDX_ONLY flag in
  3436         -    ** pc.plan.wsFlags. Otherwise, set the bLookup variable to true.  */
  3437         -    if( pIdx ){
  3438         -      Bitmask m = pSrc->colUsed;
  3439         -      int j;
  3440         -      for(j=0; j<pIdx->nColumn; j++){
  3441         -        int x = pIdx->aiColumn[j];
  3442         -        if( x<BMS-1 ){
  3443         -          m &= ~(((Bitmask)1)<<x);
  3444         -        }
  3445         -      }
  3446         -      if( m==0 ){
  3447         -        pc.plan.wsFlags |= WHERE_IDX_ONLY;
  3448         -      }else{
  3449         -        bLookup = 1;
  3450         -      }
  3451         -    }
  3452         -
  3453         -    /*
  3454         -    ** Estimate the number of rows of output.  For an "x IN (SELECT...)"
  3455         -    ** constraint, do not let the estimate exceed half the rows in the table.
  3456         -    */
  3457         -    pc.plan.nRow = (double)(aiRowEst[pc.plan.nEq] * nInMul);
  3458         -    if( bInEst && pc.plan.nRow*2>aiRowEst[0] ){
  3459         -      pc.plan.nRow = aiRowEst[0]/2;
  3460         -      nInMul = (int)(pc.plan.nRow / aiRowEst[pc.plan.nEq]);
  3461         -    }
  3462         -
  3463         -#ifdef SQLITE_ENABLE_STAT3
  3464         -    /* If the constraint is of the form x=VALUE or x IN (E1,E2,...)
  3465         -    ** and we do not think that values of x are unique and if histogram
  3466         -    ** data is available for column x, then it might be possible
  3467         -    ** to get a better estimate on the number of rows based on
  3468         -    ** VALUE and how common that value is according to the histogram.
  3469         -    */
  3470         -    if( pc.plan.nRow>(double)1 && pc.plan.nEq==1
  3471         -     && pFirstTerm!=0 && aiRowEst[1]>1 ){
  3472         -      assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 );
  3473         -      if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){
  3474         -        testcase( pFirstTerm->eOperator & WO_EQ );
  3475         -        testcase( pFirstTerm->eOperator & WO_EQUIV );
  3476         -        testcase( pFirstTerm->eOperator & WO_ISNULL );
  3477         -        whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight,
  3478         -                          &pc.plan.nRow);
  3479         -      }else if( bInEst==0 ){
  3480         -        assert( pFirstTerm->eOperator & WO_IN );
  3481         -        whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList,
  3482         -                       &pc.plan.nRow);
  3483         -      }
  3484         -    }
  3485         -#endif /* SQLITE_ENABLE_STAT3 */
  3486         -
  3487         -    /* Adjust the number of output rows and downward to reflect rows
  3488         -    ** that are excluded by range constraints.
  3489         -    */
  3490         -    pc.plan.nRow = pc.plan.nRow/rangeDiv;
  3491         -    if( pc.plan.nRow<1 ) pc.plan.nRow = 1;
  3492         -
  3493         -    /* Experiments run on real SQLite databases show that the time needed
  3494         -    ** to do a binary search to locate a row in a table or index is roughly
  3495         -    ** log10(N) times the time to move from one row to the next row within
  3496         -    ** a table or index.  The actual times can vary, with the size of
  3497         -    ** records being an important factor.  Both moves and searches are
  3498         -    ** slower with larger records, presumably because fewer records fit
  3499         -    ** on one page and hence more pages have to be fetched.
  3500         -    **
  3501         -    ** The ANALYZE command and the sqlite_stat1 and sqlite_stat3 tables do
  3502         -    ** not give us data on the relative sizes of table and index records.
  3503         -    ** So this computation assumes table records are about twice as big
  3504         -    ** as index records
  3505         -    */
  3506         -    if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED|WHERE_OB_UNIQUE))
  3507         -                                                              ==WHERE_IDX_ONLY
  3508         -     && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
  3509         -     && sqlite3GlobalConfig.bUseCis
  3510         -     && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan)
  3511         -    ){
  3512         -      /* This index is not useful for indexing, but it is a covering index.
  3513         -      ** A full-scan of the index might be a little faster than a full-scan
  3514         -      ** of the table, so give this case a cost slightly less than a table
  3515         -      ** scan. */
  3516         -      pc.rCost = aiRowEst[0]*3 + pProbe->nColumn;
  3517         -      pc.plan.wsFlags |= WHERE_COVER_SCAN|WHERE_COLUMN_RANGE;
  3518         -    }else if( (pc.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){
  3519         -      /* The cost of a full table scan is a number of move operations equal
  3520         -      ** to the number of rows in the table.
  3521         -      **
  3522         -      ** We add an additional 4x penalty to full table scans.  This causes
  3523         -      ** the cost function to err on the side of choosing an index over
  3524         -      ** choosing a full scan.  This 4x full-scan penalty is an arguable
  3525         -      ** decision and one which we expect to revisit in the future.  But
  3526         -      ** it seems to be working well enough at the moment.
  3527         -      */
  3528         -      pc.rCost = aiRowEst[0]*4;
  3529         -      pc.plan.wsFlags &= ~WHERE_IDX_ONLY;
  3530         -      if( pIdx ){
  3531         -        pc.plan.wsFlags &= ~WHERE_ORDERED;
  3532         -        pc.plan.nOBSat = nPriorSat;
  3533         -      }
  3534         -    }else{
  3535         -      log10N = estLog(aiRowEst[0]);
  3536         -      pc.rCost = pc.plan.nRow;
  3537         -      if( pIdx ){
  3538         -        if( bLookup ){
  3539         -          /* For an index lookup followed by a table lookup:
  3540         -          **    nInMul index searches to find the start of each index range
  3541         -          **  + nRow steps through the index
  3542         -          **  + nRow table searches to lookup the table entry using the rowid
  3543         -          */
  3544         -          pc.rCost += (nInMul + pc.plan.nRow)*log10N;
  3545         -        }else{
  3546         -          /* For a covering index:
  3547         -          **     nInMul index searches to find the initial entry 
  3548         -          **   + nRow steps through the index
  3549         -          */
  3550         -          pc.rCost += nInMul*log10N;
  3551         -        }
  3552         -      }else{
  3553         -        /* For a rowid primary key lookup:
  3554         -        **    nInMult table searches to find the initial entry for each range
  3555         -        **  + nRow steps through the table
  3556         -        */
  3557         -        pc.rCost += nInMul*log10N;
  3558         -      }
  3559         -    }
  3560         -
  3561         -    /* Add in the estimated cost of sorting the result.  Actual experimental
  3562         -    ** measurements of sorting performance in SQLite show that sorting time
  3563         -    ** adds C*N*log10(N) to the cost, where N is the number of rows to be 
  3564         -    ** sorted and C is a factor between 1.95 and 4.3.  We will split the
  3565         -    ** difference and select C of 3.0.
  3566         -    */
  3567         -    if( bSort ){
  3568         -      double m = estLog(pc.plan.nRow*(nOrderBy - pc.plan.nOBSat)/nOrderBy);
  3569         -      m *= (double)(pc.plan.nOBSat ? 2 : 3);
  3570         -      pc.rCost += pc.plan.nRow*m;
  3571         -    }
  3572         -    if( bDist ){
  3573         -      pc.rCost += pc.plan.nRow*estLog(pc.plan.nRow)*3;
  3574         -    }
  3575         -
  3576         -    /**** Cost of using this index has now been computed ****/
  3577         -
  3578         -    /* If there are additional constraints on this table that cannot
  3579         -    ** be used with the current index, but which might lower the number
  3580         -    ** of output rows, adjust the nRow value accordingly.  This only 
  3581         -    ** matters if the current index is the least costly, so do not bother
  3582         -    ** with this step if we already know this index will not be chosen.
  3583         -    ** Also, never reduce the output row count below 2 using this step.
  3584         -    **
  3585         -    ** It is critical that the notValid mask be used here instead of
  3586         -    ** the notReady mask.  When computing an "optimal" index, the notReady
  3587         -    ** mask will only have one bit set - the bit for the current table.
  3588         -    ** The notValid mask, on the other hand, always has all bits set for
  3589         -    ** tables that are not in outer loops.  If notReady is used here instead
  3590         -    ** of notValid, then a optimal index that depends on inner joins loops
  3591         -    ** might be selected even when there exists an optimal index that has
  3592         -    ** no such dependency.
  3593         -    */
  3594         -    if( pc.plan.nRow>2 && pc.rCost<=p->cost.rCost ){
  3595         -      int k;                       /* Loop counter */
  3596         -      int nSkipEq = pc.plan.nEq;   /* Number of == constraints to skip */
  3597         -      int nSkipRange = nBound;     /* Number of < constraints to skip */
  3598         -      Bitmask thisTab;             /* Bitmap for pSrc */
  3599         -
  3600         -      thisTab = getMask(pWC->pMaskSet, iCur);
  3601         -      for(pTerm=pWC->a, k=pWC->nTerm; pc.plan.nRow>2 && k; k--, pTerm++){
  3602         -        if( pTerm->wtFlags & TERM_VIRTUAL ) continue;
  3603         -        if( (pTerm->prereqAll & p->notValid)!=thisTab ) continue;
  3604         -        if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){
  3605         -          if( nSkipEq ){
  3606         -            /* Ignore the first pc.plan.nEq equality matches since the index
  3607         -            ** has already accounted for these */
  3608         -            nSkipEq--;
  3609         -          }else{
  3610         -            /* Assume each additional equality match reduces the result
  3611         -            ** set size by a factor of 10 */
  3612         -            pc.plan.nRow /= 10;
  3613         -          }
  3614         -        }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){
  3615         -          if( nSkipRange ){
  3616         -            /* Ignore the first nSkipRange range constraints since the index
  3617         -            ** has already accounted for these */
  3618         -            nSkipRange--;
  3619         -          }else{
  3620         -            /* Assume each additional range constraint reduces the result
  3621         -            ** set size by a factor of 3.  Indexed range constraints reduce
  3622         -            ** the search space by a larger factor: 4.  We make indexed range
  3623         -            ** more selective intentionally because of the subjective 
  3624         -            ** observation that indexed range constraints really are more
  3625         -            ** selective in practice, on average. */
  3626         -            pc.plan.nRow /= 3;
  3627         -          }
  3628         -        }else if( (pTerm->eOperator & WO_NOOP)==0 ){
  3629         -          /* Any other expression lowers the output row count by half */
  3630         -          pc.plan.nRow /= 2;
  3631         -        }
  3632         -      }
  3633         -      if( pc.plan.nRow<2 ) pc.plan.nRow = 2;
  3634         -    }
  3635         -
  3636         -
  3637         -    WHERETRACE((
  3638         -      "      nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%08x\n"
  3639         -      "      notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f\n"
  3640         -      "      used=0x%llx nOBSat=%d\n",
  3641         -      pc.plan.nEq, nInMul, (int)rangeDiv, bSort, bLookup, pc.plan.wsFlags,
  3642         -      p->notReady, log10N, pc.plan.nRow, pc.rCost, pc.used,
  3643         -      pc.plan.nOBSat
  3644         -    ));
  3645         -
  3646         -    /* If this index is the best we have seen so far, then record this
  3647         -    ** index and its cost in the p->cost structure.
  3648         -    */
  3649         -    if( (!pIdx || pc.plan.wsFlags) && compareCost(&pc, &p->cost) ){
  3650         -      p->cost = pc;
  3651         -      p->cost.plan.wsFlags &= wsFlagMask;
  3652         -      p->cost.plan.u.pIdx = pIdx;
  3653         -    }
  3654         -
  3655         -    /* If there was an INDEXED BY clause, then only that one index is
  3656         -    ** considered. */
  3657         -    if( pSrc->pIndex ) break;
  3658         -
  3659         -    /* Reset masks for the next index in the loop */
  3660         -    wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE);
  3661         -    eqTermMask = idxEqTermMask;
  3662         -  }
  3663         -
  3664         -  /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag
  3665         -  ** is set, then reverse the order that the index will be scanned
  3666         -  ** in. This is used for application testing, to help find cases
  3667         -  ** where application behavior depends on the (undefined) order that
  3668         -  ** SQLite outputs rows in in the absence of an ORDER BY clause.  */
  3669         -  if( !p->pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){
  3670         -    p->cost.plan.wsFlags |= WHERE_REVERSE;
  3671         -  }
  3672         -
  3673         -  assert( p->pOrderBy || (p->cost.plan.wsFlags&WHERE_ORDERED)==0 );
  3674         -  assert( p->cost.plan.u.pIdx==0 || (p->cost.plan.wsFlags&WHERE_ROWID_EQ)==0 );
  3675         -  assert( pSrc->pIndex==0 
  3676         -       || p->cost.plan.u.pIdx==0 
  3677         -       || p->cost.plan.u.pIdx==pSrc->pIndex 
  3678         -  );
  3679         -
  3680         -  WHERETRACE(("   best index is %s cost=%.1f\n",
  3681         -         p->cost.plan.u.pIdx ? p->cost.plan.u.pIdx->zName : "ipk",
  3682         -         p->cost.rCost));
  3683         -  
  3684         -  bestOrClauseIndex(p);
  3685         -  bestAutomaticIndex(p);
  3686         -  p->cost.plan.wsFlags |= eqTermMask;
  3687         -}
  3688         -
  3689         -/*
  3690         -** Find the query plan for accessing table pSrc->pTab. Write the
  3691         -** best query plan and its cost into the WhereCost object supplied 
  3692         -** as the last parameter. This function may calculate the cost of
  3693         -** both real and virtual table scans.
  3694         -**
  3695         -** This function does not take ORDER BY or DISTINCT into account.  Nor
  3696         -** does it remember the virtual table query plan.  All it does is compute
  3697         -** the cost while determining if an OR optimization is applicable.  The
  3698         -** details will be reconsidered later if the optimization is found to be
  3699         -** applicable.
  3700         -*/
  3701         -static void bestIndex(WhereBestIdx *p){
  3702         -#ifndef SQLITE_OMIT_VIRTUALTABLE
  3703         -  if( IsVirtual(p->pSrc->pTab) ){
  3704         -    sqlite3_index_info *pIdxInfo = 0;
  3705         -    p->ppIdxInfo = &pIdxInfo;
  3706         -    bestVirtualIndex(p);
  3707         -    assert( pIdxInfo!=0 || p->pParse->db->mallocFailed );
  3708         -    if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){
  3709         -      sqlite3_free(pIdxInfo->idxStr);
  3710         -    }
  3711         -    sqlite3DbFree(p->pParse->db, pIdxInfo);
  3712         -  }else
  3713         -#endif
  3714         -  {
  3715         -    bestBtreeIndex(p);
  3716         -  }
  3717         -}
  3718         -
  3719   2806   /*
  3720   2807   ** Disable a term in the WHERE clause.  Except, do not disable the term
  3721   2808   ** if it controls a LEFT OUTER JOIN and it did not originate in the ON
  3722   2809   ** or USING clause of that join.
  3723   2810   **
  3724   2811   ** Consider the term t2.z='ok' in the following queries:
  3725   2812   **
................................................................................
  3809   2896   ** this routine sets up a loop that will iterate over all values of X.
  3810   2897   */
  3811   2898   static int codeEqualityTerm(
  3812   2899     Parse *pParse,      /* The parsing context */
  3813   2900     WhereTerm *pTerm,   /* The term of the WHERE clause to be coded */
  3814   2901     WhereLevel *pLevel, /* The level of the FROM clause we are working on */
  3815   2902     int iEq,            /* Index of the equality term within this level */
         2903  +  int bRev,           /* True for reverse-order IN operations */
  3816   2904     int iTarget         /* Attempt to leave results in this register */
  3817   2905   ){
  3818   2906     Expr *pX = pTerm->pExpr;
  3819   2907     Vdbe *v = pParse->pVdbe;
  3820   2908     int iReg;                  /* Register holding results */
  3821   2909   
  3822   2910     assert( iTarget>0 );
................................................................................
  3826   2914       iReg = iTarget;
  3827   2915       sqlite3VdbeAddOp2(v, OP_Null, 0, iReg);
  3828   2916   #ifndef SQLITE_OMIT_SUBQUERY
  3829   2917     }else{
  3830   2918       int eType;
  3831   2919       int iTab;
  3832   2920       struct InLoop *pIn;
  3833         -    u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0;
         2921  +    WhereLoop *pLoop = pLevel->pWLoop;
  3834   2922   
  3835         -    if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 
  3836         -      && pLevel->plan.u.pIdx->aSortOrder[iEq]
         2923  +    if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
         2924  +      && pLoop->u.btree.pIndex!=0
         2925  +      && pLoop->u.btree.pIndex->aSortOrder[iEq]
  3837   2926       ){
  3838   2927         testcase( iEq==0 );
  3839         -      testcase( iEq==pLevel->plan.u.pIdx->nColumn-1 );
  3840         -      testcase( iEq>0 && iEq+1<pLevel->plan.u.pIdx->nColumn );
  3841   2928         testcase( bRev );
  3842   2929         bRev = !bRev;
  3843   2930       }
  3844   2931       assert( pX->op==TK_IN );
  3845   2932       iReg = iTarget;
  3846   2933       eType = sqlite3FindInIndex(pParse, pX, 0);
  3847   2934       if( eType==IN_INDEX_INDEX_DESC ){
  3848   2935         testcase( bRev );
  3849   2936         bRev = !bRev;
  3850   2937       }
  3851   2938       iTab = pX->iTable;
  3852   2939       sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0);
  3853         -    assert( pLevel->plan.wsFlags & WHERE_IN_ABLE );
         2940  +    assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 );
         2941  +    pLoop->wsFlags |= WHERE_IN_ABLE;
  3854   2942       if( pLevel->u.in.nIn==0 ){
  3855   2943         pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
  3856   2944       }
  3857   2945       pLevel->u.in.nIn++;
  3858   2946       pLevel->u.in.aInLoop =
  3859   2947          sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop,
  3860   2948                                 sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn);
................................................................................
  3916   3004   ** no conversion should be attempted before using a t2.b value as part of
  3917   3005   ** a key to search the index. Hence the first byte in the returned affinity
  3918   3006   ** string in this example would be set to SQLITE_AFF_NONE.
  3919   3007   */
  3920   3008   static int codeAllEqualityTerms(
  3921   3009     Parse *pParse,        /* Parsing context */
  3922   3010     WhereLevel *pLevel,   /* Which nested loop of the FROM we are coding */
  3923         -  WhereClause *pWC,     /* The WHERE clause */
  3924         -  Bitmask notReady,     /* Which parts of FROM have not yet been coded */
         3011  +  int bRev,             /* Reverse the order of IN operators */
  3925   3012     int nExtraReg,        /* Number of extra registers to allocate */
  3926   3013     char **pzAff          /* OUT: Set to point to affinity string */
  3927   3014   ){
  3928         -  int nEq = pLevel->plan.nEq;   /* The number of == or IN constraints to code */
         3015  +  int nEq;                      /* The number of == or IN constraints to code */
  3929   3016     Vdbe *v = pParse->pVdbe;      /* The vm under construction */
  3930   3017     Index *pIdx;                  /* The index being used for this loop */
  3931         -  int iCur = pLevel->iTabCur;   /* The cursor of the table */
  3932   3018     WhereTerm *pTerm;             /* A single constraint term */
         3019  +  WhereLoop *pLoop;             /* The WhereLoop object */
  3933   3020     int j;                        /* Loop counter */
  3934   3021     int regBase;                  /* Base register */
  3935   3022     int nReg;                     /* Number of registers to allocate */
  3936   3023     char *zAff;                   /* Affinity string to return */
  3937   3024   
  3938   3025     /* This module is only called on query plans that use an index. */
  3939         -  assert( pLevel->plan.wsFlags & WHERE_INDEXED );
  3940         -  pIdx = pLevel->plan.u.pIdx;
         3026  +  pLoop = pLevel->pWLoop;
         3027  +  assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 );
         3028  +  nEq = pLoop->u.btree.nEq;
         3029  +  pIdx = pLoop->u.btree.pIndex;
         3030  +  assert( pIdx!=0 );
  3941   3031   
  3942   3032     /* Figure out how many memory cells we will need then allocate them.
  3943   3033     */
  3944   3034     regBase = pParse->nMem + 1;
  3945         -  nReg = pLevel->plan.nEq + nExtraReg;
         3035  +  nReg = pLoop->u.btree.nEq + nExtraReg;
  3946   3036     pParse->nMem += nReg;
  3947   3037   
  3948   3038     zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx));
  3949   3039     if( !zAff ){
  3950   3040       pParse->db->mallocFailed = 1;
  3951   3041     }
  3952   3042   
  3953   3043     /* Evaluate the equality constraints
  3954   3044     */
  3955   3045     assert( pIdx->nColumn>=nEq );
  3956   3046     for(j=0; j<nEq; j++){
  3957   3047       int r1;
  3958         -    int k = pIdx->aiColumn[j];
  3959         -    pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx);
  3960         -    if( pTerm==0 ) break;
         3048  +    pTerm = pLoop->aLTerm[j];
         3049  +    assert( pTerm!=0 );
  3961   3050       /* The following true for indices with redundant columns. 
  3962   3051       ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */
  3963   3052       testcase( (pTerm->wtFlags & TERM_CODED)!=0 );
  3964   3053       testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */
  3965         -    r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, regBase+j);
         3054  +    r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j);
  3966   3055       if( r1!=regBase+j ){
  3967   3056         if( nReg==1 ){
  3968   3057           sqlite3ReleaseTempReg(pParse, regBase);
  3969   3058           regBase = r1;
  3970   3059         }else{
  3971   3060           sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j);
  3972   3061         }
................................................................................
  4026   3115   **
  4027   3116   **   "a=? AND b>?"
  4028   3117   **
  4029   3118   ** The returned pointer points to memory obtained from sqlite3DbMalloc().
  4030   3119   ** It is the responsibility of the caller to free the buffer when it is
  4031   3120   ** no longer required.
  4032   3121   */
  4033         -static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){
  4034         -  WherePlan *pPlan = &pLevel->plan;
  4035         -  Index *pIndex = pPlan->u.pIdx;
  4036         -  int nEq = pPlan->nEq;
         3122  +static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){
         3123  +  Index *pIndex = pLoop->u.btree.pIndex;
         3124  +  int nEq = pLoop->u.btree.nEq;
  4037   3125     int i, j;
  4038   3126     Column *aCol = pTab->aCol;
  4039   3127     int *aiColumn = pIndex->aiColumn;
  4040   3128     StrAccum txt;
  4041   3129   
  4042         -  if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){
         3130  +  if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){
  4043   3131       return 0;
  4044   3132     }
  4045   3133     sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH);
  4046   3134     txt.db = db;
  4047   3135     sqlite3StrAccumAppend(&txt, " (", 2);
  4048   3136     for(i=0; i<nEq; i++){
  4049   3137       explainAppendTerm(&txt, i, aCol[aiColumn[i]].zName, "=");
  4050   3138     }
  4051   3139   
  4052   3140     j = i;
  4053         -  if( pPlan->wsFlags&WHERE_BTM_LIMIT ){
         3141  +  if( pLoop->wsFlags&WHERE_BTM_LIMIT ){
  4054   3142       char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName;
  4055   3143       explainAppendTerm(&txt, i++, z, ">");
  4056   3144     }
  4057         -  if( pPlan->wsFlags&WHERE_TOP_LIMIT ){
         3145  +  if( pLoop->wsFlags&WHERE_TOP_LIMIT ){
  4058   3146       char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName;
  4059   3147       explainAppendTerm(&txt, i, z, "<");
  4060   3148     }
  4061   3149     sqlite3StrAccumAppend(&txt, ")", 1);
  4062   3150     return sqlite3StrAccumFinish(&txt);
  4063   3151   }
  4064   3152   
................................................................................
  4073   3161     SrcList *pTabList,              /* Table list this loop refers to */
  4074   3162     WhereLevel *pLevel,             /* Scan to write OP_Explain opcode for */
  4075   3163     int iLevel,                     /* Value for "level" column of output */
  4076   3164     int iFrom,                      /* Value for "from" column of output */
  4077   3165     u16 wctrlFlags                  /* Flags passed to sqlite3WhereBegin() */
  4078   3166   ){
  4079   3167     if( pParse->explain==2 ){
  4080         -    u32 flags = pLevel->plan.wsFlags;
  4081   3168       struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
  4082   3169       Vdbe *v = pParse->pVdbe;      /* VM being constructed */
  4083   3170       sqlite3 *db = pParse->db;     /* Database handle */
  4084   3171       char *zMsg;                   /* Text to add to EQP output */
  4085         -    sqlite3_int64 nRow;           /* Expected number of rows visited by scan */
  4086   3172       int iId = pParse->iSelectId;  /* Select id (left-most output column) */
  4087   3173       int isSearch;                 /* True for a SEARCH. False for SCAN. */
         3174  +    WhereLoop *pLoop;             /* The controlling WhereLoop object */
         3175  +    u32 flags;                    /* Flags that describe this loop */
  4088   3176   
         3177  +    pLoop = pLevel->pWLoop;
         3178  +    flags = pLoop->wsFlags;
  4089   3179       if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return;
  4090   3180   
  4091         -    isSearch = (pLevel->plan.nEq>0)
  4092         -             || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
  4093         -             || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
         3181  +    isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
         3182  +            || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
         3183  +            || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
  4094   3184   
  4095   3185       zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN");
  4096   3186       if( pItem->pSelect ){
  4097   3187         zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId);
  4098   3188       }else{
  4099   3189         zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName);
  4100   3190       }
  4101   3191   
  4102   3192       if( pItem->zAlias ){
  4103   3193         zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
  4104   3194       }
  4105         -    if( (flags & WHERE_INDEXED)!=0 ){
  4106         -      char *zWhere = explainIndexRange(db, pLevel, pItem->pTab);
  4107         -      zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, 
  4108         -          ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""),
  4109         -          ((flags & WHERE_IDX_ONLY)?"COVERING ":""),
  4110         -          ((flags & WHERE_TEMP_INDEX)?"":" "),
  4111         -          ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName),
  4112         -          zWhere
  4113         -      );
         3195  +    if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0
         3196  +     && ALWAYS(pLoop->u.btree.pIndex!=0)
         3197  +    ){
         3198  +      char *zWhere = explainIndexRange(db, pLoop, pItem->pTab);
         3199  +      zMsg = sqlite3MAppendf(db, zMsg,
         3200  +               ((flags & WHERE_AUTO_INDEX) ? 
         3201  +                   "%s USING AUTOMATIC %sINDEX%.0s%s" :
         3202  +                   "%s USING %sINDEX %s%s"), 
         3203  +               zMsg, ((flags & WHERE_IDX_ONLY) ? "COVERING " : ""),
         3204  +               pLoop->u.btree.pIndex->zName, zWhere);
  4114   3205         sqlite3DbFree(db, zWhere);
  4115         -    }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){
         3206  +    }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){
  4116   3207         zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg);
  4117   3208   
  4118         -      if( flags&WHERE_ROWID_EQ ){
         3209  +      if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){
  4119   3210           zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg);
  4120   3211         }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
  4121   3212           zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid<?)", zMsg);
  4122   3213         }else if( flags&WHERE_BTM_LIMIT ){
  4123   3214           zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>?)", zMsg);
  4124         -      }else if( flags&WHERE_TOP_LIMIT ){
         3215  +      }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){
  4125   3216           zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid<?)", zMsg);
  4126   3217         }
  4127   3218       }
  4128   3219   #ifndef SQLITE_OMIT_VIRTUALTABLE
  4129   3220       else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
  4130         -      sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
  4131   3221         zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
  4132         -                  pVtabIdx->idxNum, pVtabIdx->idxStr);
         3222  +                  pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
  4133   3223       }
  4134   3224   #endif
  4135         -    if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){
  4136         -      testcase( wctrlFlags & WHERE_ORDERBY_MIN );
  4137         -      nRow = 1;
  4138         -    }else{
  4139         -      nRow = (sqlite3_int64)pLevel->plan.nRow;
  4140         -    }
  4141         -    zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow);
         3225  +    zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg);
  4142   3226       sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC);
  4143   3227     }
  4144   3228   }
  4145   3229   #else
  4146   3230   # define explainOneScan(u,v,w,x,y,z)
  4147   3231   #endif /* SQLITE_OMIT_EXPLAIN */
  4148   3232   
................................................................................
  4150   3234   /*
  4151   3235   ** Generate code for the start of the iLevel-th loop in the WHERE clause
  4152   3236   ** implementation described by pWInfo.
  4153   3237   */
  4154   3238   static Bitmask codeOneLoopStart(
  4155   3239     WhereInfo *pWInfo,   /* Complete information about the WHERE clause */
  4156   3240     int iLevel,          /* Which level of pWInfo->a[] should be coded */
  4157         -  u16 wctrlFlags,      /* One of the WHERE_* flags defined in sqliteInt.h */
  4158   3241     Bitmask notReady     /* Which tables are currently available */
  4159   3242   ){
  4160   3243     int j, k;            /* Loop counters */
  4161   3244     int iCur;            /* The VDBE cursor for the table */
  4162   3245     int addrNxt;         /* Where to jump to continue with the next IN case */
  4163   3246     int omitTable;       /* True if we use the index only */
  4164   3247     int bRev;            /* True if we need to scan in reverse order */
  4165   3248     WhereLevel *pLevel;  /* The where level to be coded */
         3249  +  WhereLoop *pLoop;    /* The WhereLoop object being coded */
  4166   3250     WhereClause *pWC;    /* Decomposition of the entire WHERE clause */
  4167   3251     WhereTerm *pTerm;               /* A WHERE clause term */
  4168   3252     Parse *pParse;                  /* Parsing context */
  4169   3253     Vdbe *v;                        /* The prepared stmt under constructions */
  4170   3254     struct SrcList_item *pTabItem;  /* FROM clause term being coded */
  4171   3255     int addrBrk;                    /* Jump here to break out of the loop */
  4172   3256     int addrCont;                   /* Jump here to continue with next cycle */
  4173   3257     int iRowidReg = 0;        /* Rowid is stored in this register, if not zero */
  4174   3258     int iReleaseReg = 0;      /* Temp register to free before returning */
  4175   3259     Bitmask newNotReady;      /* Return value */
  4176   3260   
  4177   3261     pParse = pWInfo->pParse;
  4178   3262     v = pParse->pVdbe;
  4179         -  pWC = pWInfo->pWC;
         3263  +  pWC = &pWInfo->sWC;
  4180   3264     pLevel = &pWInfo->a[iLevel];
         3265  +  pLoop = pLevel->pWLoop;
  4181   3266     pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
  4182   3267     iCur = pTabItem->iCursor;
  4183         -  bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0;
  4184         -  omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 
  4185         -           && (wctrlFlags & WHERE_FORCE_TABLE)==0;
         3268  +  bRev = (pWInfo->revMask>>iLevel)&1;
         3269  +  omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 
         3270  +           && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0;
  4186   3271     VdbeNoopComment((v, "Begin Join Loop %d", iLevel));
  4187   3272   
  4188   3273     /* Create labels for the "break" and "continue" instructions
  4189   3274     ** for the current loop.  Jump to addrBrk to break out of a loop.
  4190   3275     ** Jump to cont to go immediately to the next iteration of the
  4191   3276     ** loop.
  4192   3277     **
................................................................................
  4215   3300       pLevel->p2 =  sqlite3VdbeAddOp1(v, OP_Yield, regYield);
  4216   3301       VdbeComment((v, "next row of co-routine %s", pTabItem->pTab->zName));
  4217   3302       sqlite3VdbeAddOp2(v, OP_If, regYield+1, addrBrk);
  4218   3303       pLevel->op = OP_Goto;
  4219   3304     }else
  4220   3305   
  4221   3306   #ifndef SQLITE_OMIT_VIRTUALTABLE
  4222         -  if(  (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){
  4223         -    /* Case 0:  The table is a virtual-table.  Use the VFilter and VNext
         3307  +  if(  (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
         3308  +    /* Case 1:  The table is a virtual-table.  Use the VFilter and VNext
  4224   3309       **          to access the data.
  4225   3310       */
  4226   3311       int iReg;   /* P3 Value for OP_VFilter */
  4227   3312       int addrNotFound;
  4228         -    sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
  4229         -    int nConstraint = pVtabIdx->nConstraint;
  4230         -    struct sqlite3_index_constraint_usage *aUsage =
  4231         -                                                pVtabIdx->aConstraintUsage;
  4232         -    const struct sqlite3_index_constraint *aConstraint =
  4233         -                                                pVtabIdx->aConstraint;
         3313  +    int nConstraint = pLoop->nLTerm;
  4234   3314   
  4235   3315       sqlite3ExprCachePush(pParse);
  4236   3316       iReg = sqlite3GetTempRange(pParse, nConstraint+2);
  4237   3317       addrNotFound = pLevel->addrBrk;
  4238         -    for(j=1; j<=nConstraint; j++){
  4239         -      for(k=0; k<nConstraint; k++){
  4240         -        if( aUsage[k].argvIndex==j ){
  4241         -          int iTarget = iReg+j+1;
  4242         -          pTerm = &pWC->a[aConstraint[k].iTermOffset];
  4243         -          if( pTerm->eOperator & WO_IN ){
  4244         -            codeEqualityTerm(pParse, pTerm, pLevel, k, iTarget);
  4245         -            addrNotFound = pLevel->addrNxt;
  4246         -          }else{
  4247         -            sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget);
  4248         -          }
  4249         -          break;
  4250         -        }
  4251         -      }
  4252         -      if( k==nConstraint ) break;
  4253         -    }
  4254         -    sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg);
  4255         -    sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1);
  4256         -    sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pVtabIdx->idxStr,
  4257         -                      pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC);
  4258         -    pVtabIdx->needToFreeIdxStr = 0;
  4259   3318       for(j=0; j<nConstraint; j++){
  4260         -      if( aUsage[j].omit ){
  4261         -        int iTerm = aConstraint[j].iTermOffset;
  4262         -        disableTerm(pLevel, &pWC->a[iTerm]);
         3319  +      int iTarget = iReg+j+2;
         3320  +      pTerm = pLoop->aLTerm[j];
         3321  +      if( pTerm==0 ) continue;
         3322  +      if( pTerm->eOperator & WO_IN ){
         3323  +        codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget);
         3324  +        addrNotFound = pLevel->addrNxt;
         3325  +      }else{
         3326  +        sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget);
         3327  +      }
         3328  +    }
         3329  +    sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg);
         3330  +    sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1);
         3331  +    sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg,
         3332  +                      pLoop->u.vtab.idxStr,
         3333  +                      pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC);
         3334  +    pLoop->u.vtab.needFree = 0;
         3335  +    for(j=0; j<nConstraint && j<16; j++){
         3336  +      if( (pLoop->u.vtab.omitMask>>j)&1 ){
         3337  +        disableTerm(pLevel, pLoop->aLTerm[j]);
  4263   3338         }
  4264   3339       }
  4265   3340       pLevel->op = OP_VNext;
  4266   3341       pLevel->p1 = iCur;
  4267   3342       pLevel->p2 = sqlite3VdbeCurrentAddr(v);
  4268   3343       sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
  4269   3344       sqlite3ExprCachePop(pParse, 1);
  4270   3345     }else
  4271   3346   #endif /* SQLITE_OMIT_VIRTUALTABLE */
  4272   3347   
  4273         -  if( pLevel->plan.wsFlags & WHERE_ROWID_EQ ){
  4274         -    /* Case 1:  We can directly reference a single row using an
         3348  +  if( (pLoop->wsFlags & WHERE_IPK)!=0
         3349  +   && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0
         3350  +  ){
         3351  +    /* Case 2:  We can directly reference a single row using an
  4275   3352       **          equality comparison against the ROWID field.  Or
  4276   3353       **          we reference multiple rows using a "rowid IN (...)"
  4277   3354       **          construct.
  4278   3355       */
         3356  +    assert( pLoop->u.btree.nEq==1 );
  4279   3357       iReleaseReg = sqlite3GetTempReg(pParse);
  4280         -    pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0);
         3358  +    pTerm = pLoop->aLTerm[0];
  4281   3359       assert( pTerm!=0 );
  4282   3360       assert( pTerm->pExpr!=0 );
  4283   3361       assert( omitTable==0 );
  4284   3362       testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */
  4285         -    iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, iReleaseReg);
         3363  +    iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg);
  4286   3364       addrNxt = pLevel->addrNxt;
  4287   3365       sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt);
  4288   3366       sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg);
  4289   3367       sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1);
  4290   3368       sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
  4291   3369       VdbeComment((v, "pk"));
  4292   3370       pLevel->op = OP_Noop;
  4293         -  }else if( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ){
  4294         -    /* Case 2:  We have an inequality comparison against the ROWID field.
         3371  +  }else if( (pLoop->wsFlags & WHERE_IPK)!=0
         3372  +         && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0
         3373  +  ){
         3374  +    /* Case 3:  We have an inequality comparison against the ROWID field.
  4295   3375       */
  4296   3376       int testOp = OP_Noop;
  4297   3377       int start;
  4298   3378       int memEndValue = 0;
  4299   3379       WhereTerm *pStart, *pEnd;
  4300   3380   
  4301   3381       assert( omitTable==0 );
  4302         -    pStart = findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0);
  4303         -    pEnd = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0);
         3382  +    j = 0;
         3383  +    pStart = pEnd = 0;
         3384  +    if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++];
         3385  +    if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++];
         3386  +    assert( pStart!=0 || pEnd!=0 );
  4304   3387       if( bRev ){
  4305   3388         pTerm = pStart;
  4306   3389         pStart = pEnd;
  4307   3390         pEnd = pTerm;
  4308   3391       }
  4309   3392       if( pStart ){
  4310   3393         Expr *pX;             /* The expression that defines the start bound */
................................................................................
  4319   3402              /* TK_LT */  OP_SeekLt,
  4320   3403              /* TK_GE */  OP_SeekGe
  4321   3404         };
  4322   3405         assert( TK_LE==TK_GT+1 );      /* Make sure the ordering.. */
  4323   3406         assert( TK_LT==TK_GT+2 );      /*  ... of the TK_xx values... */
  4324   3407         assert( TK_GE==TK_GT+3 );      /*  ... is correcct. */
  4325   3408   
         3409  +      assert( (pStart->wtFlags & TERM_VNULL)==0 );
  4326   3410         testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */
  4327   3411         pX = pStart->pExpr;
  4328   3412         assert( pX!=0 );
  4329         -      assert( pStart->leftCursor==iCur );
         3413  +      testcase( pStart->leftCursor!=iCur ); /* transitive constraints */
  4330   3414         r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
  4331   3415         sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1);
  4332   3416         VdbeComment((v, "pk"));
  4333   3417         sqlite3ExprCacheAffinityChange(pParse, r1, 1);
  4334   3418         sqlite3ReleaseTempReg(pParse, rTemp);
  4335   3419         disableTerm(pLevel, pStart);
  4336   3420       }else{
  4337   3421         sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk);
  4338   3422       }
  4339   3423       if( pEnd ){
  4340   3424         Expr *pX;
  4341   3425         pX = pEnd->pExpr;
  4342   3426         assert( pX!=0 );
  4343         -      assert( pEnd->leftCursor==iCur );
         3427  +      assert( (pEnd->wtFlags & TERM_VNULL)==0 );
         3428  +      testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */
  4344   3429         testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */
  4345   3430         memEndValue = ++pParse->nMem;
  4346   3431         sqlite3ExprCode(pParse, pX->pRight, memEndValue);
  4347   3432         if( pX->op==TK_LT || pX->op==TK_GT ){
  4348   3433           testOp = bRev ? OP_Le : OP_Ge;
  4349   3434         }else{
  4350   3435           testOp = bRev ? OP_Lt : OP_Gt;
................................................................................
  4351   3436         }
  4352   3437         disableTerm(pLevel, pEnd);
  4353   3438       }
  4354   3439       start = sqlite3VdbeCurrentAddr(v);
  4355   3440       pLevel->op = bRev ? OP_Prev : OP_Next;
  4356   3441       pLevel->p1 = iCur;
  4357   3442       pLevel->p2 = start;
  4358         -    if( pStart==0 && pEnd==0 ){
  4359         -      pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
  4360         -    }else{
  4361         -      assert( pLevel->p5==0 );
  4362         -    }
         3443  +    assert( pLevel->p5==0 );
  4363   3444       if( testOp!=OP_Noop ){
  4364   3445         iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse);
  4365   3446         sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg);
  4366   3447         sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
  4367   3448         sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg);
  4368   3449         sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL);
  4369   3450       }
  4370         -  }else if( pLevel->plan.wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ) ){
  4371         -    /* Case 3: A scan using an index.
         3451  +  }else if( pLoop->wsFlags & WHERE_INDEXED ){
         3452  +    /* Case 4: A scan using an index.
  4372   3453       **
  4373   3454       **         The WHERE clause may contain zero or more equality 
  4374   3455       **         terms ("==" or "IN" operators) that refer to the N
  4375   3456       **         left-most columns of the index. It may also contain
  4376   3457       **         inequality constraints (>, <, >= or <=) on the indexed
  4377   3458       **         column that immediately follows the N equalities. Only 
  4378   3459       **         the right-most column can be an inequality - the rest must
................................................................................
  4410   3491         OP_SeekLe            /* 7: (start_constraints  &&  startEq &&  bRev) */
  4411   3492       };
  4412   3493       static const u8 aEndOp[] = {
  4413   3494         OP_Noop,             /* 0: (!end_constraints) */
  4414   3495         OP_IdxGE,            /* 1: (end_constraints && !bRev) */
  4415   3496         OP_IdxLT             /* 2: (end_constraints && bRev) */
  4416   3497       };
  4417         -    int nEq = pLevel->plan.nEq;  /* Number of == or IN terms */
  4418         -    int isMinQuery = 0;          /* If this is an optimized SELECT min(x).. */
         3498  +    int nEq = pLoop->u.btree.nEq;  /* Number of == or IN terms */
         3499  +    int isMinQuery = 0;            /* If this is an optimized SELECT min(x).. */
  4419   3500       int regBase;                 /* Base register holding constraint values */
  4420   3501       int r1;                      /* Temp register */
  4421   3502       WhereTerm *pRangeStart = 0;  /* Inequality constraint at range start */
  4422   3503       WhereTerm *pRangeEnd = 0;    /* Inequality constraint at range end */
  4423   3504       int startEq;                 /* True if range start uses ==, >= or <= */
  4424   3505       int endEq;                   /* True if range end uses ==, >= or <= */
  4425   3506       int start_constraints;       /* Start of range is constrained */
................................................................................
  4427   3508       Index *pIdx;                 /* The index we will be using */
  4428   3509       int iIdxCur;                 /* The VDBE cursor for the index */
  4429   3510       int nExtraReg = 0;           /* Number of extra registers needed */
  4430   3511       int op;                      /* Instruction opcode */
  4431   3512       char *zStartAff;             /* Affinity for start of range constraint */
  4432   3513       char *zEndAff;               /* Affinity for end of range constraint */
  4433   3514   
  4434         -    pIdx = pLevel->plan.u.pIdx;
         3515  +    pIdx = pLoop->u.btree.pIndex;
  4435   3516       iIdxCur = pLevel->iIdxCur;
  4436         -    k = (nEq==pIdx->nColumn ? -1 : pIdx->aiColumn[nEq]);
  4437   3517   
  4438   3518       /* If this loop satisfies a sort order (pOrderBy) request that 
  4439   3519       ** was passed to this function to implement a "SELECT min(x) ..." 
  4440   3520       ** query, then the caller will only allow the loop to run for
  4441   3521       ** a single iteration. This means that the first row returned
  4442   3522       ** should not have a NULL value stored in 'x'. If column 'x' is
  4443   3523       ** the first one after the nEq equality constraints in the index,
  4444   3524       ** this requires some special handling.
  4445   3525       */
  4446         -    if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0
  4447         -     && (pLevel->plan.wsFlags&WHERE_ORDERED)
         3526  +    if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0
         3527  +     && (pWInfo->bOBSat!=0)
  4448   3528        && (pIdx->nColumn>nEq)
  4449   3529       ){
  4450   3530         /* assert( pOrderBy->nExpr==1 ); */
  4451   3531         /* assert( pOrderBy->a[0].pExpr->iColumn==pIdx->aiColumn[nEq] ); */
  4452   3532         isMinQuery = 1;
  4453   3533         nExtraReg = 1;
  4454   3534       }
  4455   3535   
  4456   3536       /* Find any inequality constraint terms for the start and end 
  4457   3537       ** of the range. 
  4458   3538       */
  4459         -    if( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ){
  4460         -      pRangeEnd = findTerm(pWC, iCur, k, notReady, (WO_LT|WO_LE), pIdx);
         3539  +    j = nEq;
         3540  +    if( pLoop->wsFlags & WHERE_BTM_LIMIT ){
         3541  +      pRangeStart = pLoop->aLTerm[j++];
  4461   3542         nExtraReg = 1;
  4462   3543       }
  4463         -    if( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ){
  4464         -      pRangeStart = findTerm(pWC, iCur, k, notReady, (WO_GT|WO_GE), pIdx);
         3544  +    if( pLoop->wsFlags & WHERE_TOP_LIMIT ){
         3545  +      pRangeEnd = pLoop->aLTerm[j++];
  4465   3546         nExtraReg = 1;
  4466   3547       }
  4467   3548   
  4468   3549       /* Generate code to evaluate all constraint terms using == or IN
  4469   3550       ** and store the values of those terms in an array of registers
  4470   3551       ** starting at regBase.
  4471   3552       */
  4472         -    regBase = codeAllEqualityTerms(
  4473         -        pParse, pLevel, pWC, notReady, nExtraReg, &zStartAff
  4474         -    );
         3553  +    regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff);
  4475   3554       zEndAff = sqlite3DbStrDup(pParse->db, zStartAff);
  4476   3555       addrNxt = pLevel->addrNxt;
  4477   3556   
  4478   3557       /* If we are doing a reverse order scan on an ascending index, or
  4479   3558       ** a forward order scan on a descending index, interchange the 
  4480   3559       ** start and end terms (pRangeStart and pRangeEnd).
  4481   3560       */
  4482   3561       if( (nEq<pIdx->nColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC))
  4483   3562        || (bRev && pIdx->nColumn==nEq)
  4484   3563       ){
  4485   3564         SWAP(WhereTerm *, pRangeEnd, pRangeStart);
  4486   3565       }
  4487   3566   
  4488         -    testcase( pRangeStart && pRangeStart->eOperator & WO_LE );
  4489         -    testcase( pRangeStart && pRangeStart->eOperator & WO_GE );
  4490         -    testcase( pRangeEnd && pRangeEnd->eOperator & WO_LE );
  4491         -    testcase( pRangeEnd && pRangeEnd->eOperator & WO_GE );
         3567  +    testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 );
         3568  +    testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 );
         3569  +    testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 );
         3570  +    testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 );
  4492   3571       startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE);
  4493   3572       endEq =   !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE);
  4494   3573       start_constraints = pRangeStart || nEq>0;
  4495   3574   
  4496   3575       /* Seek the index cursor to the start of the range. */
  4497   3576       nConstraint = nEq;
  4498   3577       if( pRangeStart ){
................................................................................
  4574   3653       }
  4575   3654   
  4576   3655       /* If there are inequality constraints, check that the value
  4577   3656       ** of the table column that the inequality contrains is not NULL.
  4578   3657       ** If it is, jump to the next iteration of the loop.
  4579   3658       */
  4580   3659       r1 = sqlite3GetTempReg(pParse);
  4581         -    testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT );
  4582         -    testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT );
  4583         -    if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){
         3660  +    testcase( pLoop->wsFlags & WHERE_BTM_LIMIT );
         3661  +    testcase( pLoop->wsFlags & WHERE_TOP_LIMIT );
         3662  +    if( (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){
  4584   3663         sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1);
  4585   3664         sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont);
  4586   3665       }
  4587   3666       sqlite3ReleaseTempReg(pParse, r1);
  4588   3667   
  4589   3668       /* Seek the table cursor, if required */
  4590   3669       disableTerm(pLevel, pRangeStart);
................................................................................
  4595   3674         sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
  4596   3675         sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg);  /* Deferred seek */
  4597   3676       }
  4598   3677   
  4599   3678       /* Record the instruction used to terminate the loop. Disable 
  4600   3679       ** WHERE clause terms made redundant by the index range scan.
  4601   3680       */
  4602         -    if( pLevel->plan.wsFlags & WHERE_UNIQUE ){
         3681  +    if( pLoop->wsFlags & WHERE_ONEROW ){
  4603   3682         pLevel->op = OP_Noop;
  4604   3683       }else if( bRev ){
  4605   3684         pLevel->op = OP_Prev;
  4606   3685       }else{
  4607   3686         pLevel->op = OP_Next;
  4608   3687       }
  4609   3688       pLevel->p1 = iIdxCur;
  4610         -    if( pLevel->plan.wsFlags & WHERE_COVER_SCAN ){
         3689  +    if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){
  4611   3690         pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
  4612   3691       }else{
  4613   3692         assert( pLevel->p5==0 );
  4614   3693       }
  4615   3694     }else
  4616   3695   
  4617   3696   #ifndef SQLITE_OMIT_OR_OPTIMIZATION
  4618         -  if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){
  4619         -    /* Case 4:  Two or more separately indexed terms connected by OR
         3697  +  if( pLoop->wsFlags & WHERE_MULTI_OR ){
         3698  +    /* Case 5:  Two or more separately indexed terms connected by OR
  4620   3699       **
  4621   3700       ** Example:
  4622   3701       **
  4623   3702       **   CREATE TABLE t1(a,b,c,d);
  4624   3703       **   CREATE INDEX i1 ON t1(a);
  4625   3704       **   CREATE INDEX i2 ON t1(b);
  4626   3705       **   CREATE INDEX i3 ON t1(c);
................................................................................
  4665   3744       int regRowid = 0;                         /* Register holding rowid */
  4666   3745       int iLoopBody = sqlite3VdbeMakeLabel(v);  /* Start of loop body */
  4667   3746       int iRetInit;                             /* Address of regReturn init */
  4668   3747       int untestedTerms = 0;             /* Some terms not completely tested */
  4669   3748       int ii;                            /* Loop counter */
  4670   3749       Expr *pAndExpr = 0;                /* An ".. AND (...)" expression */
  4671   3750      
  4672         -    pTerm = pLevel->plan.u.pTerm;
         3751  +    pTerm = pLoop->aLTerm[0];
  4673   3752       assert( pTerm!=0 );
  4674   3753       assert( pTerm->eOperator & WO_OR );
  4675   3754       assert( (pTerm->wtFlags & TERM_ORINFO)!=0 );
  4676   3755       pOrWc = &pTerm->u.pOrInfo->wc;
  4677   3756       pLevel->op = OP_Return;
  4678   3757       pLevel->p1 = regReturn;
  4679   3758   
................................................................................
  4684   3763       if( pWInfo->nLevel>1 ){
  4685   3764         int nNotReady;                 /* The number of notReady tables */
  4686   3765         struct SrcList_item *origSrc;     /* Original list of tables */
  4687   3766         nNotReady = pWInfo->nLevel - iLevel - 1;
  4688   3767         pOrTab = sqlite3StackAllocRaw(pParse->db,
  4689   3768                               sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0]));
  4690   3769         if( pOrTab==0 ) return notReady;
  4691         -      pOrTab->nAlloc = (i16)(nNotReady + 1);
         3770  +      pOrTab->nAlloc = (u8)(nNotReady + 1);
  4692   3771         pOrTab->nSrc = pOrTab->nAlloc;
  4693   3772         memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem));
  4694   3773         origSrc = pWInfo->pTabList->a;
  4695   3774         for(k=1; k<=nNotReady; k++){
  4696   3775           memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k]));
  4697   3776         }
  4698   3777       }else{
................................................................................
  4706   3785       ** immediately following the OP_Return at the bottom of the loop. This
  4707   3786       ** is required in a few obscure LEFT JOIN cases where control jumps
  4708   3787       ** over the top of the loop into the body of it. In this case the 
  4709   3788       ** correct response for the end-of-loop code (the OP_Return) is to 
  4710   3789       ** fall through to the next instruction, just as an OP_Next does if
  4711   3790       ** called on an uninitialized cursor.
  4712   3791       */
  4713         -    if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
         3792  +    if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
  4714   3793         regRowset = ++pParse->nMem;
  4715   3794         regRowid = ++pParse->nMem;
  4716   3795         sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset);
  4717   3796       }
  4718   3797       iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn);
  4719   3798   
  4720   3799       /* If the original WHERE clause is z of the form:  (x1 OR x2 OR ...) AND y
................................................................................
  4731   3810       ** is not contained in the ON clause of a LEFT JOIN.
  4732   3811       ** See ticket http://www.sqlite.org/src/info/f2369304e4
  4733   3812       */
  4734   3813       if( pWC->nTerm>1 ){
  4735   3814         int iTerm;
  4736   3815         for(iTerm=0; iTerm<pWC->nTerm; iTerm++){
  4737   3816           Expr *pExpr = pWC->a[iTerm].pExpr;
         3817  +        if( &pWC->a[iTerm] == pTerm ) continue;
  4738   3818           if( ExprHasProperty(pExpr, EP_FromJoin) ) continue;
  4739         -        if( pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_ORINFO) ) continue;
         3819  +        if( pWC->a[iTerm].wtFlags & (TERM_ORINFO) ) continue;
  4740   3820           if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue;
  4741   3821           pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
  4742   3822           pAndExpr = sqlite3ExprAnd(pParse->db, pAndExpr, pExpr);
  4743   3823         }
  4744   3824         if( pAndExpr ){
  4745   3825           pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0);
  4746   3826         }
................................................................................
  4757   3837           }
  4758   3838           /* Loop through table entries that match term pOrTerm. */
  4759   3839           pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0,
  4760   3840                           WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY |
  4761   3841                           WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur);
  4762   3842           assert( pSubWInfo || pParse->nErr || pParse->db->mallocFailed );
  4763   3843           if( pSubWInfo ){
  4764         -          WhereLevel *pLvl;
         3844  +          WhereLoop *pSubLoop;
  4765   3845             explainOneScan(
  4766   3846                 pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
  4767   3847             );
  4768         -          if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
         3848  +          if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
  4769   3849               int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
  4770   3850               int r;
  4771   3851               r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, 
  4772   3852                                            regRowid, 0);
  4773   3853               sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset,
  4774   3854                                    sqlite3VdbeCurrentAddr(v)+2, r, iSet);
  4775   3855             }
................................................................................
  4790   3870             ** If the call to sqlite3WhereBegin() above resulted in a scan that
  4791   3871             ** uses an index, and this is either the first OR-connected term
  4792   3872             ** processed or the index is the same as that used by all previous
  4793   3873             ** terms, set pCov to the candidate covering index. Otherwise, set 
  4794   3874             ** pCov to NULL to indicate that no candidate covering index will 
  4795   3875             ** be available.
  4796   3876             */
  4797         -          pLvl = &pSubWInfo->a[0];
  4798         -          if( (pLvl->plan.wsFlags & WHERE_INDEXED)!=0
  4799         -           && (pLvl->plan.wsFlags & WHERE_TEMP_INDEX)==0
  4800         -           && (ii==0 || pLvl->plan.u.pIdx==pCov)
         3877  +          pSubLoop = pSubWInfo->a[0].pWLoop;
         3878  +          assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
         3879  +          if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0
         3880  +           && (ii==0 || pSubLoop->u.btree.pIndex==pCov)
  4801   3881             ){
  4802         -            assert( pLvl->iIdxCur==iCovCur );
  4803         -            pCov = pLvl->plan.u.pIdx;
         3882  +            assert( pSubWInfo->a[0].iIdxCur==iCovCur );
         3883  +            pCov = pSubLoop->u.btree.pIndex;
  4804   3884             }else{
  4805   3885               pCov = 0;
  4806   3886             }
  4807   3887   
  4808   3888             /* Finish the loop through table entries that match term pOrTerm. */
  4809   3889             sqlite3WhereEnd(pSubWInfo);
  4810   3890           }
................................................................................
  4822   3902   
  4823   3903       if( pWInfo->nLevel>1 ) sqlite3StackFree(pParse->db, pOrTab);
  4824   3904       if( !untestedTerms ) disableTerm(pLevel, pTerm);
  4825   3905     }else
  4826   3906   #endif /* SQLITE_OMIT_OR_OPTIMIZATION */
  4827   3907   
  4828   3908     {
  4829         -    /* Case 5:  There is no usable index.  We must do a complete
         3909  +    /* Case 6:  There is no usable index.  We must do a complete
  4830   3910       **          scan of the entire table.
  4831   3911       */
  4832   3912       static const u8 aStep[] = { OP_Next, OP_Prev };
  4833   3913       static const u8 aStart[] = { OP_Rewind, OP_Last };
  4834   3914       assert( bRev==0 || bRev==1 );
  4835         -    assert( omitTable==0 );
  4836   3915       pLevel->op = aStep[bRev];
  4837   3916       pLevel->p1 = iCur;
  4838   3917       pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
  4839   3918       pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
  4840   3919     }
  4841         -  newNotReady = notReady & ~getMask(pWC->pMaskSet, iCur);
         3920  +  newNotReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur);
  4842   3921   
  4843   3922     /* Insert code to test every subexpression that can be completely
  4844   3923     ** computed using the current set of tables.
  4845   3924     **
  4846   3925     ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through
  4847   3926     ** the use of indices become tests that are evaluated against each row of
  4848   3927     ** the relevant input tables.
................................................................................
  4878   3957     for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){
  4879   3958       Expr *pE;
  4880   3959       WhereTerm *pAlt;
  4881   3960       Expr sEq;
  4882   3961       if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
  4883   3962       if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue;
  4884   3963       if( pTerm->leftCursor!=iCur ) continue;
         3964  +    if( pLevel->iLeftJoin ) continue;
  4885   3965       pE = pTerm->pExpr;
  4886   3966       assert( !ExprHasProperty(pE, EP_FromJoin) );
  4887   3967       assert( (pTerm->prereqRight & newNotReady)!=0 );
  4888   3968       pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0);
  4889   3969       if( pAlt==0 ) continue;
  4890   3970       if( pAlt->wtFlags & (TERM_CODED) ) continue;
         3971  +    testcase( pAlt->eOperator & WO_EQ );
         3972  +    testcase( pAlt->eOperator & WO_IN );
  4891   3973       VdbeNoopComment((v, "begin transitive constraint"));
  4892   3974       sEq = *pAlt->pExpr;
  4893   3975       sEq.pLeft = pE->pLeft;
  4894   3976       sqlite3ExprIfFalse(pParse, &sEq, addrCont, SQLITE_JUMPIFNULL);
  4895   3977     }
  4896   3978   
  4897   3979     /* For a LEFT OUTER JOIN, generate code that will record the fact that
................................................................................
  4916   3998       }
  4917   3999     }
  4918   4000     sqlite3ReleaseTempReg(pParse, iReleaseReg);
  4919   4001   
  4920   4002     return newNotReady;
  4921   4003   }
  4922   4004   
  4923         -#if defined(SQLITE_TEST)
  4924         -/*
  4925         -** The following variable holds a text description of query plan generated
  4926         -** by the most recent call to sqlite3WhereBegin().  Each call to WhereBegin
  4927         -** overwrites the previous.  This information is used for testing and
  4928         -** analysis only.
  4929         -*/
  4930         -char sqlite3_query_plan[BMS*2*40];  /* Text of the join */
  4931         -static int nQPlan = 0;              /* Next free slow in _query_plan[] */
  4932         -
  4933         -#endif /* SQLITE_TEST */
  4934         -
         4005  +#ifdef WHERETRACE_ENABLED
         4006  +/*
         4007  +** Print a WhereLoop object for debugging purposes
         4008  +*/
         4009  +static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){
         4010  +  int nb = 1+(pTabList->nSrc+7)/8;
         4011  +  struct SrcList_item *pItem = pTabList->a + p->iTab;
         4012  +  Table *pTab = pItem->pTab;
         4013  +  sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
         4014  +                     p->iTab, nb, p->maskSelf, nb, p->prereq);
         4015  +  sqlite3DebugPrintf(" %12s",
         4016  +                     pItem->zAlias ? pItem->zAlias : pTab->zName);
         4017  +  if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
         4018  +    if( p->u.btree.pIndex ){
         4019  +      const char *zName = p->u.btree.pIndex->zName;
         4020  +      if( zName==0 ) zName = "ipk";
         4021  +      if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
         4022  +        int i = sqlite3Strlen30(zName) - 1;
         4023  +        while( zName[i]!='_' ) i--;
         4024  +        zName += i;
         4025  +      }
         4026  +      sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
         4027  +    }else{
         4028  +      sqlite3DebugPrintf("%20s","");
         4029  +    }
         4030  +  }else{
         4031  +    char *z;
         4032  +    if( p->u.vtab.idxStr ){
         4033  +      z = sqlite3_mprintf("(%d,\"%s\",%x)",
         4034  +                p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
         4035  +    }else{
         4036  +      z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
         4037  +    }
         4038  +    sqlite3DebugPrintf(" %-19s", z);
         4039  +    sqlite3_free(z);
         4040  +  }
         4041  +  sqlite3DebugPrintf(" f %04x N %d", p->wsFlags, p->nLTerm);
         4042  +  sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
         4043  +}
         4044  +#endif
         4045  +
         4046  +/*
         4047  +** Convert bulk memory into a valid WhereLoop that can be passed
         4048  +** to whereLoopClear harmlessly.
         4049  +*/
         4050  +static void whereLoopInit(WhereLoop *p){
         4051  +  p->aLTerm = p->aLTermSpace;
         4052  +  p->nLTerm = 0;
         4053  +  p->nLSlot = ArraySize(p->aLTermSpace);
         4054  +  p->wsFlags = 0;
         4055  +}
         4056  +
         4057  +/*
         4058  +** Clear the WhereLoop.u union.  Leave WhereLoop.pLTerm intact.
         4059  +*/
         4060  +static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
         4061  +  if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
         4062  +    if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
         4063  +      sqlite3_free(p->u.vtab.idxStr);
         4064  +      p->u.vtab.needFree = 0;
         4065  +      p->u.vtab.idxStr = 0;
         4066  +    }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
         4067  +      sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
         4068  +      sqlite3DbFree(db, p->u.btree.pIndex);
         4069  +      p->u.btree.pIndex = 0;
         4070  +    }
         4071  +  }
         4072  +}
         4073  +
         4074  +/*
         4075  +** Deallocate internal memory used by a WhereLoop object
         4076  +*/
         4077  +static void whereLoopClear(sqlite3 *db, WhereLoop *p){
         4078  +  if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm);
         4079  +  whereLoopClearUnion(db, p);
         4080  +  whereLoopInit(p);
         4081  +}
         4082  +
         4083  +/*
         4084  +** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
         4085  +*/
         4086  +static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
         4087  +  WhereTerm **paNew;
         4088  +  if( p->nLSlot>=n ) return SQLITE_OK;
         4089  +  n = (n+7)&~7;
         4090  +  paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n);
         4091  +  if( paNew==0 ) return SQLITE_NOMEM;
         4092  +  memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
         4093  +  if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm);
         4094  +  p->aLTerm = paNew;
         4095  +  p->nLSlot = n;
         4096  +  return SQLITE_OK;
         4097  +}
         4098  +
         4099  +/*
         4100  +** Transfer content from the second pLoop into the first.
         4101  +*/
         4102  +static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
         4103  +  if( whereLoopResize(db, pTo, pFrom->nLTerm) ) return SQLITE_NOMEM;
         4104  +  whereLoopClearUnion(db, pTo);
         4105  +  memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
         4106  +  memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
         4107  +  if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
         4108  +    pFrom->u.vtab.needFree = 0;
         4109  +  }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
         4110  +    pFrom->u.btree.pIndex = 0;
         4111  +  }
         4112  +  return SQLITE_OK;
         4113  +}
         4114  +
         4115  +/*
         4116  +** Delete a WhereLoop object
         4117  +*/
         4118  +static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
         4119  +  whereLoopClear(db, p);
         4120  +  sqlite3DbFree(db, p);
         4121  +}
  4935   4122   
  4936   4123   /*
  4937   4124   ** Free a WhereInfo structure
  4938   4125   */
  4939   4126   static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
  4940   4127     if( ALWAYS(pWInfo) ){
  4941         -    int i;
  4942         -    for(i=0; i<pWInfo->nLevel; i++){
  4943         -      sqlite3_index_info *pInfo = pWInfo->a[i].pIdxInfo;
  4944         -      if( pInfo ){
  4945         -        /* assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed ); */
  4946         -        if( pInfo->needToFreeIdxStr ){
  4947         -          sqlite3_free(pInfo->idxStr);
  4948         -        }
  4949         -        sqlite3DbFree(db, pInfo);
  4950         -      }
  4951         -      if( pWInfo->a[i].plan.wsFlags & WHERE_TEMP_INDEX ){
  4952         -        Index *pIdx = pWInfo->a[i].plan.u.pIdx;
  4953         -        if( pIdx ){
  4954         -          sqlite3DbFree(db, pIdx->zColAff);
  4955         -          sqlite3DbFree(db, pIdx);
  4956         -        }
  4957         -      }
         4128  +    whereClauseClear(&pWInfo->sWC);
         4129  +    while( pWInfo->pLoops ){
         4130  +      WhereLoop *p = pWInfo->pLoops;
         4131  +      pWInfo->pLoops = p->pNextLoop;
         4132  +      whereLoopDelete(db, p);
  4958   4133       }
  4959         -    whereClauseClear(pWInfo->pWC);
  4960   4134       sqlite3DbFree(db, pWInfo);
  4961   4135     }
  4962   4136   }
  4963   4137   
         4138  +/*
         4139  +** Insert or replace a WhereLoop entry using the template supplied.
         4140  +**
         4141  +** An existing WhereLoop entry might be overwritten if the new template
         4142  +** is better and has fewer dependencies.  Or the template will be ignored
         4143  +** and no insert will occur if an existing WhereLoop is faster and has
         4144  +** fewer dependencies than the template.  Otherwise a new WhereLoop is
         4145  +** added based on the template.
         4146  +**
         4147  +** If pBuilder->pOrSet is not NULL then we only care about only the
         4148  +** prerequisites and rRun and nOut costs of the N best loops.  That
         4149  +** information is gathered in the pBuilder->pOrSet object.  This special
         4150  +** processing mode is used only for OR clause processing.
         4151  +**
         4152  +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
         4153  +** still might overwrite similar loops with the new template if the
         4154  +** template is better.  Loops may be overwritten if the following 
         4155  +** conditions are met:
         4156  +**
         4157  +**    (1)  They have the same iTab.
         4158  +**    (2)  They have the same iSortIdx.
         4159  +**    (3)  The template has same or fewer dependencies than the current loop
         4160  +**    (4)  The template has the same or lower cost than the current loop
         4161  +**    (5)  The template uses more terms of the same index but has no additional
         4162  +**         dependencies          
         4163  +*/
         4164  +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
         4165  +  WhereLoop **ppPrev, *p, *pNext = 0;
         4166  +  WhereInfo *pWInfo = pBuilder->pWInfo;
         4167  +  sqlite3 *db = pWInfo->pParse->db;
         4168  +
         4169  +  /* If pBuilder->pOrSet is defined, then only keep track of the costs
         4170  +  ** and prereqs.
         4171  +  */
         4172  +  if( pBuilder->pOrSet!=0 ){
         4173  +#if WHERETRACE_ENABLED
         4174  +    u16 n = pBuilder->pOrSet->n;
         4175  +    int x =
         4176  +#endif
         4177  +    whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
         4178  +                                    pTemplate->nOut);
         4179  +#if WHERETRACE_ENABLED
         4180  +    if( sqlite3WhereTrace & 0x8 ){
         4181  +      sqlite3DebugPrintf(x?"   or-%d:  ":"   or-X:  ", n);
         4182  +      whereLoopPrint(pTemplate, pWInfo->pTabList);
         4183  +    }
         4184  +#endif
         4185  +    return SQLITE_OK;
         4186  +  }
         4187  +
         4188  +  /* Search for an existing WhereLoop to overwrite, or which takes
         4189  +  ** priority over pTemplate.
         4190  +  */
         4191  +  for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){
         4192  +    if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
         4193  +      /* If either the iTab or iSortIdx values for two WhereLoop are different
         4194  +      ** then those WhereLoops need to be considered separately.  Neither is
         4195  +      ** a candidate to replace the other. */
         4196  +      continue;
         4197  +    }
         4198  +    /* In the current implementation, the rSetup value is either zero
         4199  +    ** or the cost of building an automatic index (NlogN) and the NlogN
         4200  +    ** is the same for compatible WhereLoops. */
         4201  +    assert( p->rSetup==0 || pTemplate->rSetup==0 
         4202  +                 || p->rSetup==pTemplate->rSetup );
         4203  +
         4204  +    /* whereLoopAddBtree() always generates and inserts the automatic index
         4205  +    ** case first.  Hence compatible candidate WhereLoops never have a larger
         4206  +    ** rSetup. Call this SETUP-INVARIANT */
         4207  +    assert( p->rSetup>=pTemplate->rSetup );
         4208  +
         4209  +    if( (p->prereq & pTemplate->prereq)==p->prereq
         4210  +     && p->rSetup<=pTemplate->rSetup
         4211  +     && p->rRun<=pTemplate->rRun
         4212  +    ){
         4213  +      /* This branch taken when p is equal or better than pTemplate in 
         4214  +      ** all of (1) dependences (2) setup-cost, and (3) run-cost. */
         4215  +      assert( p->rSetup==pTemplate->rSetup );
         4216  +      if( p->nLTerm<pTemplate->nLTerm
         4217  +       && (p->wsFlags & WHERE_INDEXED)!=0
         4218  +       && (pTemplate->wsFlags & WHERE_INDEXED)!=0
         4219  +       && p->u.btree.pIndex==pTemplate->u.btree.pIndex
         4220  +       && p->prereq==pTemplate->prereq
         4221  +      ){
         4222  +        /* Overwrite an existing WhereLoop with an similar one that uses
         4223  +        ** more terms of the index */
         4224  +        pNext = p->pNextLoop;
         4225  +        break;
         4226  +      }else{
         4227  +        /* pTemplate is not helpful.
         4228  +        ** Return without changing or adding anything */
         4229  +        goto whereLoopInsert_noop;
         4230  +      }
         4231  +    }
         4232  +    if( (p->prereq & pTemplate->prereq)==pTemplate->prereq
         4233  +     && p->rRun>=pTemplate->rRun
         4234  +     && ALWAYS(p->rSetup>=pTemplate->rSetup) /* See SETUP-INVARIANT above */
         4235  +    ){
         4236  +      /* Overwrite an existing WhereLoop with a better one: one that is
         4237  +      ** better at one of (1) dependences, (2) setup-cost, or (3) run-cost
         4238  +      ** and is no worse in any of those categories. */
         4239  +      pNext = p->pNextLoop;
         4240  +      break;
         4241  +    }
         4242  +  }
         4243  +
         4244  +  /* If we reach this point it means that either p[] should be overwritten
         4245  +  ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
         4246  +  ** WhereLoop and insert it.
         4247  +  */
         4248  +#if WHERETRACE_ENABLED
         4249  +  if( sqlite3WhereTrace & 0x8 ){
         4250  +    if( p!=0 ){
         4251  +      sqlite3DebugPrintf("ins-del:  ");
         4252  +      whereLoopPrint(p, pWInfo->pTabList);
         4253  +    }
         4254  +    sqlite3DebugPrintf("ins-new:  ");
         4255  +    whereLoopPrint(pTemplate, pWInfo->pTabList);
         4256  +  }
         4257  +#endif
         4258  +  if( p==0 ){
         4259  +    p = sqlite3DbMallocRaw(db, sizeof(WhereLoop));
         4260  +    if( p==0 ) return SQLITE_NOMEM;
         4261  +    whereLoopInit(p);
         4262  +  }
         4263  +  whereLoopXfer(db, p, pTemplate);
         4264  +  p->pNextLoop = pNext;
         4265  +  *ppPrev = p;
         4266  +  if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
         4267  +    Index *pIndex = p->u.btree.pIndex;
         4268  +    if( pIndex && pIndex->tnum==0 ){
         4269  +      p->u.btree.pIndex = 0;
         4270  +    }
         4271  +  }
         4272  +  return SQLITE_OK;
         4273  +
         4274  +  /* Jump here if the insert is a no-op */
         4275  +whereLoopInsert_noop:
         4276  +#if WHERETRACE_ENABLED
         4277  +  if( sqlite3WhereTrace & 0x8 ){
         4278  +    sqlite3DebugPrintf("ins-noop: ");
         4279  +    whereLoopPrint(pTemplate, pWInfo->pTabList);
         4280  +  }
         4281  +#endif
         4282  +  return SQLITE_OK;  
         4283  +}
         4284  +
         4285  +/*
         4286  +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex.
         4287  +** Try to match one more.
         4288  +**
         4289  +** If pProbe->tnum==0, that means pIndex is a fake index used for the
         4290  +** INTEGER PRIMARY KEY.
         4291  +*/
         4292  +static int whereLoopAddBtreeIndex(
         4293  +  WhereLoopBuilder *pBuilder,     /* The WhereLoop factory */
         4294  +  struct SrcList_item *pSrc,      /* FROM clause term being analyzed */
         4295  +  Index *pProbe,                  /* An index on pSrc */
         4296  +  WhereCost nInMul                /* log(Number of iterations due to IN) */
         4297  +){
         4298  +  WhereInfo *pWInfo = pBuilder->pWInfo;  /* WHERE analyse context */
         4299  +  Parse *pParse = pWInfo->pParse;        /* Parsing context */
         4300  +  sqlite3 *db = pParse->db;       /* Database connection malloc context */
         4301  +  WhereLoop *pNew;                /* Template WhereLoop under construction */
         4302  +  WhereTerm *pTerm;               /* A WhereTerm under consideration */
         4303  +  int opMask;                     /* Valid operators for constraints */
         4304  +  WhereScan scan;                 /* Iterator for WHERE terms */
         4305  +  Bitmask saved_prereq;           /* Original value of pNew->prereq */
         4306  +  u16 saved_nLTerm;               /* Original value of pNew->nLTerm */
         4307  +  int saved_nEq;                  /* Original value of pNew->u.btree.nEq */
         4308  +  u32 saved_wsFlags;              /* Original value of pNew->wsFlags */
         4309  +  WhereCost saved_nOut;           /* Original value of pNew->nOut */
         4310  +  int iCol;                       /* Index of the column in the table */
         4311  +  int rc = SQLITE_OK;             /* Return code */
         4312  +  WhereCost nRowEst;              /* Estimated index selectivity */
         4313  +  WhereCost rLogSize;             /* Logarithm of table size */
         4314  +  WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
         4315  +
         4316  +  pNew = pBuilder->pNew;
         4317  +  if( db->mallocFailed ) return SQLITE_NOMEM;
         4318  +
         4319  +  assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
         4320  +  assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
         4321  +  if( pNew->wsFlags & WHERE_BTM_LIMIT ){
         4322  +    opMask = WO_LT|WO_LE;
         4323  +  }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){
         4324  +    opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE;
         4325  +  }else{
         4326  +    opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE;
         4327  +  }
         4328  +  if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
         4329  +
         4330  +  assert( pNew->u.btree.nEq<=pProbe->nColumn );
         4331  +  if( pNew->u.btree.nEq < pProbe->nColumn ){
         4332  +    iCol = pProbe->aiColumn[pNew->u.btree.nEq];
         4333  +    nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]);
         4334  +    if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1;
         4335  +  }else{
         4336  +    iCol = -1;
         4337  +    nRowEst = 0;
         4338  +  }
         4339  +  pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol,
         4340  +                        opMask, pProbe);
         4341  +  saved_nEq = pNew->u.btree.nEq;
         4342  +  saved_nLTerm = pNew->nLTerm;
         4343  +  saved_wsFlags = pNew->wsFlags;
         4344  +  saved_prereq = pNew->prereq;
         4345  +  saved_nOut = pNew->nOut;
         4346  +  pNew->rSetup = 0;
         4347  +  rLogSize = estLog(whereCost(pProbe->aiRowEst[0]));
         4348  +  for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
         4349  +    int nIn = 0;
         4350  +    if( pTerm->prereqRight & pNew->maskSelf ) continue;
         4351  +#ifdef SQLITE_ENABLE_STAT3
         4352  +    if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){
         4353  +      continue; /* skip IS NOT NULL constraints on a NOT NULL column */
         4354  +    }
         4355  +#endif
         4356  +    pNew->wsFlags = saved_wsFlags;
         4357  +    pNew->u.btree.nEq = saved_nEq;
         4358  +    pNew->nLTerm = saved_nLTerm;
         4359  +    if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
         4360  +    pNew->aLTerm[pNew->nLTerm++] = pTerm;
         4361  +    pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
         4362  +    pNew->rRun = rLogSize; /* Baseline cost is log2(N).  Adjustments below */
         4363  +    if( pTerm->eOperator & WO_IN ){
         4364  +      Expr *pExpr = pTerm->pExpr;
         4365  +      pNew->wsFlags |= WHERE_COLUMN_IN;
         4366  +      if( ExprHasProperty(pExpr, EP_xIsSelect) ){
         4367  +        /* "x IN (SELECT ...)":  TUNING: the SELECT returns 25 rows */
         4368  +        nIn = 46;  assert( 46==whereCost(25) );
         4369  +      }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
         4370  +        /* "x IN (value, value, ...)" */
         4371  +        nIn = whereCost(pExpr->x.pList->nExpr);
         4372  +      }
         4373  +      pNew->rRun += nIn;
         4374  +      pNew->u.btree.nEq++;
         4375  +      pNew->nOut = nRowEst + nInMul + nIn;
         4376  +    }else if( pTerm->eOperator & (WO_EQ) ){
         4377  +      assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0
         4378  +                  || nInMul==0 );
         4379  +      pNew->wsFlags |= WHERE_COLUMN_EQ;
         4380  +      if( iCol<0  
         4381  +       || (pProbe->onError!=OE_None && nInMul==0
         4382  +           && pNew->u.btree.nEq==pProbe->nColumn-1)
         4383  +      ){
         4384  +        assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 );
         4385  +        pNew->wsFlags |= WHERE_ONEROW;
         4386  +      }
         4387  +      pNew->u.btree.nEq++;
         4388  +      pNew->nOut = nRowEst + nInMul;
         4389  +    }else if( pTerm->eOperator & (WO_ISNULL) ){
         4390  +      pNew->wsFlags |= WHERE_COLUMN_NULL;
         4391  +      pNew->u.btree.nEq++;
         4392  +      /* TUNING: IS NULL selects 2 rows */
         4393  +      nIn = 10;  assert( 10==whereCost(2) );
         4394  +      pNew->nOut = nRowEst + nInMul + nIn;
         4395  +    }else if( pTerm->eOperator & (WO_GT|WO_GE) ){
         4396  +      testcase( pTerm->eOperator & WO_GT );
         4397  +      testcase( pTerm->eOperator & WO_GE );
         4398  +      pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
         4399  +      pBtm = pTerm;
         4400  +      pTop = 0;
         4401  +    }else{
         4402  +      assert( pTerm->eOperator & (WO_LT|WO_LE) );
         4403  +      testcase( pTerm->eOperator & WO_LT );
         4404  +      testcase( pTerm->eOperator & WO_LE );
         4405  +      pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
         4406  +      pTop = pTerm;
         4407  +      pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
         4408  +                     pNew->aLTerm[pNew->nLTerm-2] : 0;
         4409  +    }
         4410  +    if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
         4411  +      /* Adjust nOut and rRun for STAT3 range values */
         4412  +      WhereCost rDiv;
         4413  +      whereRangeScanEst(pParse, pProbe, pNew->u.btree.nEq,
         4414  +                        pBtm, pTop, &rDiv);
         4415  +      pNew->nOut = saved_nOut>rDiv+10 ? saved_nOut - rDiv : 10;
         4416  +    }
         4417  +#ifdef SQLITE_ENABLE_STAT3
         4418  +    if( pNew->u.btree.nEq==1 && pProbe->nSample
         4419  +     &&  OptimizationEnabled(db, SQLITE_Stat3) ){
         4420  +      tRowcnt nOut = 0;
         4421  +      if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){
         4422  +        testcase( pTerm->eOperator & WO_EQ );
         4423  +        testcase( pTerm->eOperator & WO_ISNULL );
         4424  +        rc = whereEqualScanEst(pParse, pProbe, pTerm->pExpr->pRight, &nOut);
         4425  +      }else if( (pTerm->eOperator & WO_IN)
         4426  +             &&  !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)  ){
         4427  +        rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut);
         4428  +      }
         4429  +      if( rc==SQLITE_OK ) pNew->nOut = whereCost(nOut);
         4430  +    }
         4431  +#endif
         4432  +    if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){
         4433  +      /* Each row involves a step of the index, then a binary search of
         4434  +      ** the main table */
         4435  +      pNew->rRun =  whereCostAdd(pNew->rRun, rLogSize>27 ? rLogSize-17 : 10);
         4436  +    }
         4437  +    /* Step cost for each output row */
         4438  +    pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut);
         4439  +    /* TBD: Adjust nOut for additional constraints */
         4440  +    rc = whereLoopInsert(pBuilder, pNew);
         4441  +    if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
         4442  +     && pNew->u.btree.nEq<(pProbe->nColumn + (pProbe->zName!=0))
         4443  +    ){
         4444  +      whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
         4445  +    }
         4446  +  }
         4447  +  pNew->prereq = saved_prereq;
         4448  +  pNew->u.btree.nEq = saved_nEq;
         4449  +  pNew->wsFlags = saved_wsFlags;
         4450  +  pNew->nOut = saved_nOut;
         4451  +  pNew->nLTerm = saved_nLTerm;
         4452  +  return rc;
         4453  +}
         4454  +
         4455  +/*
         4456  +** Return True if it is possible that pIndex might be useful in
         4457  +** implementing the ORDER BY clause in pBuilder.
         4458  +**
         4459  +** Return False if pBuilder does not contain an ORDER BY clause or
         4460  +** if there is no way for pIndex to be useful in implementing that
         4461  +** ORDER BY clause.
         4462  +*/
         4463  +static int indexMightHelpWithOrderBy(
         4464  +  WhereLoopBuilder *pBuilder,
         4465  +  Index *pIndex,
         4466  +  int iCursor
         4467  +){
         4468  +  ExprList *pOB;
         4469  +  int ii, jj;
         4470  +
         4471  +  if( pIndex->bUnordered ) return 0;
         4472  +  if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
         4473  +  for(ii=0; ii<pOB->nExpr; ii++){
         4474  +    Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr);
         4475  +    if( pExpr->op!=TK_COLUMN ) return 0;
         4476  +    if( pExpr->iTable==iCursor ){
         4477  +      for(jj=0; jj<pIndex->nColumn; jj++){
         4478  +        if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
         4479  +      }
         4480  +    }
         4481  +  }
         4482  +  return 0;
         4483  +}
         4484  +
         4485  +/*
         4486  +** Return a bitmask where 1s indicate that the corresponding column of
         4487  +** the table is used by an index.  Only the first 63 columns are considered.
         4488  +*/
         4489  +static Bitmask columnsInIndex(Index *pIdx){
         4490  +  Bitmask m = 0;
         4491  +  int j;
         4492  +  for(j=pIdx->nColumn-1; j>=0; j--){
         4493  +    int x = pIdx->aiColumn[j];
         4494  +    testcase( x==BMS-1 );
         4495  +    testcase( x==BMS-2 );
         4496  +    if( x<BMS-1 ) m |= MASKBIT(x);
         4497  +  }
         4498  +  return m;
         4499  +}
         4500  +
         4501  +
         4502  +/*
         4503  +** Add all WhereLoop objects for a single table of the join where the table
         4504  +** is idenfied by pBuilder->pNew->iTab.  That table is guaranteed to be
         4505  +** a b-tree table, not a virtual table.
         4506  +*/
         4507  +static int whereLoopAddBtree(
         4508  +  WhereLoopBuilder *pBuilder, /* WHERE clause information */
         4509  +  Bitmask mExtra              /* Extra prerequesites for using this table */
         4510  +){
         4511  +  WhereInfo *pWInfo;          /* WHERE analysis context */
         4512  +  Index *pProbe;              /* An index we are evaluating */
         4513  +  Index sPk;                  /* A fake index object for the primary key */
         4514  +  tRowcnt aiRowEstPk[2];      /* The aiRowEst[] value for the sPk index */
         4515  +  int aiColumnPk = -1;        /* The aColumn[] value for the sPk index */
         4516  +  SrcList *pTabList;          /* The FROM clause */
         4517  +  struct SrcList_item *pSrc;  /* The FROM clause btree term to add */
         4518  +  WhereLoop *pNew;            /* Template WhereLoop object */
         4519  +  int rc = SQLITE_OK;         /* Return code */
         4520  +  int iSortIdx = 1;           /* Index number */
         4521  +  int b;                      /* A boolean value */
         4522  +  WhereCost rSize;            /* number of rows in the table */
         4523  +  WhereCost rLogSize;         /* Logarithm of the number of rows in the table */
         4524  +  
         4525  +  pNew = pBuilder->pNew;
         4526  +  pWInfo = pBuilder->pWInfo;
         4527  +  pTabList = pWInfo->pTabList;
         4528  +  pSrc = pTabList->a + pNew->iTab;
         4529  +  assert( !IsVirtual(pSrc->pTab) );
         4530  +
         4531  +  if( pSrc->pIndex ){
         4532  +    /* An INDEXED BY clause specifies a particular index to use */
         4533  +    pProbe = pSrc->pIndex;
         4534  +  }else{
         4535  +    /* There is no INDEXED BY clause.  Create a fake Index object in local
         4536  +    ** variable sPk to represent the rowid primary key index.  Make this
         4537  +    ** fake index the first in a chain of Index objects with all of the real
         4538  +    ** indices to follow */
         4539  +    Index *pFirst;                  /* First of real indices on the table */
         4540  +    memset(&sPk, 0, sizeof(Index));
         4541  +    sPk.nColumn = 1;
         4542  +    sPk.aiColumn = &aiColumnPk;
         4543  +    sPk.aiRowEst = aiRowEstPk;
         4544  +    sPk.onError = OE_Replace;
         4545  +    sPk.pTable = pSrc->pTab;
         4546  +    aiRowEstPk[0] = pSrc->pTab->nRowEst;
         4547  +    aiRowEstPk[1] = 1;
         4548  +    pFirst = pSrc->pTab->pIndex;
         4549  +    if( pSrc->notIndexed==0 ){
         4550  +      /* The real indices of the table are only considered if the
         4551  +      ** NOT INDEXED qualifier is omitted from the FROM clause */
         4552  +      sPk.pNext = pFirst;
         4553  +    }
         4554  +    pProbe = &sPk;
         4555  +  }
         4556  +  rSize = whereCost(pSrc->pTab->nRowEst);
         4557  +  rLogSize = estLog(rSize);
         4558  +
         4559  +  /* Automatic indexes */
         4560  +  if( !pBuilder->pOrSet
         4561  +   && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
         4562  +   && pSrc->pIndex==0
         4563  +   && !pSrc->viaCoroutine
         4564  +   && !pSrc->notIndexed
         4565  +   && !pSrc->isCorrelated
         4566  +  ){
         4567  +    /* Generate auto-index WhereLoops */
         4568  +    WhereClause *pWC = pBuilder->pWC;
         4569  +    WhereTerm *pTerm;
         4570  +    WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
         4571  +    for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
         4572  +      if( pTerm->prereqRight & pNew->maskSelf ) continue;
         4573  +      if( termCanDriveIndex(pTerm, pSrc, 0) ){
         4574  +        pNew->u.btree.nEq = 1;
         4575  +        pNew->u.btree.pIndex = 0;
         4576  +        pNew->nLTerm = 1;
         4577  +        pNew->aLTerm[0] = pTerm;
         4578  +        /* TUNING: One-time cost for computing the automatic index is
         4579  +        ** approximately 7*N*log2(N) where N is the number of rows in
         4580  +        ** the table being indexed. */
         4581  +        pNew->rSetup = rLogSize + rSize + 28;  assert( 28==whereCost(7) );
         4582  +        /* TUNING: Each index lookup yields 20 rows in the table.  This
         4583  +        ** is more than the usual guess of 10 rows, since we have no way
         4584  +        ** of knowning how selective the index will ultimately be.  It would
         4585  +        ** not be unreasonable to make this value much larger. */
         4586  +        pNew->nOut = 43;  assert( 43==whereCost(20) );
         4587  +        pNew->rRun = whereCostAdd(rLogSize,pNew->nOut);
         4588  +        pNew->wsFlags = WHERE_AUTO_INDEX;
         4589  +        pNew->prereq = mExtra | pTerm->prereqRight;
         4590  +        rc = whereLoopInsert(pBuilder, pNew);
         4591  +      }
         4592  +    }
         4593  +  }
         4594  +
         4595  +  /* Loop over all indices
         4596  +  */
         4597  +  for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){
         4598  +    pNew->u.btree.nEq = 0;
         4599  +    pNew->nLTerm = 0;
         4600  +    pNew->iSortIdx = 0;
         4601  +    pNew->rSetup = 0;
         4602  +    pNew->prereq = mExtra;
         4603  +    pNew->nOut = rSize;
         4604  +    pNew->u.btree.pIndex = pProbe;
         4605  +    b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
         4606  +    /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
         4607  +    assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
         4608  +    if( pProbe->tnum<=0 ){
         4609  +      /* Integer primary key index */
         4610  +      pNew->wsFlags = WHERE_IPK;
         4611  +
         4612  +      /* Full table scan */
         4613  +      pNew->iSortIdx = b ? iSortIdx : 0;
         4614  +      /* TUNING: Cost of full table scan is 3*(N + log2(N)).
         4615  +      **  +  The extra 3 factor is to encourage the use of indexed lookups
         4616  +      **     over full scans.  A smaller constant 2 is used for covering
         4617  +      **     index scans so that a covering index scan will be favored over
         4618  +      **     a table scan. */
         4619  +      pNew->rRun = whereCostAdd(rSize,rLogSize) + 16;
         4620  +      rc = whereLoopInsert(pBuilder, pNew);
         4621  +      if( rc ) break;
         4622  +    }else{
         4623  +      Bitmask m = pSrc->colUsed & ~columnsInIndex(pProbe);
         4624  +      pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED;
         4625  +
         4626  +      /* Full scan via index */
         4627  +      if( b
         4628  +       || ( m==0
         4629  +         && pProbe->bUnordered==0
         4630  +         && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
         4631  +         && sqlite3GlobalConfig.bUseCis
         4632  +         && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
         4633  +          )
         4634  +      ){
         4635  +        pNew->iSortIdx = b ? iSortIdx : 0;
         4636  +        if( m==0 ){
         4637  +          /* TUNING: Cost of a covering index scan is 2*(N + log2(N)).
         4638  +          **  +  The extra 2 factor is to encourage the use of indexed lookups
         4639  +          **     over index scans.  A table scan uses a factor of 3 so that
         4640  +          **     index scans are favored over table scans.
         4641  +          **  +  If this covering index might also help satisfy the ORDER BY
         4642  +          **     clause, then the cost is fudged down slightly so that this
         4643  +          **     index is favored above other indices that have no hope of
         4644  +          **     helping with the ORDER BY. */
         4645  +          pNew->rRun = 10 + whereCostAdd(rSize,rLogSize) - b;
         4646  +        }else{
         4647  +          assert( b!=0 ); 
         4648  +          /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N)
         4649  +          ** which we will simplify to just N*log2(N) */
         4650  +          pNew->rRun = rSize + rLogSize;
         4651  +        }
         4652  +        rc = whereLoopInsert(pBuilder, pNew);
         4653  +        if( rc ) break;
         4654  +      }
         4655  +    }
         4656  +    rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
         4657  +
         4658  +    /* If there was an INDEXED BY clause, then only that one index is
         4659  +    ** considered. */
         4660  +    if( pSrc->pIndex ) break;
         4661  +  }
         4662  +  return rc;
         4663  +}
         4664  +
         4665  +#ifndef SQLITE_OMIT_VIRTUALTABLE
         4666  +/*
         4667  +** Add all WhereLoop objects for a table of the join identified by
         4668  +** pBuilder->pNew->iTab.  That table is guaranteed to be a virtual table.
         4669  +*/
         4670  +static int whereLoopAddVirtual(
         4671  +  WhereLoopBuilder *pBuilder   /* WHERE clause information */
         4672  +){
         4673  +  WhereInfo *pWInfo;           /* WHERE analysis context */
         4674  +  Parse *pParse;               /* The parsing context */
         4675  +  WhereClause *pWC;            /* The WHERE clause */
         4676  +  struct SrcList_item *pSrc;   /* The FROM clause term to search */
         4677  +  Table *pTab;
         4678  +  sqlite3 *db;
         4679  +  sqlite3_index_info *pIdxInfo;
         4680  +  struct sqlite3_index_constraint *pIdxCons;
         4681  +  struct sqlite3_index_constraint_usage *pUsage;
         4682  +  WhereTerm *pTerm;
         4683  +  int i, j;
         4684  +  int iTerm, mxTerm;
         4685  +  int nConstraint;
         4686  +  int seenIn = 0;              /* True if an IN operator is seen */
         4687  +  int seenVar = 0;             /* True if a non-constant constraint is seen */
         4688  +  int iPhase;                  /* 0: const w/o IN, 1: const, 2: no IN,  2: IN */
         4689  +  WhereLoop *pNew;
         4690  +  int rc = SQLITE_OK;
         4691  +
         4692  +  pWInfo = pBuilder->pWInfo;
         4693  +  pParse = pWInfo->pParse;
         4694  +  db = pParse->db;
         4695  +  pWC = pBuilder->pWC;
         4696  +  pNew = pBuilder->pNew;
         4697  +  pSrc = &pWInfo->pTabList->a[pNew->iTab];
         4698  +  pTab = pSrc->pTab;
         4699  +  assert( IsVirtual(pTab) );
         4700  +  pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy);
         4701  +  if( pIdxInfo==0 ) return SQLITE_NOMEM;
         4702  +  pNew->prereq = 0;
         4703  +  pNew->rSetup = 0;
         4704  +  pNew->wsFlags = WHERE_VIRTUALTABLE;
         4705  +  pNew->nLTerm = 0;
         4706  +  pNew->u.vtab.needFree = 0;
         4707  +  pUsage = pIdxInfo->aConstraintUsage;
         4708  +  nConstraint = pIdxInfo->nConstraint;
         4709  +  if( whereLoopResize(db, pNew, nConstraint) ){
         4710  +    sqlite3DbFree(db, pIdxInfo);
         4711  +    return SQLITE_NOMEM;
         4712  +  }
         4713  +
         4714  +  for(iPhase=0; iPhase<=3; iPhase++){
         4715  +    if( !seenIn && (iPhase&1)!=0 ){
         4716  +      iPhase++;
         4717  +      if( iPhase>3 ) break;
         4718  +    }
         4719  +    if( !seenVar && iPhase>1 ) break;
         4720  +    pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
         4721  +    for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){
         4722  +      j = pIdxCons->iTermOffset;
         4723  +      pTerm = &pWC->a[j];
         4724  +      switch( iPhase ){
         4725  +        case 0:    /* Constants without IN operator */
         4726  +          pIdxCons->usable = 0;
         4727  +          if( (pTerm->eOperator & WO_IN)!=0 ){
         4728  +            seenIn = 1;
         4729  +          }
         4730  +          if( pTerm->prereqRight!=0 ){
         4731  +            seenVar = 1;
         4732  +          }else if( (pTerm->eOperator & WO_IN)==0 ){
         4733  +            pIdxCons->usable = 1;
         4734  +          }
         4735  +          break;
         4736  +        case 1:    /* Constants with IN operators */
         4737  +          assert( seenIn );
         4738  +          pIdxCons->usable = (pTerm->prereqRight==0);
         4739  +          break;
         4740  +        case 2:    /* Variables without IN */
         4741  +          assert( seenVar );
         4742  +          pIdxCons->usable = (pTerm->eOperator & WO_IN)==0;
         4743  +          break;
         4744  +        default:   /* Variables with IN */
         4745  +          assert( seenVar && seenIn );
         4746  +          pIdxCons->usable = 1;
         4747  +          break;
         4748  +      }
         4749  +    }
         4750  +    memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint);
         4751  +    if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr);
         4752  +    pIdxInfo->idxStr = 0;
         4753  +    pIdxInfo->idxNum = 0;
         4754  +    pIdxInfo->needToFreeIdxStr = 0;
         4755  +    pIdxInfo->orderByConsumed = 0;
         4756  +    pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
         4757  +    rc = vtabBestIndex(pParse, pTab, pIdxInfo);
         4758  +    if( rc ) goto whereLoopAddVtab_exit;
         4759  +    pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
         4760  +    pNew->prereq = 0;
         4761  +    mxTerm = -1;
         4762  +    assert( pNew->nLSlot>=nConstraint );
         4763  +    for(i=0; i<nConstraint; i++) pNew->aLTerm[i] = 0;
         4764  +    pNew->u.vtab.omitMask = 0;
         4765  +    for(i=0; i<nConstraint; i++, pIdxCons++){
         4766  +      if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
         4767  +        j = pIdxCons->iTermOffset;
         4768  +        if( iTerm>=nConstraint
         4769  +         || j<0
         4770  +         || j>=pWC->nTerm
         4771  +         || pNew->aLTerm[iTerm]!=0
         4772  +        ){
         4773  +          rc = SQLITE_ERROR;
         4774  +          sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName);
         4775  +          goto whereLoopAddVtab_exit;
         4776  +        }
         4777  +        testcase( iTerm==nConstraint-1 );
         4778  +        testcase( j==0 );
         4779  +        testcase( j==pWC->nTerm-1 );
         4780  +        pTerm = &pWC->a[j];
         4781  +        pNew->prereq |= pTerm->prereqRight;
         4782  +        assert( iTerm<pNew->nLSlot );
         4783  +        pNew->aLTerm[iTerm] = pTerm;
         4784  +        if( iTerm>mxTerm ) mxTerm = iTerm;
         4785  +        testcase( iTerm==15 );
         4786  +        testcase( iTerm==16 );
         4787  +        if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<<iTerm;
         4788  +        if( (pTerm->eOperator & WO_IN)!=0 ){
         4789  +          if( pUsage[i].omit==0 ){
         4790  +            /* Do not attempt to use an IN constraint if the virtual table
         4791  +            ** says that the equivalent EQ constraint cannot be safely omitted.
         4792  +            ** If we do attempt to use such a constraint, some rows might be
         4793  +            ** repeated in the output. */
         4794  +            break;
         4795  +          }
         4796  +          /* A virtual table that is constrained by an IN clause may not
         4797  +          ** consume the ORDER BY clause because (1) the order of IN terms
         4798  +          ** is not necessarily related to the order of output terms and
         4799  +          ** (2) Multiple outputs from a single IN value will not merge
         4800  +          ** together.  */
         4801  +          pIdxInfo->orderByConsumed = 0;
         4802  +        }
         4803  +      }
         4804  +    }
         4805  +    if( i>=nConstraint ){
         4806  +      pNew->nLTerm = mxTerm+1;
         4807  +      assert( pNew->nLTerm<=pNew->nLSlot );
         4808  +      pNew->u.vtab.idxNum = pIdxInfo->idxNum;
         4809  +      pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
         4810  +      pIdxInfo->needToFreeIdxStr = 0;
         4811  +      pNew->u.vtab.idxStr = pIdxInfo->idxStr;
         4812  +      pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0)
         4813  +                                     && pIdxInfo->orderByConsumed);
         4814  +      pNew->rSetup = 0;
         4815  +      pNew->rRun = whereCostFromDouble(pIdxInfo->estimatedCost);
         4816  +      /* TUNING: Every virtual table query returns 25 rows */
         4817  +      pNew->nOut = 46;  assert( 46==whereCost(25) );
         4818  +      whereLoopInsert(pBuilder, pNew);
         4819  +      if( pNew->u.vtab.needFree ){
         4820  +        sqlite3_free(pNew->u.vtab.idxStr);
         4821  +        pNew->u.vtab.needFree = 0;
         4822  +      }
         4823  +    }
         4824  +  }  
         4825  +
         4826  +whereLoopAddVtab_exit:
         4827  +  if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr);
         4828  +  sqlite3DbFree(db, pIdxInfo);
         4829  +  return rc;
         4830  +}
         4831  +#endif /* SQLITE_OMIT_VIRTUALTABLE */
         4832  +
         4833  +/*
         4834  +** Add WhereLoop entries to handle OR terms.  This works for either
         4835  +** btrees or virtual tables.
         4836  +*/
         4837  +static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){
         4838  +  WhereInfo *pWInfo = pBuilder->pWInfo;
         4839  +  WhereClause *pWC;
         4840  +  WhereLoop *pNew;
         4841  +  WhereTerm *pTerm, *pWCEnd;
         4842  +  int rc = SQLITE_OK;
         4843  +  int iCur;
         4844  +  WhereClause tempWC;
         4845  +  WhereLoopBuilder sSubBuild;
         4846  +  WhereOrSet sSum, sCur, sPrev;
         4847  +  struct SrcList_item *pItem;
         4848  +  
         4849  +  pWC = pBuilder->pWC;
         4850  +  if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK;
         4851  +  pWCEnd = pWC->a + pWC->nTerm;
         4852  +  pNew = pBuilder->pNew;
         4853  +
         4854  +  for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
         4855  +    if( (pTerm->eOperator & WO_OR)!=0
         4856  +     && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 
         4857  +    ){
         4858  +      WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
         4859  +      WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
         4860  +      WhereTerm *pOrTerm;
         4861  +      int once = 1;
         4862  +      int i, j;
         4863  +    
         4864  +      pItem = pWInfo->pTabList->a + pNew->iTab;
         4865  +      iCur = pItem->iCursor;
         4866  +      sSubBuild = *pBuilder;
         4867  +      sSubBuild.pOrderBy = 0;
         4868  +      sSubBuild.pOrSet = &sCur;
         4869  +
         4870  +      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
         4871  +        if( (pOrTerm->eOperator & WO_AND)!=0 ){
         4872  +          sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
         4873  +        }else if( pOrTerm->leftCursor==iCur ){
         4874  +          tempWC.pWInfo = pWC->pWInfo;
         4875  +          tempWC.pOuter = pWC;
         4876  +          tempWC.op = TK_AND;
         4877  +          tempWC.nTerm = 1;
         4878  +          tempWC.a = pOrTerm;
         4879  +          sSubBuild.pWC = &tempWC;
         4880  +        }else{
         4881  +          continue;
         4882  +        }
         4883  +        sCur.n = 0;
         4884  +#ifndef SQLITE_OMIT_VIRTUALTABLE
         4885  +        if( IsVirtual(pItem->pTab) ){
         4886  +          rc = whereLoopAddVirtual(&sSubBuild);
         4887  +          for(i=0; i<sCur.n; i++) sCur.a[i].prereq |= mExtra;
         4888  +        }else
         4889  +#endif
         4890  +        {
         4891  +          rc = whereLoopAddBtree(&sSubBuild, mExtra);
         4892  +        }
         4893  +        assert( rc==SQLITE_OK || sCur.n==0 );
         4894  +        if( sCur.n==0 ){
         4895  +          sSum.n = 0;
         4896  +          break;
         4897  +        }else if( once ){
         4898  +          whereOrMove(&sSum, &sCur);
         4899  +          once = 0;
         4900  +        }else{
         4901  +          whereOrMove(&sPrev, &sSum);
         4902  +          sSum.n = 0;
         4903  +          for(i=0; i<sPrev.n; i++){
         4904  +            for(j=0; j<sCur.n; j++){
         4905  +              whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
         4906  +                            whereCostAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
         4907  +                            whereCostAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
         4908  +            }
         4909  +          }
         4910  +        }
         4911  +      }
         4912  +      pNew->nLTerm = 1;
         4913  +      pNew->aLTerm[0] = pTerm;
         4914  +      pNew->wsFlags = WHERE_MULTI_OR;
         4915  +      pNew->rSetup = 0;
         4916  +      pNew->iSortIdx = 0;
         4917  +      memset(&pNew->u, 0, sizeof(pNew->u));
         4918  +      for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
         4919  +        /* TUNING: Multiple by 3.5 for the secondary table lookup */
         4920  +        pNew->rRun = sSum.a[i].rRun + 18;
         4921  +        pNew->nOut = sSum.a[i].nOut;
         4922  +        pNew->prereq = sSum.a[i].prereq;
         4923  +        rc = whereLoopInsert(pBuilder, pNew);
         4924  +      }
         4925  +    }
         4926  +  }
         4927  +  return rc;
         4928  +}
         4929  +
         4930  +/*
         4931  +** Add all WhereLoop objects for all tables 
         4932  +*/
         4933  +static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
         4934  +  WhereInfo *pWInfo = pBuilder->pWInfo;
         4935  +  Bitmask mExtra = 0;
         4936  +  Bitmask mPrior = 0;
         4937  +  int iTab;
         4938  +  SrcList *pTabList = pWInfo->pTabList;
         4939  +  struct SrcList_item *pItem;
         4940  +  sqlite3 *db = pWInfo->pParse->db;
         4941  +  int nTabList = pWInfo->nLevel;
         4942  +  int rc = SQLITE_OK;
         4943  +  u8 priorJoinType = 0;
         4944  +  WhereLoop *pNew;
         4945  +
         4946  +  /* Loop over the tables in the join, from left to right */
         4947  +  pNew = pBuilder->pNew;
         4948  +  whereLoopInit(pNew);
         4949  +  for(iTab=0, pItem=pTabList->a; iTab<nTabList; iTab++, pItem++){
         4950  +    pNew->iTab = iTab;
         4951  +    pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor);
         4952  +    if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){
         4953  +      mExtra = mPrior;
         4954  +    }
         4955  +    priorJoinType = pItem->jointype;
         4956  +    if( IsVirtual(pItem->pTab) ){
         4957  +      rc = whereLoopAddVirtual(pBuilder);
         4958  +    }else{
         4959  +      rc = whereLoopAddBtree(pBuilder, mExtra);
         4960  +    }
         4961  +    if( rc==SQLITE_OK ){
         4962  +      rc = whereLoopAddOr(pBuilder, mExtra);
         4963  +    }
         4964  +    mPrior |= pNew->maskSelf;
         4965  +    if( rc || db->mallocFailed ) break;
         4966  +  }
         4967  +  whereLoopClear(db, pNew);
         4968  +  return rc;
         4969  +}
         4970  +
         4971  +/*
         4972  +** Examine a WherePath (with the addition of the extra WhereLoop of the 5th
         4973  +** parameters) to see if it outputs rows in the requested ORDER BY
         4974  +** (or GROUP BY) without requiring a separate sort operation.  Return:
         4975  +** 
         4976  +**    0:  ORDER BY is not satisfied.  Sorting required
         4977  +**    1:  ORDER BY is satisfied.      Omit sorting
         4978  +**   -1:  Unknown at this time
         4979  +**
         4980  +** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
         4981  +** strict.  With GROUP BY and DISTINCT the only requirement is that
         4982  +** equivalent rows appear immediately adjacent to one another.  GROUP BY
         4983  +** and DISTINT do not require rows to appear in any particular order as long
         4984  +** as equivelent rows are grouped together.  Thus for GROUP BY and DISTINCT
         4985  +** the pOrderBy terms can be matched in any order.  With ORDER BY, the 
         4986  +** pOrderBy terms must be matched in strict left-to-right order.
         4987  +*/
         4988  +static int wherePathSatisfiesOrderBy(
         4989  +  WhereInfo *pWInfo,    /* The WHERE clause */
         4990  +  ExprList *pOrderBy,   /* ORDER BY or GROUP BY or DISTINCT clause to check */
         4991  +  WherePath *pPath,     /* The WherePath to check */
         4992  +  u16 wctrlFlags,       /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */
         4993  +  u16 nLoop,            /* Number of entries in pPath->aLoop[] */
         4994  +  WhereLoop *pLast,     /* Add this WhereLoop to the end of pPath->aLoop[] */
         4995  +  Bitmask *pRevMask     /* OUT: Mask of WhereLoops to run in reverse order */
         4996  +){
         4997  +  u8 revSet;            /* True if rev is known */
         4998  +  u8 rev;               /* Composite sort order */
         4999  +  u8 revIdx;            /* Index sort order */
         5000  +  u8 isOrderDistinct;   /* All prior WhereLoops are order-distinct */
         5001  +  u8 distinctColumns;   /* True if the loop has UNIQUE NOT NULL columns */
         5002  +  u8 isMatch;           /* iColumn matches a term of the ORDER BY clause */
         5003  +  u16 nColumn;          /* Number of columns in pIndex */
         5004  +  u16 nOrderBy;         /* Number terms in the ORDER BY clause */
         5005  +  int iLoop;            /* Index of WhereLoop in pPath being processed */
         5006  +  int i, j;             /* Loop counters */
         5007  +  int iCur;             /* Cursor number for current WhereLoop */
         5008  +  int iColumn;          /* A column number within table iCur */
         5009  +  WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
         5010  +  WhereTerm *pTerm;     /* A single term of the WHERE clause */
         5011  +  Expr *pOBExpr;        /* An expression from the ORDER BY clause */
         5012  +  CollSeq *pColl;       /* COLLATE function from an ORDER BY clause term */
         5013  +  Index *pIndex;        /* The index associated with pLoop */
         5014  +  sqlite3 *db = pWInfo->pParse->db;  /* Database connection */
         5015  +  Bitmask obSat = 0;    /* Mask of ORDER BY terms satisfied so far */
         5016  +  Bitmask obDone;       /* Mask of all ORDER BY terms */
         5017  +  Bitmask orderDistinctMask;  /* Mask of all well-ordered loops */
         5018  +  Bitmask ready;              /* Mask of inner loops */
         5019  +
         5020  +  /*
         5021  +  ** We say the WhereLoop is "one-row" if it generates no more than one
         5022  +  ** row of output.  A WhereLoop is one-row if all of the following are true:
         5023  +  **  (a) All index columns match with WHERE_COLUMN_EQ.
         5024  +  **  (b) The index is unique
         5025  +  ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
         5026  +  ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
         5027  +  **
         5028  +  ** We say the WhereLoop is "order-distinct" if the set of columns from
         5029  +  ** that WhereLoop that are in the ORDER BY clause are different for every
         5030  +  ** row of the WhereLoop.  Every one-row WhereLoop is automatically
         5031  +  ** order-distinct.   A WhereLoop that has no columns in the ORDER BY clause
         5032  +  ** is not order-distinct. To be order-distinct is not quite the same as being
         5033  +  ** UNIQUE since a UNIQUE column or index can have multiple rows that 
         5034  +  ** are NULL and NULL values are equivalent for the purpose of order-distinct.
         5035  +  ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
         5036  +  **
         5037  +  ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
         5038  +  ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
         5039  +  ** automatically order-distinct.
         5040  +  */
         5041  +
         5042  +  assert( pOrderBy!=0 );
         5043  +
         5044  +  /* Sortability of virtual tables is determined by the xBestIndex method
         5045  +  ** of the virtual table itself */
         5046  +  if( pLast->wsFlags & WHERE_VIRTUALTABLE ){
         5047  +    testcase( nLoop>0 );  /* True when outer loops are one-row and match 
         5048  +                          ** no ORDER BY terms */
         5049  +    return pLast->u.vtab.isOrdered;
         5050  +  }
         5051  +  if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
         5052  +
         5053  +  nOrderBy = pOrderBy->nExpr;
         5054  +  testcase( nOrderBy==BMS-1 );
         5055  +  if( nOrderBy>BMS-1 ) return 0;  /* Cannot optimize overly large ORDER BYs */
         5056  +  isOrderDistinct = 1;
         5057  +  obDone = MASKBIT(nOrderBy)-1;
         5058  +  orderDistinctMask = 0;
         5059  +  ready = 0;
         5060  +  for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
         5061  +    if( iLoop>0 ) ready |= pLoop->maskSelf;
         5062  +    pLoop = iLoop<nLoop ? pPath->aLoop[iLoop] : pLast;
         5063  +    assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 );
         5064  +    iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
         5065  +
         5066  +    /* Mark off any ORDER BY term X that is a column in the table of
         5067  +    ** the current loop for which there is term in the WHERE
         5068  +    ** clause of the form X IS NULL or X=? that reference only outer
         5069  +    ** loops.
         5070  +    */
         5071  +    for(i=0; i<nOrderBy; i++){
         5072  +      if( MASKBIT(i) & obSat ) continue;
         5073  +      pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr);
         5074  +      if( pOBExpr->op!=TK_COLUMN ) continue;
         5075  +      if( pOBExpr->iTable!=iCur ) continue;
         5076  +      pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
         5077  +                       ~ready, WO_EQ|WO_ISNULL, 0);
         5078  +      if( pTerm==0 ) continue;
         5079  +      if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){
         5080  +        const char *z1, *z2;
         5081  +        pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
         5082  +        if( !pColl ) pColl = db->pDfltColl;
         5083  +        z1 = pColl->zName;
         5084  +        pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr);
         5085  +        if( !pColl ) pColl = db->pDfltColl;
         5086  +        z2 = pColl->zName;
         5087  +        if( sqlite3StrICmp(z1, z2)!=0 ) continue;
         5088  +      }
         5089  +      obSat |= MASKBIT(i);
         5090  +    }
         5091  +
         5092  +    if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
         5093  +      if( pLoop->wsFlags & WHERE_IPK ){
         5094  +        pIndex = 0;
         5095  +        nColumn = 0;
         5096  +      }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
         5097  +        return 0;
         5098  +      }else{
         5099  +        nColumn = pIndex->nColumn;
         5100  +        isOrderDistinct = pIndex->onError!=OE_None;
         5101  +      }
         5102  +
         5103  +      /* Loop through all columns of the index and deal with the ones
         5104  +      ** that are not constrained by == or IN.
         5105  +      */
         5106  +      rev = revSet = 0;
         5107  +      distinctColumns = 0;
         5108  +      for(j=0; j<=nColumn; j++){
         5109  +        u8 bOnce;   /* True to run the ORDER BY search loop */
         5110  +
         5111  +        /* Skip over == and IS NULL terms */
         5112  +        if( j<pLoop->u.btree.nEq
         5113  +         && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0
         5114  +        ){
         5115  +          if( i & WO_ISNULL ){
         5116  +            testcase( isOrderDistinct );
         5117  +            isOrderDistinct = 0;
         5118  +          }
         5119  +          continue;  
         5120  +        }
         5121  +
         5122  +        /* Get the column number in the table (iColumn) and sort order
         5123  +        ** (revIdx) for the j-th column of the index.
         5124  +        */
         5125  +        if( j<nColumn ){
         5126  +          /* Normal index columns */
         5127  +          iColumn = pIndex->aiColumn[j];
         5128  +          revIdx = pIndex->aSortOrder[j];
         5129  +          if( iColumn==pIndex->pTable->iPKey ) iColumn = -1;
         5130  +        }else{
         5131  +          /* The ROWID column at the end */
         5132  +          assert( j==nColumn );
         5133  +          iColumn = -1;
         5134  +          revIdx = 0;
         5135  +        }
         5136  +
         5137  +        /* An unconstrained column that might be NULL means that this
         5138  +        ** WhereLoop is not well-ordered 
         5139  +        */
         5140  +        if( isOrderDistinct
         5141  +         && iColumn>=0
         5142  +         && j>=pLoop->u.btree.nEq
         5143  +         && pIndex->pTable->aCol[iColumn].notNull==0
         5144  +        ){
         5145  +          isOrderDistinct = 0;
         5146  +        }
         5147  +
         5148  +        /* Find the ORDER BY term that corresponds to the j-th column
         5149  +        ** of the index and and mark that ORDER BY term off 
         5150  +        */
         5151  +        bOnce = 1;
         5152  +        isMatch = 0;
         5153  +        for(i=0; bOnce && i<nOrderBy; i++){
         5154  +          if( MASKBIT(i) & obSat ) continue;
         5155  +          pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr);
         5156  +          testcase( wctrlFlags & WHERE_GROUPBY );
         5157  +          testcase( wctrlFlags & WHERE_DISTINCTBY );
         5158  +          if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
         5159  +          if( pOBExpr->op!=TK_COLUMN ) continue;
         5160  +          if( pOBExpr->iTable!=iCur ) continue;
         5161  +          if( pOBExpr->iColumn!=iColumn ) continue;
         5162  +          if( iColumn>=0 ){
         5163  +            pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
         5164  +            if( !pColl ) pColl = db->pDfltColl;
         5165  +            if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
         5166  +          }
         5167  +          isMatch = 1;
         5168  +          break;
         5169  +        }
         5170  +        if( isMatch ){
         5171  +          if( iColumn<0 ){
         5172  +            testcase( distinctColumns==0 );
         5173  +            distinctColumns = 1;
         5174  +          }
         5175  +          obSat |= MASKBIT(i);
         5176  +          if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){
         5177  +            /* Make sure the sort order is compatible in an ORDER BY clause.
         5178  +            ** Sort order is irrelevant for a GROUP BY clause. */
         5179  +            if( revSet ){
         5180  +              if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) return 0;
         5181  +            }else{
         5182  +              rev = revIdx ^ pOrderBy->a[i].sortOrder;
         5183  +              if( rev ) *pRevMask |= MASKBIT(iLoop);
         5184  +              revSet = 1;
         5185  +            }
         5186  +          }
         5187  +        }else{
         5188  +          /* No match found */
         5189  +          if( j==0 || j<nColumn ){
         5190  +            testcase( isOrderDistinct!=0 );
         5191  +            isOrderDistinct = 0;
         5192  +          }
         5193  +          break;
         5194  +        }
         5195  +      } /* end Loop over all index columns */
         5196  +      if( distinctColumns ){
         5197  +        testcase( isOrderDistinct==0 );
         5198  +        isOrderDistinct = 1;
         5199  +      }
         5200  +    } /* end-if not one-row */
         5201  +
         5202  +    /* Mark off any other ORDER BY terms that reference pLoop */
         5203  +    if( isOrderDistinct ){
         5204  +      orderDistinctMask |= pLoop->maskSelf;
         5205  +      for(i=0; i<nOrderBy; i++){
         5206  +        Expr *p;
         5207  +        if( MASKBIT(i) & obSat ) continue;
         5208  +        p = pOrderBy->a[i].pExpr;
         5209  +        if( (exprTableUsage(&pWInfo->sMaskSet, p)&~orderDistinctMask)==0 ){
         5210  +          obSat |= MASKBIT(i);
         5211  +        }
         5212  +      }
         5213  +<